From 8f05f9ea722c56068deef930a5fb68caa66f99d4 Mon Sep 17 00:00:00 2001 From: youben11 Date: Tue, 13 Sep 2022 16:32:44 +0100 Subject: [PATCH] ci: change package release process it should be now faster using the same compilation cache as the test build. --- .github/workflows/continuous-integration.yml | 62 +++++++++++++++---- .../Dockerfile.release_manylinux_2_28_x86_64 | 25 -------- compiler/Makefile | 24 ------- compiler/setup.py | 6 +- 4 files changed, 52 insertions(+), 65 deletions(-) delete mode 100644 builders/Dockerfile.release_manylinux_2_28_x86_64 diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 23f3942f0..d567fd8e6 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -360,12 +360,20 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python: [37, 38, 39, 310] + include: + - python: 37 + python_dir: "cp37-cp37m" + - python: 38 + python_dir: "cp38-cp38" + - python: 39 + python_dir: "cp39-cp39" + - python: 310 + python_dir: "cp310-cp310" outputs: - python-package-name-linux-py37: ${{ steps.build-wheel-linux.outputs.ASSET_NAME_PY37 }} - python-package-name-linux-py38: ${{ steps.build-wheel-linux.outputs.ASSET_NAME_PY38 }} - python-package-name-linux-py39: ${{ steps.build-wheel-linux.outputs.ASSET_NAME_PY39 }} - python-package-name-linux-py310: ${{ steps.build-wheel-linux.outputs.ASSET_NAME_PY310 }} + python-package-name-linux-py37: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME_PY37 }} + python-package-name-linux-py38: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME_PY38 }} + python-package-name-linux-py39: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME_PY39 }} + python-package-name-linux-py310: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME_PY310 }} needs: CreateRelease steps: # A SSH private key is required as some dependencies are from private repos @@ -388,14 +396,44 @@ jobs: cd compiler make concrete-optimizer-lib + - name: Update Python Version + run: cd compiler && make update-python-version + - name: Login to Github Container Registry run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io - - name: Build - id: build-wheel-linux + - name: Build Wheel + uses: addnab/docker-run-action@v3 + with: + registry: ghcr.io + image: ${{ env.DOCKER_IMAGE_TEST }} + username: ${{ secrets.GHCR_LOGIN }} + password: ${{ secrets.GHCR_PASSWORD }} + options: >- + -v ${{ github.workspace }}/llvm-project:/llvm-project + -v ${{ github.workspace }}/compiler:/compiler + -v ${{ github.workspace }}/wheels:/wheels + shell: bash + run: | + set -e + cd /compiler + rm -rf /build + export PYTHON_EXEC=/opt/python/${{ matrix.python_dir }}/bin/python + $PYTHON_EXEC -m pip install -r /llvm-project/mlir/python/requirements.txt + # setup env variable for wheel building + export CONCRETE_COMPILER_Python3_EXECUTABLE=$PYTHON_EXEC + export CONCRETE_COMPILER_BUILD_DIR=/build + /opt/python/${{ matrix.python_dir }}/bin/pip wheel -vvv --no-deps -w /wheels . + # We need to run it twice: the first will generate the directories, so that + # the second run can find the packages via find_namespace_packages + /opt/python/${{ matrix.python_dir }}/bin/pip wheel -vvv --no-deps -w /wheels . + auditwheel repair /wheels/*.whl --plat manylinux_2_28_x86_64 -w /wheels + echo "Debug: ccache statistics (after the build):" + ccache -s + + - name: Set Outputs + id: set-output-wheel-linux run: | - cd compiler - make package_py${{ matrix.python }} echo "::set-output name=ASSET_NAME::`find ${{ github.workspace }}/wheels/ -name *manylinux*.whl | rev |cut -d "/" -f 1 |rev `" # used later for python package test echo "::set-output name=ASSET_NAME_PY${{ matrix.python }}::`find ${{ github.workspace }}/wheels/ -name *manylinux*.whl | rev |cut -d "/" -f 1 |rev `" @@ -406,8 +444,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GH_TOKEN_RELEASE }} with: upload_url: ${{ needs.CreateRelease.outputs.upload_url }} - asset_path: ${{ github.workspace }}/wheels/${{ steps.build-wheel-linux.outputs.ASSET_NAME }} - asset_name: ${{ steps.build-wheel-linux.outputs.ASSET_NAME }} + asset_path: ${{ github.workspace }}/wheels/${{ steps.set-output-wheel-linux.outputs.ASSET_NAME }} + asset_name: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME }} asset_content_type: application/zip BuildAndPushTarballLinux: @@ -597,7 +635,7 @@ jobs: run: | cd compiler pip install pytest - pytest -vs -m "not parallel" tests/python + pytest -vs tests/python TestPythonPackageMacOS: runs-on: macos-11 diff --git a/builders/Dockerfile.release_manylinux_2_28_x86_64 b/builders/Dockerfile.release_manylinux_2_28_x86_64 deleted file mode 100644 index 97fdefd4e..000000000 --- a/builders/Dockerfile.release_manylinux_2_28_x86_64 +++ /dev/null @@ -1,25 +0,0 @@ -FROM quay.io/pypa/manylinux_2_28_x86_64 - -RUN dnf update -y -RUN dnf install -y ninja-build hwloc-devel -# Install boost -ADD https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/boost_1_71_0.tar.gz /boost_1_71_0.tar.gz -RUN tar -xzvf /boost_1_71_0.tar.gz -WORKDIR /boost_1_71_0 -RUN ./bootstrap.sh && ./b2 --with-filesystem install -# Set the python path. Options: [cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310] -ARG python_tag=cp38-cp38 -# Install python deps -RUN /opt/python/${python_tag}/bin/pip install numpy pybind11==2.8 PyYAML -# Setup HPX -COPY --from=ghcr.io/zama-ai/hpx:libc_2_28 /hpx /hpx -ENV HPX_INSTALL_DIR=/hpx/build -# Setup LLVM -COPY /llvm-project /llvm-project -# Setup and build compiler -COPY /compiler /compiler -WORKDIR /compiler -RUN make DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=/opt/python/${python_tag}/bin/python python-bindings -# Build wheel -RUN /opt/python/${python_tag}/bin/pip wheel --no-deps -w /wheels . -RUN auditwheel repair /wheels/*.whl --plat manylinux_2_28_x86_64 -w /wheels diff --git a/compiler/Makefile b/compiler/Makefile index ee84ac765..fdc922179 100644 --- a/compiler/Makefile +++ b/compiler/Makefile @@ -273,26 +273,6 @@ file-check: build-initialized not: build-initialized cmake --build $(BUILD_DIR) --target not -# Python packages - -define build_image_and_copy_wheels - docker image build -t concrete-compiler-manylinux:$(1) --build-arg python_tag=$(1) -f ../builders/Dockerfile.release_manylinux_2_28_x86_64 .. - docker container run --rm -v ${PWD}/../wheels:/wheels_volume concrete-compiler-manylinux:$(1) cp -r /wheels/. /wheels_volume/. -endef - - -package_py37: update_python_version - $(call build_image_and_copy_wheels,cp37-cp37m) - -package_py38: update_python_version - $(call build_image_and_copy_wheels,cp38-cp38) - -package_py39: update_python_version - $(call build_image_and_copy_wheels,cp39-cp39) - -package_py310: update_python_version - $(call build_image_and_copy_wheels,cp310-cp310) - release_tarballs: docker image build -t concrete-compiler-manylinux:linux_x86_64_tarball -f ../builders/Dockerfile.release_tarball_linux_x86_64 .. docker container run --rm -v ${PWD}/../tarballs:/tarballs_volume concrete-compiler-manylinux:linux_x86_64_tarball cp -r /tarballs/. /tarballs_volume/. @@ -316,10 +296,6 @@ python_lint: add-deps \ file-check \ not \ - package_py37 \ - package_py38 \ - package_py39 \ - package_py310 \ release_tarballs \ update_python_version \ python_lint \ diff --git a/compiler/setup.py b/compiler/setup.py index 8087b6ff5..6c16b857d 100644 --- a/compiler/setup.py +++ b/compiler/setup.py @@ -35,13 +35,11 @@ class MakeBuild(build_ext): self.build_extension(ext) def build_extension(self, ext): - cmd = ["make", f"BUILD_DIR={build_dir()}"] + cmd = ["make", "DATAFLOW_EXECUTION_ENABLED=ON", "CCACHE=ON"] py_exec = os.environ.get("CONCRETE_COMPILER_Python3_EXECUTABLE") if py_exec: cmd.append(f"Python3_EXECUTABLE={py_exec}") - ccache = os.environ.get("CONCRETE_COMPILER_CCACHE") - if ccache: - cmd.append(f"CCACHE={ccache}") + cmd.append(f"BUILD_DIR={build_dir()}") cmd.append("python-bindings") subprocess.check_call(cmd)