From ff4a0076a13f8c7653de0a84f38af1db76515e83 Mon Sep 17 00:00:00 2001 From: youben11 Date: Tue, 6 Dec 2022 08:07:07 +0100 Subject: [PATCH] ci: fix release tarball process add install target in Makefile to copy necessary libs, bins, and includes to an installation directory. Use this install target to package deps into a tarball with new installation instructions. --- .github/workflows/assets/Installation.md | 7 ++-- .github/workflows/continuous-integration.yml | 35 +++++++++++++++---- .../Dockerfile.release_tarball_linux_x86_64 | 24 ------------- compiler/Makefile | 23 +++++++++--- 4 files changed, 50 insertions(+), 39 deletions(-) delete mode 100644 builders/Dockerfile.release_tarball_linux_x86_64 diff --git a/.github/workflows/assets/Installation.md b/.github/workflows/assets/Installation.md index f5227ee8b..da3bcf67c 100644 --- a/.github/workflows/assets/Installation.md +++ b/.github/workflows/assets/Installation.md @@ -2,9 +2,10 @@ You can either install the compiler in user space or globally (you need root/sudo access): -1. User space install: extract the tarball to a chosen path and add chosen/path/concretecompiler/bin to your $PATH. +1. User space install: extract the tarball to a chosen path and make the lib, bin, and include directories accessible depending on your needs. 2. Global install: extract the tarball to a temporary path , and copy -- temporary/path/concretecompiler/bin/concretecompiler to /usr/bin (or a directory in $PATH) -- temporary/path/concretecompiler/lib/libConcretelangRuntime.so to /usr/lib (or another lib folder) +- temporary/path/concretecompiler/bin/* inside /usr/local/bin/ (or a directory in $PATH) +- temporary/path/concretecompiler/lib/* inside /usr/local/lib/ (or another lib folder) +- temporary/path/concretecompiler/include/* inside /usr/local/include/ (or another include folder) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index b7f356db7..e41db3317 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -481,12 +481,33 @@ jobs: - name: Login to Github Container Registry run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io - - name: Build - id: build-tarball - run: | - cd compiler - make release-tarballs + - name: Build Tarball + uses: addnab/docker-run-action@v3 + with: + registry: ghcr.io + image: ${{ env.DOCKER_IMAGE_TEST }} + username: ${{ secrets.GHCR_LOGIN }} + password: ${{ secrets.GHCR_PASSWORD }} + options: >- + -v ${{ github.workspace }}/llvm-project:/llvm-project + -v ${{ github.workspace }}/compiler:/compiler + -v ${{ github.workspace }}/tarballs:/tarballs + -v ${{ github.workspace }}/.github/workflows/assets/Installation.md:/Installation.md + shell: bash + run: | + set -e + cd /compiler + rm -rf /build + make BINDINGS_PYTHON_ENABLED=OFF BUILD_DIR=/build INSTALL_PREFIX=/tarballs/ install + echo "Debug: ccache statistics (after the build):" + ccache -s + # package installation file and make tarball + cp /Installation.md /tarballs/concretecompiler/ + cd /tarballs && tar -czvf concretecompiler.tar.gz concretecompiler + - name: Tag Tarball + id: tag-tarball + run: | TAG="$(git describe --tags --abbrev=0)" sudo cp "${{ github.workspace }}/tarballs/concretecompiler.tar.gz" "${{ github.workspace }}/tarballs/concretecompiler-${TAG}-x86_64-linux-gnu.tar.gz" @@ -498,8 +519,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GH_TOKEN_RELEASE }} with: upload_url: ${{ needs.CreateRelease.outputs.upload_url }} - asset_path: ${{ github.workspace }}/tarballs/${{ steps.build-tarball.outputs.ASSET_NAME }} - asset_name: ${{ steps.build-tarball.outputs.ASSET_NAME }} + asset_path: ${{ github.workspace }}/tarballs/${{ steps.tag-tarball.outputs.ASSET_NAME }} + asset_name: ${{ steps.tag-tarball.outputs.ASSET_NAME }} asset_content_type: application/tar+gzip BuildAndPushPackagesMacOS: diff --git a/builders/Dockerfile.release_tarball_linux_x86_64 b/builders/Dockerfile.release_tarball_linux_x86_64 deleted file mode 100644 index 9d3f72739..000000000 --- a/builders/Dockerfile.release_tarball_linux_x86_64 +++ /dev/null @@ -1,24 +0,0 @@ -FROM quay.io/pypa/manylinux_2_24_x86_64 - -RUN apt-get update -RUN DEBIAN_FRONTEND="noninteractive" apt-get install -y build-essential ninja-build -RUN curl https://sh.rustup.rs -sSf | sh -s -- -y -# Setup gcc7 -COPY --from=ghcr.io/zama-ai/gcc7:latest /gcc7 /gcc7 -ENV PATH=/gcc7/bin:$PATH -ENV LD_LIBRARY_PATH=/gcc7/lib/:/gcc7/lib64/:$LD_LIBRARY_PATH -ENV CC=/gcc7/bin/gcc-7.5.0 -ENV CXX=/gcc7/bin/g++-7.5.0 -# Setup LLVM -COPY /llvm-project /llvm-project -# Setup and build compiler -COPY .github/workflows/assets/Installation.md / -COPY /compiler /compiler -WORKDIR /compiler -RUN make -e CXX_COMPILER=$CXX CC_COMPILER=$CC BINDINGS_PYTHON_ENABLED=OFF concretecompiler -# Build tarball -RUN mkdir -p /tarballs/concretecompiler/lib /tarballs/concretecompiler/bin && \ - cp /compiler/build/bin/concretecompiler /tarballs/concretecompiler/bin && \ - cp /compiler/build/lib/libConcretelangRuntime.so /tarballs/concretecompiler/lib && \ - cp /Installation.md /tarballs/concretecompiler/ -RUN cd /tarballs && tar -czvf concretecompiler.tar.gz concretecompiler diff --git a/compiler/Makefile b/compiler/Makefile index 130dcf7c7..05c6dda67 100644 --- a/compiler/Makefile +++ b/compiler/Makefile @@ -8,6 +8,9 @@ CC_COMPILER= CXX_COMPILER= CUDA_SUPPORT?=OFF CONCRETE_CORE_PATH?= $(shell pwd)/concrete-core +INSTALL_PREFIX?=/usr/local/ +INSTALL_PATH=$(abspath $(INSTALL_PREFIX))/concretecompiler/ +MAKEFILE_ROOT_DIR=$(shell pwd) CONCRETE_OPTIMIZER_DIR ?= $(shell pwd)/concrete-optimizer @@ -383,10 +386,6 @@ mlir-opt: build-initialized mlir-translate: build-initialized cmake --build $(BUILD_DIR) --target mlir-translate -release-tarballs: - docker image build -t concrete-compiler-manylinux:linux_x86_64_tarball -f ../builders/Dockerfile.release_tarball_linux_x86_64 .. - docker container run --rm -v ${PWD}/../tarballs:/tarballs_volume concrete-compiler-manylinux:linux_x86_64_tarball cp -r /tarballs/. /tarballs_volume/. - update-python-version: echo "__version__ = \"`git describe --tags --abbrev=0 | grep -e '[0-9].*' -o`\"" > lib/Bindings/Python/version.txt @@ -405,6 +404,21 @@ check-rust-format: rust-format: cd lib/Bindings/Rust && cargo fmt +install: concretecompiler concrete-optimizer-lib CAPI + $(info Install prefix set to $(INSTALL_PREFIX)) + $(info Installing under $(INSTALL_PATH)) + mkdir -p $(INSTALL_PATH)/include + cp -R $(abspath $(BUILD_DIR))/bin $(INSTALL_PATH) + cp -R $(abspath $(BUILD_DIR))/lib $(INSTALL_PATH) + cp $(LIB_CONCRETE_OPTIMIZER_CPP) $(INSTALL_PATH)/lib/ + cp $(CONCRETE_OPTIMIZER_DIR)/concrete-optimizer-cpp/src/cpp/concrete-optimizer.hpp $(INSTALL_PATH)/include + cd $(MAKEFILE_ROOT_DIR)/include && find . -iregex '^.*\.\(h\|hpp\|td\)$$' -exec cp --parents {} $(INSTALL_PATH)/include \; + cd $(MAKEFILE_ROOT_DIR)/../llvm-project/llvm/include && find . -iregex '^.*\.\(h\|hpp\|td\)$$' -exec cp --parents {} $(INSTALL_PATH)/include \; + cd $(MAKEFILE_ROOT_DIR)/../llvm-project/mlir/include && find . -iregex '^.*\.\(h\|hpp\|td\)$$' -exec cp --parents {} $(INSTALL_PATH)/include \; + cd $(abspath $(BUILD_DIR))/include && find . -iname '*.inc' -exec cp --parents {} $(INSTALL_PATH)/include \; + cd $(abspath $(BUILD_DIR))/tools/concretelang/include && find . -iname '*.inc' -exec cp --parents {} $(INSTALL_PATH)/include \; + cd $(abspath $(BUILD_DIR))/tools/mlir/include && find . -iname '*.inc' -exec cp --parents {} $(INSTALL_PATH)/include \; + .PHONY: build-initialized \ build-end-to-end-jit \ concretecompiler \ @@ -412,7 +426,6 @@ rust-format: add-deps \ file-check \ not \ - release-tarballs \ update-python-version \ python-lint \ python-format \