From 1b3f9aa0486f3156186a636fdd23676143e64670 Mon Sep 17 00:00:00 2001 From: Arthur Meyre Date: Mon, 15 Nov 2021 16:45:25 +0100 Subject: [PATCH] chore: use pip package for docker - comment out package-watcher as it's not compatible with the new docker waiting on the compiler proper release workflow to update this - update setup_env target to be docker and bare env friendly (workaround) - update torch_requirements.txt refs #809 --- .github/workflows/continuous-integration.yaml | 34 ++++- .github/workflows/package-watcher.yaml | 128 +++++++++--------- Makefile | 8 ++ docker/Dockerfile.concretefhe-env | 29 +++- .../Dockerfile.concretefhe-env.dockerignore | 1 + script/actions_utils/gh_dl_release.sh | 125 +++++++++++++++++ torch_requirements.txt | 12 +- 7 files changed, 260 insertions(+), 77 deletions(-) create mode 100644 docker/Dockerfile.concretefhe-env.dockerignore create mode 100755 script/actions_utils/gh_dl_release.sh diff --git a/.github/workflows/continuous-integration.yaml b/.github/workflows/continuous-integration.yaml index 9e3bcb7d9..79d3cef93 100644 --- a/.github/workflows/continuous-integration.yaml +++ b/.github/workflows/continuous-integration.yaml @@ -9,6 +9,11 @@ on: # Allows you to run this workflow manually from the Actions tab workflow_dispatch: + inputs: + rebuild-env-docker: + description: "String to indicate whether the docker should be rebuilt, true or false" + required: true + default: "false" # Allows external webhook trigger repository_dispatch: @@ -22,7 +27,7 @@ on: - cron: '0 22 * * 0' env: - FORCE_REBUILD_DOCKER: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'repository_dispatch' && github.event.action == 'rebuild-env-docker') }} + FORCE_REBUILD_DOCKER: ${{ (github.event_name == 'workflow_dispatch' && fromJSON(github.event.inputs.rebuild-env-docker)) || (github.event_name == 'repository_dispatch' && github.event.action == 'rebuild-env-docker') }} ENV_DOCKERFILE: docker/Dockerfile.concretefhe-env PREFLIGHT_IMAGE_BASE: ghcr.io/zama-ai/concretefhe-env:preflight LATEST_IMAGE: ghcr.io/zama-ai/concretefhe-env:latest @@ -40,6 +45,7 @@ jobs: runs-on: ubuntu-20.04 outputs: image: ${{ steps.set_image.outputs.image || env.LATEST_IMAGE }} + compiler-tag: ${{ steps.set_image.outputs.compiler-tag }} needs-push: ${{ env.BUILD_DOCKER }} force-rebuild-docker: ${{ env.FORCE_REBUILD_DOCKER }} report: ${{ steps.report.outputs.report || 'Did not run.' }} @@ -83,9 +89,13 @@ jobs: echo "Docker image up to date." echo "BUILD_DOCKER=false" >> "$GITHUB_ENV" fi - - name: Set prefligh Docker image + # https://github.com/zama-ai/concretefhe-internal/issues/809 + # Remove gh_dl_release call once package is on PyPi + - name: Set prefligh Docker image download compiler id: set_image if: ${{ fromJSON(env.BUILD_DOCKER) }} + env: + WHEEL: concretefhe_compiler-0.1.0-cp38-cp38-manylinux_2_24_x86_64.whl run: | PREFLIGHT_IMAGE_TAG=$(echo ${{ github.ref }} | sed -e 's/\//-/g') PREFLIGHT_IMAGE="${PREFLIGHT_IMAGE_BASE}-${PREFLIGHT_IMAGE_TAG}" @@ -93,6 +103,17 @@ jobs: echo "::set-output name=image::${PREFLIGHT_IMAGE}" echo "PREFLIGHT_IMAGE=${PREFLIGHT_IMAGE}" >> "$GITHUB_ENV" echo "LABEL_SHA1=${LABEL_SHA1}" >> "$GITHUB_ENV" + + mkdir -p pkg + ./script/actions_utils/gh_dl_release.sh \ + --token ${{ secrets.BOT_TOKEN }} \ + --org-repo zama-ai/homomorphizer \ + --file "${WHEEL}" \ + --dest-file "pkg/${WHEEL}" \ + --compiler-tag-output-file compiler-output-tag.txt + + COMPILER_TAG=$(cat compiler-output-tag.txt) + echo "::set-output name=compiler-tag::${COMPILER_TAG}" # Disabled buildx for now as we are seeing a lot of fails on layer pushes # - name: Set up Docker Buildx # if: ${{ fromJSON(env.BUILD_DOCKER) }} @@ -423,6 +444,7 @@ jobs: runs-on: ubuntu-20.04 env: PREFLIGHT_IMAGE: ${{ needs.build-preflight-docker.outputs.image }} + COMPILER_TAG: ${{ needs.build-preflight-docker.outputs.compiler-tag }} steps: - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 @@ -435,12 +457,12 @@ jobs: - name: Pull preflight image run: | docker pull "${PREFLIGHT_IMAGE}" - - name: Retag to latest and zamalang_sha1-concretefhe_sha1 and push + # https://github.com/zama-ai/concretefhe-internal/issues/809 + # update once release workflow is ok on the compiler side + - name: Retag to latest and concretefhe_compiler_version-concretefhe_sha1 and push run: | SHA1=$(git rev-parse HEAD) - ZAMALANG_SHA1=$(docker inspect "${PREFLIGHT_IMAGE}" | \ - jq -rc '.[0].Config.Labels["commit-sha"]') - TAGGED_IMAGE="${BASE_IMAGE}:${ZAMALANG_SHA1}-${SHA1}" + TAGGED_IMAGE="${BASE_IMAGE}:${COMPILER_TAG}-${SHA1}" docker tag "${PREFLIGHT_IMAGE}" "${LATEST_IMAGE}" docker tag "${PREFLIGHT_IMAGE}" "${TAGGED_IMAGE}" docker push "${LATEST_IMAGE}" diff --git a/.github/workflows/package-watcher.yaml b/.github/workflows/package-watcher.yaml index 034de6b29..6aad9cfe0 100644 --- a/.github/workflows/package-watcher.yaml +++ b/.github/workflows/package-watcher.yaml @@ -1,67 +1,69 @@ -name: Package Version Checker +# Remove completely ? Wait for https://github.com/zama-ai/concretefhe-internal/issues/809 -on: - schedule: - # * is a special character in YAML so you have to quote this string - # At minute 0 for each hour from 8:00 to 22:00 inclusive from Monday to Friday inclusive - # Timezone is UTC, so Paris time is +2 during the summer and +1 during winter - - cron: '0 6-20 * * 1-5' +# name: Package Version Checker - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: +# on: +# schedule: +# # * is a special character in YAML so you have to quote this string +# # At minute 0 for each hour from 8:00 to 22:00 inclusive from Monday to Friday inclusive +# # Timezone is UTC, so Paris time is +2 during the summer and +1 during winter +# - cron: '0 6-20 * * 1-5' -env: - ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} +# # Allows you to run this workflow manually from the Actions tab +# workflow_dispatch: -jobs: - check_and_notify_build: - name: Check timestamps and open PR - runs-on: ubuntu-20.04 - steps: - - name: Should run - run: | - SHOULD_RUN=${{ secrets.PACKAGE_WATCHER_ENABLED }} - if [[ "${SHOULD_RUN}" == "PACKAGE_WATCHER_PREFIX_ENABLED" ]]; then - SHOULD_RUN="true" - echo "Running package watcher" - else - SHOULD_RUN="false" - echo "Won't run package watcher" - fi - echo "SHOULD_RUN=${SHOULD_RUN}" >> "$GITHUB_ENV" - - name: Checkout Code - if: ${{ fromJSON(env.SHOULD_RUN) }} - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 - - name: Compare image timestamps and notify - if: ${{ fromJSON(env.SHOULD_RUN) }} - run: | - ./script/actions_utils/timestamp_check_update_container.sh \ - --base_img_url \ - https://api.github.com/orgs/zama-ai/packages/container/zamalang-compiler/versions \ - --env_img_url \ - https://api.github.com/orgs/zama-ai/packages/container/concretefhe-env/versions \ - --token ${{ secrets.BOT_TOKEN }} \ - --github-env "$GITHUB_ENV" - - name: Open PR - if: ${{ fromJSON(env.SHOULD_RUN) && env.NEW_BASE_IMG_TAG != '' }} - uses: peter-evans/create-pull-request@67df31e08a133c6a77008b89689677067fef169e - with: - token: ${{ secrets.BOT_TOKEN }} - commit-message: "chore(deps): bump compiler to ${{ env.NEW_BASE_IMG_TAG }}" - branch: chore/update-docker-env-compiler - base: main - title: "Docker env compiler update to ${{ env.NEW_BASE_IMG_TAG }}" - body: "Automatic PR for docker env compiler update" - labels: dependencies - - name: Send Slack Notification - if: ${{ always() && failure() }} - continue-on-error: true - uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7 - env: - SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} - SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png - SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "Package watcher finished with status ${{ job.status }} \ - (${{ env.ACTION_RUN_URL }})" - SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} +# env: +# ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + +# jobs: +# check_and_notify_build: +# name: Check timestamps and open PR +# runs-on: ubuntu-20.04 +# steps: +# - name: Should run +# run: | +# SHOULD_RUN=${{ secrets.PACKAGE_WATCHER_ENABLED }} +# if [[ "${SHOULD_RUN}" == "PACKAGE_WATCHER_PREFIX_ENABLED" ]]; then +# SHOULD_RUN="true" +# echo "Running package watcher" +# else +# SHOULD_RUN="false" +# echo "Won't run package watcher" +# fi +# echo "SHOULD_RUN=${SHOULD_RUN}" >> "$GITHUB_ENV" +# - name: Checkout Code +# if: ${{ fromJSON(env.SHOULD_RUN) }} +# uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 +# - name: Compare image timestamps and notify +# if: ${{ fromJSON(env.SHOULD_RUN) }} +# run: | +# ./script/actions_utils/timestamp_check_update_container.sh \ +# --base_img_url \ +# https://api.github.com/orgs/zama-ai/packages/container/zamalang-compiler/versions \ +# --env_img_url \ +# https://api.github.com/orgs/zama-ai/packages/container/concretefhe-env/versions \ +# --token ${{ secrets.BOT_TOKEN }} \ +# --github-env "$GITHUB_ENV" +# - name: Open PR +# if: ${{ fromJSON(env.SHOULD_RUN) && env.NEW_BASE_IMG_TAG != '' }} +# uses: peter-evans/create-pull-request@67df31e08a133c6a77008b89689677067fef169e +# with: +# token: ${{ secrets.BOT_TOKEN }} +# commit-message: "chore(deps): bump compiler to ${{ env.NEW_BASE_IMG_TAG }}" +# branch: chore/update-docker-env-compiler +# base: main +# title: "Docker env compiler update to ${{ env.NEW_BASE_IMG_TAG }}" +# body: "Automatic PR for docker env compiler update" +# labels: dependencies +# - name: Send Slack Notification +# if: ${{ always() && failure() }} +# continue-on-error: true +# uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7 +# env: +# SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} +# SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png +# SLACK_COLOR: ${{ job.status }} +# SLACK_MESSAGE: "Package watcher finished with status ${{ job.status }} \ +# (${{ env.ACTION_RUN_URL }})" +# SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} +# SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} diff --git a/Makefile b/Makefile index 130c0c76d..f3ed80b81 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,14 @@ setup_env: poetry install poetry run python -m pip install -r torch_requirements.txt \ -f https://download.pytorch.org/whl/torch_stable.html + @# This is required to be friendly in the docker and on bare systems until the package is on pip + @# https://github.com/zama-ai/concretefhe-internal/issues/809 + if [[ -d /pkg ]]; then \ + NUM_PKG=$$(ls /pkg | wc -l); \ + if [[ "$${NUM_PKG}" != "0" ]]; then \ + poetry run python -m pip install /pkg/*.whl; \ + fi; \ + fi .PHONY: setup_env sync_env: diff --git a/docker/Dockerfile.concretefhe-env b/docker/Dockerfile.concretefhe-env index 049d101b0..d93e01207 100644 --- a/docker/Dockerfile.concretefhe-env +++ b/docker/Dockerfile.concretefhe-env @@ -1,18 +1,43 @@ -FROM ghcr.io/zama-ai/zamalang-compiler:3bf493e83cd9d5be127adce4a4bde9dd4be229f1 +FROM ubuntu:20.04 + +# Remove once compiler is on PyPi https://github.com/zama-ai/concretefhe-internal/issues/809 +ARG WHEEL +ENV TZ=Europe/Paris +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone RUN apt-get update && apt-get upgrade --no-install-recommends -y && \ apt-get install --no-install-recommends -y \ + build-essential \ curl \ + python3-pip \ python3.8 \ + python3.8-dev \ python3.8-tk \ python3.8-venv \ python-is-python3 \ git \ graphviz* \ + make \ pandoc \ shellcheck && \ rm -rf /var/lib/apt/lists/* && \ pip install --no-cache-dir --upgrade pip && \ pip install --no-cache-dir poetry -ENV LD_PRELOAD=${RT_LIB}:${LD_PRELOAD} +# Remove once compiler is on PyPi https://github.com/zama-ai/concretefhe-internal/issues/809 +WORKDIR /pkg +COPY pkg/${WHEEL} . + +ENV SRC_DIR_NAME=src + +RUN echo "source /${SRC_DIR_NAME}/.docker_venv/bin/activate" >> /root/.bashrc && \ + echo "if [[ \"\$?\" != \"0\" ]]; then" >> /root/.bashrc && \ + echo " python3 -m venv /${SRC_DIR_NAME}/.docker_venv" >> /root/.bashrc && \ + echo " source /${SRC_DIR_NAME}/.docker_venv/bin/activate" >> /root/.bashrc && \ + echo " cd /${SRC_DIR_NAME}/ && make setup_env" >> /root/.bashrc && \ + echo "fi" >> /root/.bashrc && \ + echo "export MPLBACKEND=TkAgg" >> /root/.bashrc + +WORKDIR /${SRC_DIR_NAME} + +CMD ["/bin/bash"] diff --git a/docker/Dockerfile.concretefhe-env.dockerignore b/docker/Dockerfile.concretefhe-env.dockerignore new file mode 100644 index 000000000..12baba301 --- /dev/null +++ b/docker/Dockerfile.concretefhe-env.dockerignore @@ -0,0 +1 @@ +!script/actions_utils/gh_dl_release.sh diff --git a/script/actions_utils/gh_dl_release.sh b/script/actions_utils/gh_dl_release.sh new file mode 100755 index 000000000..9107bfc65 --- /dev/null +++ b/script/actions_utils/gh_dl_release.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# +# Adapted from https://gist.github.com/maxim/6e15aa45ba010ab030c4 +# +# gh-dl-release! It works! +# +# This script downloads an asset from latest or specific Github release of a +# private repo. Feel free to extract more of the variables into command line +# parameters. +# +# PREREQUISITES +# +# curl, wget, jq +# +# USAGE +# +# Set all the variables inside the script, make sure you chmod +x it, then +# to download specific version to my_app.tar.gz: +# +# gh-dl-release 2.1.1 my_app.tar.gz +# +# to download latest version: +# +# gh-dl-release latest latest.tar.gz +# +# If your version/tag doesn't match, the script will exit with error. + +TOKEN= +ORG_REPO= +# the name of your release asset file, e.g. build.tar.gz +FILE= +DEST_FILE= +VERSION="latest" +COMPILER_TAG_OUTPUT_FILE=debug.txt + +while [ -n "$1" ] +do + case "$1" in + "--token" ) + shift + TOKEN="$1" + ;; + + "--org-repo" ) + shift + ORG_REPO="$1" + ;; + + "--version" ) + shift + VERSION="$1" + ;; + + "--dest-file" ) + shift + DEST_FILE="$1" + ;; + + "--file" ) + shift + FILE="$1" + ;; + + "--compiler-tag-output-file") + shift + COMPILER_TAG_OUTPUT_FILE="$1" + ;; + + *) + echo "Unknown param : $1" + exit 1 + ;; + esac + shift +done + +alias errcho='>&2 echo' + +DEST_DIR=$(dirname "${DEST_FILE}") +mkdir -p "${DEST_DIR}" + +if [[ "${VERSION}" == "latest" ]]; then + # Github should return the latest release first. + jq_parser=".[0]" +else + jq_parser=". | map(select(.tag_name == \"${VERSION}\"))[0]" +fi; + +release_json=$(curl -H "Authorization: token ${TOKEN}" \ +-H "Accept: application/vnd.github.v3.raw" \ +"https://api.github.com/repos/${ORG_REPO}/releases" | jq "${jq_parser}") + +echo "Release json:" +echo "${release_json}" + +asset_json=$(echo "${release_json}" | jq ".assets | map(select(.name | contains(\"${FILE}\")))[0]") + +echo "Asset json:" +echo "${asset_json}" + +release_tag=$(echo "${release_json}" | jq -rc '.tag_name') +asset_id=$(echo "${asset_json}" | jq -rc '.id') + +release_tag="${release_tag//-/_}" + +echo "Release tag: ${release_tag}" +echo "Asset id: ${asset_id}" + +if [[ "${asset_id}" == "null" ]]; then + errcho "ERROR: version not found ${VERSION}" + exit 1 +fi + +echo "Downloading..." + +wget --auth-no-challenge --header='Accept:application/octet-stream' \ + "https://${TOKEN}:@api.github.com/repos/${ORG_REPO}/releases/assets/${asset_id}" \ + -O "${DEST_FILE}" + +err_code=$? + +echo "Done." +echo "CONCRETEFHE_COMPILER_TAG=${release_tag}" >> "${COMPILER_TAG_OUTPUT_FILE}" + +exit "${err_code}" diff --git a/torch_requirements.txt b/torch_requirements.txt index e3d601f0c..7c50dd4cf 100644 --- a/torch_requirements.txt +++ b/torch_requirements.txt @@ -1,7 +1,7 @@ -torch==1.9.0; sys_platform=="darwin" -torch==1.9.0+cpu; sys_platform=="linux" -torch==1.9.0+cpu; sys_platform=="win32" +torch==1.10.0; sys_platform=="darwin" +torch==1.10.0+cpu; sys_platform=="linux" +torch==1.10.0+cpu; sys_platform=="win32" -torchvision==0.10.0; sys_platform=="darwin" -torchvision==0.10.0+cpu; sys_platform=="linux" -torchvision==0.10.0+cpu; sys_platform=="win32" +torchvision==0.11.1; sys_platform=="darwin" +torchvision==0.11.1+cpu; sys_platform=="linux" +torchvision==0.11.1+cpu; sys_platform=="win32"