chore: use pip package for docker

- comment out package-watcher as it's not compatible with the new docker waiting on the compiler proper release workflow to update this - update setup_env target to be docker and bare env friendly (workaround) - update torch_requirements.txt refs #809
2026-02-08 19:44:57 -05:00 · 2021-11-15 16:45:25 +01:00
parent 46a018fd21
commit 1b3f9aa048
7 changed files with 260 additions and 77 deletions
--- a/.github/workflows/continuous-integration.yaml
+++ b/.github/workflows/continuous-integration.yaml
@@ -9,6 +9,11 @@ on:

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:
+    inputs:
+      rebuild-env-docker:
+        description: "String to indicate whether the docker should be rebuilt, true or false"
+        required: true
+        default: "false"

  # Allows external webhook trigger
  repository_dispatch:
@@ -22,7 +27,7 @@ on:
    - cron: '0 22 * * 0'

 env:
-  FORCE_REBUILD_DOCKER: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'repository_dispatch' && github.event.action == 'rebuild-env-docker') }}
+  FORCE_REBUILD_DOCKER: ${{ (github.event_name == 'workflow_dispatch' && fromJSON(github.event.inputs.rebuild-env-docker)) || (github.event_name == 'repository_dispatch' && github.event.action == 'rebuild-env-docker') }}
  ENV_DOCKERFILE: docker/Dockerfile.concretefhe-env
  PREFLIGHT_IMAGE_BASE: ghcr.io/zama-ai/concretefhe-env:preflight
  LATEST_IMAGE: ghcr.io/zama-ai/concretefhe-env:latest
@@ -40,6 +45,7 @@ jobs:
    runs-on: ubuntu-20.04
    outputs:
      image: ${{ steps.set_image.outputs.image || env.LATEST_IMAGE }}
+      compiler-tag: ${{ steps.set_image.outputs.compiler-tag }}
      needs-push: ${{ env.BUILD_DOCKER }}
      force-rebuild-docker: ${{ env.FORCE_REBUILD_DOCKER }}
      report: ${{ steps.report.outputs.report || 'Did not run.' }}
@@ -83,9 +89,13 @@ jobs:
            echo "Docker image up to date."
            echo "BUILD_DOCKER=false" >> "$GITHUB_ENV"
          fi
-      - name: Set prefligh Docker image
+      # https://github.com/zama-ai/concretefhe-internal/issues/809
+      # Remove gh_dl_release call once package is on PyPi
+      - name: Set prefligh Docker image download compiler
        id: set_image
        if: ${{ fromJSON(env.BUILD_DOCKER) }}
+        env:
+          WHEEL: concretefhe_compiler-0.1.0-cp38-cp38-manylinux_2_24_x86_64.whl
        run: |
          PREFLIGHT_IMAGE_TAG=$(echo ${{ github.ref }} | sed -e 's/\//-/g')
          PREFLIGHT_IMAGE="${PREFLIGHT_IMAGE_BASE}-${PREFLIGHT_IMAGE_TAG}"
@@ -93,6 +103,17 @@ jobs:
          echo "::set-output name=image::${PREFLIGHT_IMAGE}"
          echo "PREFLIGHT_IMAGE=${PREFLIGHT_IMAGE}" >> "$GITHUB_ENV"
          echo "LABEL_SHA1=${LABEL_SHA1}" >> "$GITHUB_ENV"
+
+          mkdir -p pkg
+          ./script/actions_utils/gh_dl_release.sh \
+          --token ${{ secrets.BOT_TOKEN }} \
+          --org-repo zama-ai/homomorphizer \
+          --file "${WHEEL}" \
+          --dest-file "pkg/${WHEEL}" \
+          --compiler-tag-output-file compiler-output-tag.txt
+
+          COMPILER_TAG=$(cat compiler-output-tag.txt)
+          echo "::set-output name=compiler-tag::${COMPILER_TAG}"
      # Disabled buildx for now as we are seeing a lot of fails on layer pushes
      # - name: Set up Docker Buildx
      #   if: ${{ fromJSON(env.BUILD_DOCKER) }}
@@ -423,6 +444,7 @@ jobs:
    runs-on: ubuntu-20.04
    env:
      PREFLIGHT_IMAGE: ${{ needs.build-preflight-docker.outputs.image }}
+      COMPILER_TAG: ${{ needs.build-preflight-docker.outputs.compiler-tag }}

    steps:
      - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579
@@ -435,12 +457,12 @@ jobs:
      - name: Pull preflight image
        run: |
          docker pull "${PREFLIGHT_IMAGE}"
-      - name: Retag to latest and zamalang_sha1-concretefhe_sha1 and push
+      # https://github.com/zama-ai/concretefhe-internal/issues/809
+      # update once release workflow is ok on the compiler side
+      - name: Retag to latest and concretefhe_compiler_version-concretefhe_sha1 and push
        run: |
          SHA1=$(git rev-parse HEAD)
-          ZAMALANG_SHA1=$(docker inspect "${PREFLIGHT_IMAGE}" | \
-          jq -rc '.[0].Config.Labels["commit-sha"]')
-          TAGGED_IMAGE="${BASE_IMAGE}:${ZAMALANG_SHA1}-${SHA1}"
+          TAGGED_IMAGE="${BASE_IMAGE}:${COMPILER_TAG}-${SHA1}"
          docker tag "${PREFLIGHT_IMAGE}" "${LATEST_IMAGE}"
          docker tag "${PREFLIGHT_IMAGE}" "${TAGGED_IMAGE}"
          docker push "${LATEST_IMAGE}"
--- a/.github/workflows/package-watcher.yaml
+++ b/.github/workflows/package-watcher.yaml
@@ -1,67 +1,69 @@
-name: Package Version Checker
+# Remove completely ? Wait for https://github.com/zama-ai/concretefhe-internal/issues/809

-on:
-  schedule:
-    # * is a special character in YAML so you have to quote this string
-    # At minute 0 for each hour from 8:00 to 22:00 inclusive from Monday to Friday inclusive
-    # Timezone is UTC, so Paris time is +2 during the summer and +1 during winter
-    - cron: '0 6-20 * * 1-5'
+# name: Package Version Checker

-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
+# on:
+#   schedule:
+#     # * is a special character in YAML so you have to quote this string
+#     # At minute 0 for each hour from 8:00 to 22:00 inclusive from Monday to Friday inclusive
+#     # Timezone is UTC, so Paris time is +2 during the summer and +1 during winter
+#     - cron: '0 6-20 * * 1-5'

-env:
-  ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+#   # Allows you to run this workflow manually from the Actions tab
+#   workflow_dispatch:

-jobs:
-  check_and_notify_build:
-    name: Check timestamps and open PR
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Should run
-        run: |
-          SHOULD_RUN=${{ secrets.PACKAGE_WATCHER_ENABLED }}
-          if [[ "${SHOULD_RUN}" == "PACKAGE_WATCHER_PREFIX_ENABLED" ]]; then
-            SHOULD_RUN="true"
-            echo "Running package watcher"
-          else
-            SHOULD_RUN="false"
-            echo "Won't run package watcher"
-          fi
-          echo "SHOULD_RUN=${SHOULD_RUN}" >> "$GITHUB_ENV"
-      - name: Checkout Code
-        if: ${{ fromJSON(env.SHOULD_RUN) }}
-        uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579
-      - name: Compare image timestamps and notify
-        if: ${{ fromJSON(env.SHOULD_RUN) }}
-        run: |
-          ./script/actions_utils/timestamp_check_update_container.sh \
-          --base_img_url \
-          https://api.github.com/orgs/zama-ai/packages/container/zamalang-compiler/versions \
-          --env_img_url \
-          https://api.github.com/orgs/zama-ai/packages/container/concretefhe-env/versions \
-          --token ${{ secrets.BOT_TOKEN }} \
-          --github-env "$GITHUB_ENV"
-      - name: Open PR
-        if: ${{ fromJSON(env.SHOULD_RUN) && env.NEW_BASE_IMG_TAG != '' }}
-        uses: peter-evans/create-pull-request@67df31e08a133c6a77008b89689677067fef169e
-        with:
-          token: ${{ secrets.BOT_TOKEN }}
-          commit-message: "chore(deps): bump compiler to ${{ env.NEW_BASE_IMG_TAG }}"
-          branch: chore/update-docker-env-compiler
-          base: main
-          title: "Docker env compiler update to ${{ env.NEW_BASE_IMG_TAG }}"
-          body: "Automatic PR for docker env compiler update"
-          labels: dependencies
-      - name: Send Slack Notification
-        if: ${{ always() && failure() }}
-        continue-on-error: true
-        uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7
-        env:
-          SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
-          SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
-          SLACK_COLOR: ${{ job.status }}
-          SLACK_MESSAGE: "Package watcher finished with status ${{ job.status }} \
-            (${{ env.ACTION_RUN_URL }})"
-          SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
-          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+# env:
+#   ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+
+# jobs:
+#   check_and_notify_build:
+#     name: Check timestamps and open PR
+#     runs-on: ubuntu-20.04
+#     steps:
+#       - name: Should run
+#         run: |
+#           SHOULD_RUN=${{ secrets.PACKAGE_WATCHER_ENABLED }}
+#           if [[ "${SHOULD_RUN}" == "PACKAGE_WATCHER_PREFIX_ENABLED" ]]; then
+#             SHOULD_RUN="true"
+#             echo "Running package watcher"
+#           else
+#             SHOULD_RUN="false"
+#             echo "Won't run package watcher"
+#           fi
+#           echo "SHOULD_RUN=${SHOULD_RUN}" >> "$GITHUB_ENV"
+#       - name: Checkout Code
+#         if: ${{ fromJSON(env.SHOULD_RUN) }}
+#         uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579
+#       - name: Compare image timestamps and notify
+#         if: ${{ fromJSON(env.SHOULD_RUN) }}
+#         run: |
+#           ./script/actions_utils/timestamp_check_update_container.sh \
+#           --base_img_url \
+#           https://api.github.com/orgs/zama-ai/packages/container/zamalang-compiler/versions \
+#           --env_img_url \
+#           https://api.github.com/orgs/zama-ai/packages/container/concretefhe-env/versions \
+#           --token ${{ secrets.BOT_TOKEN }} \
+#           --github-env "$GITHUB_ENV"
+#       - name: Open PR
+#         if: ${{ fromJSON(env.SHOULD_RUN) && env.NEW_BASE_IMG_TAG != '' }}
+#         uses: peter-evans/create-pull-request@67df31e08a133c6a77008b89689677067fef169e
+#         with:
+#           token: ${{ secrets.BOT_TOKEN }}
+#           commit-message: "chore(deps): bump compiler to ${{ env.NEW_BASE_IMG_TAG }}"
+#           branch: chore/update-docker-env-compiler
+#           base: main
+#           title: "Docker env compiler update to ${{ env.NEW_BASE_IMG_TAG }}"
+#           body: "Automatic PR for docker env compiler update"
+#           labels: dependencies
+#       - name: Send Slack Notification
+#         if: ${{ always() && failure() }}
+#         continue-on-error: true
+#         uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7
+#         env:
+#           SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
+#           SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
+#           SLACK_COLOR: ${{ job.status }}
+#           SLACK_MESSAGE: "Package watcher finished with status ${{ job.status }} \
+#             (${{ env.ACTION_RUN_URL }})"
+#           SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
+#           SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
--- a/8
+++ b/8
@@ -11,6 +11,14 @@ setup_env:
 	poetry install
 	poetry run python -m pip install -r torch_requirements.txt \
 		-f https://download.pytorch.org/whl/torch_stable.html
+	@# This is required to be friendly in the docker and on bare systems until the package is on pip
+	@# https://github.com/zama-ai/concretefhe-internal/issues/809
+	if [[ -d /pkg ]]; then									\
+		NUM_PKG=$$(ls /pkg | wc -l);						\
+		if [[ "$${NUM_PKG}" != "0" ]]; then					\
+			poetry run python -m pip install /pkg/*.whl;	\
+		fi;													\
+	fi
 .PHONY: setup_env

 sync_env:
--- a/docker/Dockerfile.concretefhe-env
+++ b/docker/Dockerfile.concretefhe-env
@@ -1,18 +1,43 @@
-FROM ghcr.io/zama-ai/zamalang-compiler:3bf493e83cd9d5be127adce4a4bde9dd4be229f1
+FROM ubuntu:20.04
+
+# Remove once compiler is on PyPi https://github.com/zama-ai/concretefhe-internal/issues/809
+ARG WHEEL
+ENV TZ=Europe/Paris
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone

 RUN apt-get update && apt-get upgrade --no-install-recommends -y && \
    apt-get install --no-install-recommends -y \
+    build-essential \
    curl \
+    python3-pip \
    python3.8 \
+    python3.8-dev \
    python3.8-tk \
    python3.8-venv \
    python-is-python3 \
    git \
    graphviz* \
+    make \
    pandoc \
    shellcheck && \
    rm -rf /var/lib/apt/lists/* && \
    pip install --no-cache-dir --upgrade pip && \
    pip install --no-cache-dir poetry

-ENV LD_PRELOAD=${RT_LIB}:${LD_PRELOAD}
+# Remove once compiler is on PyPi https://github.com/zama-ai/concretefhe-internal/issues/809
+WORKDIR /pkg
+COPY pkg/${WHEEL} .
+
+ENV SRC_DIR_NAME=src
+
+RUN echo "source /${SRC_DIR_NAME}/.docker_venv/bin/activate" >> /root/.bashrc && \
+    echo "if [[ \"\$?\" != \"0\" ]]; then" >> /root/.bashrc && \
+    echo "    python3 -m venv /${SRC_DIR_NAME}/.docker_venv" >> /root/.bashrc && \
+    echo "    source /${SRC_DIR_NAME}/.docker_venv/bin/activate" >> /root/.bashrc && \
+    echo "    cd /${SRC_DIR_NAME}/ && make setup_env" >> /root/.bashrc && \
+    echo "fi" >> /root/.bashrc && \
+    echo "export MPLBACKEND=TkAgg" >> /root/.bashrc
+
+WORKDIR /${SRC_DIR_NAME}
+
+CMD ["/bin/bash"]
--- a/docker/Dockerfile.concretefhe-env.dockerignore
+++ b/docker/Dockerfile.concretefhe-env.dockerignore
@@ -0,0 +1 @@
+!script/actions_utils/gh_dl_release.sh
--- a/script/actions_utils/gh_dl_release.sh
+++ b/script/actions_utils/gh_dl_release.sh
@@ -0,0 +1,125 @@
+#!/usr/bin/env bash
+#
+# Adapted from https://gist.github.com/maxim/6e15aa45ba010ab030c4
+#
+# gh-dl-release! It works!
+#
+# This script downloads an asset from latest or specific Github release of a
+# private repo. Feel free to extract more of the variables into command line
+# parameters.
+#
+# PREREQUISITES
+#
+# curl, wget, jq
+#
+# USAGE
+#
+# Set all the variables inside the script, make sure you chmod +x it, then
+# to download specific version to my_app.tar.gz:
+#
+#     gh-dl-release 2.1.1 my_app.tar.gz
+#
+# to download latest version:
+#
+#     gh-dl-release latest latest.tar.gz
+#
+# If your version/tag doesn't match, the script will exit with error.
+
+TOKEN=
+ORG_REPO=
+# the name of your release asset file, e.g. build.tar.gz
+FILE=
+DEST_FILE=
+VERSION="latest"
+COMPILER_TAG_OUTPUT_FILE=debug.txt
+
+while [ -n "$1" ]
+do
+   case "$1" in
+        "--token" )
+            shift
+            TOKEN="$1"
+            ;;
+
+        "--org-repo" )
+            shift
+            ORG_REPO="$1"
+            ;;
+
+        "--version" )
+          shift
+          VERSION="$1"
+          ;;
+
+        "--dest-file" )
+          shift
+          DEST_FILE="$1"
+          ;;
+
+        "--file" )
+          shift
+          FILE="$1"
+          ;;
+
+        "--compiler-tag-output-file")
+            shift
+            COMPILER_TAG_OUTPUT_FILE="$1"
+            ;;
+
+        *)
+            echo "Unknown param : $1"
+            exit 1
+            ;;
+   esac
+   shift
+done
+
+alias errcho='>&2 echo'
+
+DEST_DIR=$(dirname "${DEST_FILE}")
+mkdir -p "${DEST_DIR}"
+
+if [[ "${VERSION}" == "latest" ]]; then
+  # Github should return the latest release first.
+  jq_parser=".[0]"
+else
+  jq_parser=". | map(select(.tag_name == \"${VERSION}\"))[0]"
+fi;
+
+release_json=$(curl -H "Authorization: token ${TOKEN}" \
+-H "Accept: application/vnd.github.v3.raw" \
+"https://api.github.com/repos/${ORG_REPO}/releases" | jq "${jq_parser}")
+
+echo "Release json:"
+echo "${release_json}"
+
+asset_json=$(echo "${release_json}" | jq ".assets | map(select(.name | contains(\"${FILE}\")))[0]")
+
+echo "Asset json:"
+echo "${asset_json}"
+
+release_tag=$(echo "${release_json}" | jq -rc '.tag_name')
+asset_id=$(echo "${asset_json}" | jq -rc '.id')
+
+release_tag="${release_tag//-/_}"
+
+echo "Release tag: ${release_tag}"
+echo "Asset id: ${asset_id}"
+
+if [[ "${asset_id}" == "null" ]]; then
+  errcho "ERROR: version not found ${VERSION}"
+  exit 1
+fi
+
+echo "Downloading..."
+
+wget --auth-no-challenge --header='Accept:application/octet-stream' \
+  "https://${TOKEN}:@api.github.com/repos/${ORG_REPO}/releases/assets/${asset_id}" \
+  -O "${DEST_FILE}"
+
+err_code=$?
+
+echo "Done."
+echo "CONCRETEFHE_COMPILER_TAG=${release_tag}" >> "${COMPILER_TAG_OUTPUT_FILE}"
+
+exit "${err_code}"
--- a/torch_requirements.txt
+++ b/torch_requirements.txt
@@ -1,7 +1,7 @@
-torch==1.9.0; sys_platform=="darwin"
-torch==1.9.0+cpu; sys_platform=="linux"
-torch==1.9.0+cpu; sys_platform=="win32"
+torch==1.10.0; sys_platform=="darwin"
+torch==1.10.0+cpu; sys_platform=="linux"
+torch==1.10.0+cpu; sys_platform=="win32"

-torchvision==0.10.0; sys_platform=="darwin"
-torchvision==0.10.0+cpu; sys_platform=="linux"
-torchvision==0.10.0+cpu; sys_platform=="win32"
+torchvision==0.11.1; sys_platform=="darwin"
+torchvision==0.11.1+cpu; sys_platform=="linux"
+torchvision==0.11.1+cpu; sys_platform=="win32"