diff --git a/.github/workflows/continuous-integration.yaml b/.github/workflows/continuous-integration.yaml index de6d7fd54..30f6e6cc7 100644 --- a/.github/workflows/continuous-integration.yaml +++ b/.github/workflows/continuous-integration.yaml @@ -78,12 +78,14 @@ jobs: echo "Changed files:" echo "${ALL_CHANGED_FILES}" - echo "::set-output name=all::${ALL_CHANGED_FILES}" + echo "::set-output name=all::${ALL_CHANGED_FILES//$'\n'/ }" - name: Should rebuild docker check run: | set +e - echo "${{ steps.files.outputs.all }}" | grep "${ENV_DOCKERFILE}$" + ALL_CHANGED_FILES="${{ steps.files.outputs.all }}" + ALL_CHANGED_FILES="${ALL_CHANGED_FILES// /$'\n'}" + echo "${ALL_CHANGED_FILES}" | grep "${ENV_DOCKERFILE}$" DOCKERFILE_CHANGED=$? if [[ "${DOCKERFILE_CHANGED}" == "0" || "${FORCE_REBUILD_DOCKER}" == "true" ]]; then echo "Should rebuild docker image!" diff --git a/.github/workflows/package-watcher.yaml b/.github/workflows/package-watcher.yaml new file mode 100644 index 000000000..2e5ae5855 --- /dev/null +++ b/.github/workflows/package-watcher.yaml @@ -0,0 +1,83 @@ +name: Package Version Checker + +on: + schedule: + # * is a special character in YAML so you have to quote this string + # At minute 0 for each hour from 8:00 to 22:00 inclusive from Monday to Friday inclusive + # Timezone is UTC, so Paris time is +2 during the summer and +1 during winter + - cron: '0 6-20 * * 1-5' + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +env: + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + +jobs: + check_and_notify_build: + name: Check timestamps and open PR + runs-on: ubuntu-20.04 + steps: + - name: Should run + run: | + SHOULD_RUN=${{ secrets.PACKAGE_WATCHER_ENABLED }} + if [[ "${SHOULD_RUN}" == "PACKAGE_WATCHER_PREFIX_ENABLED" ]]; then + SHOULD_RUN="true" + echo "Running package watcher" + else + SHOULD_RUN="false" + echo "Won't run package watcher" + fi + echo "SHOULD_RUN=${SHOULD_RUN}" >> "$GITHUB_ENV" + - name: Checkout Code + if: ${{ fromJSON(env.SHOULD_RUN) }} + uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 + - name: Compare image timestamps and notify + if: ${{ fromJSON(env.SHOULD_RUN) }} + env: + WHEEL: concretefhe_compiler-0.1.0-cp38-cp38-manylinux_2_24_x86_64.whl + run: | + ./script/actions_utils/timestamp_check_compiler_package_update_container.sh \ + --compiler-release-endpoint-url \ + https://api.github.com/repos/zama-ai/homomorphizer/releases \ + --env_img_url \ + https://api.github.com/orgs/zama-ai/packages/container/concretefhe-env/versions \ + --file "${WHEEL}" \ + --token ${{ secrets.BOT_TOKEN }} \ + --github-env "$GITHUB_ENV" + - name: Open PR + if: ${{ fromJSON(env.SHOULD_RUN) && env.LATEST_COMPILER_PACKAGE_TIMESTAMP != '' }} + uses: peter-evans/create-pull-request@67df31e08a133c6a77008b89689677067fef169e + id: cpr + with: + token: ${{ secrets.BOT_TOKEN }} + commit-message: "chore(deps): bump compiler to ${{ env.LATEST_COMPILER_PACKAGE_TIMESTAMP }}" + branch: chore/update-docker-env-compiler + base: main + title: "Docker env compiler update to ${{ env.LATEST_COMPILER_PACKAGE_TIMESTAMP }}" + body: "Automatic PR for docker env compiler update" + labels: dependencies + - name: Self approve PR to attempt auto merge + if: ${{ fromJSON(env.SHOULD_RUN) && env.LATEST_COMPILER_PACKAGE_TIMESTAMP != '' }} + uses: hmarr/auto-approve-action@5d04a5ca6da9aeb8ca9f31a5239b96fc3e003029 + with: + github-token: ${{ secrets.BOT_TOKEN }} + pull-request-number: ${{ steps.cpr.outputs.pull-request-number }} + - name: Enable auto-merge for PR + if: ${{ fromJSON(env.SHOULD_RUN) && env.LATEST_COMPILER_PACKAGE_TIMESTAMP != '' }} + uses: peter-evans/enable-pull-request-automerge@d2ede5636b3febc92809259995e643565e675aab + with: + token: ${{ secrets.BOT_TOKEN }} + pull-request-number: ${{ steps.cpr.outputs.pull-request-number }} + - name: Send Slack Notification + if: ${{ always() && failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7 + env: + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Package watcher finished with status ${{ job.status }} \ + (${{ env.ACTION_RUN_URL }})" + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} diff --git a/docker/Dockerfile.concretefhe-env b/docker/Dockerfile.concretefhe-env index d93e01207..bbce6cdce 100644 --- a/docker/Dockerfile.concretefhe-env +++ b/docker/Dockerfile.concretefhe-env @@ -1,5 +1,8 @@ FROM ubuntu:20.04 +# Do not change the line below it will be updated automatically when the docker is regenerated +# compiler timestamp: unknown + # Remove once compiler is on PyPi https://github.com/zama-ai/concretefhe-internal/issues/809 ARG WHEEL ENV TZ=Europe/Paris diff --git a/script/actions_utils/timestamp_check_compiler_package_update_container.sh b/script/actions_utils/timestamp_check_compiler_package_update_container.sh new file mode 100755 index 000000000..35d0570c4 --- /dev/null +++ b/script/actions_utils/timestamp_check_compiler_package_update_container.sh @@ -0,0 +1,93 @@ +#!/bin/bash -e + +set -e + +FILE= +COMPILER_RELEASE_ENDPOINT_URL= +ENV_IMG_ENDPOINT_URL= +TOKEN= +ENV_DOCKERFILE=./docker/Dockerfile.concretefhe-env +GITHUB_ENV_FILE=debug.txt + +while [ -n "$1" ] +do + case "$1" in + "--file" ) + shift + FILE="$1" + ;; + + "--compiler-release-endpoint-url" ) + shift + COMPILER_RELEASE_ENDPOINT_URL="$1" + ;; + + "--env_img_url" ) + shift + ENV_IMG_ENDPOINT_URL="$1" + ;; + + "--token" ) + shift + TOKEN="$1" + ;; + + "--github-env") + shift + GITHUB_ENV_FILE="$1" + ;; + + *) + echo "Unknown param : $1" + exit 1 + ;; + esac + shift +done + +ENV_JSON=$(curl \ +-X GET \ +-H "Accept: application/vnd.github.v3+json" \ +-H "Authorization: token ${TOKEN}" \ +"${ENV_IMG_ENDPOINT_URL}") + +LATEST_ENV_IMG_JSON=$(echo "${ENV_JSON}" | \ +jq -rc '.[] | select(.metadata.container.tags[] | contains("latest"))') + +RELEASE_JSON=$(curl -H "Authorization: token ${TOKEN}" \ +-H "Accept: application/vnd.github.v3.raw" \ +"${COMPILER_RELEASE_ENDPOINT_URL}" | jq '.[0]') + +echo "Release json:" +echo "${RELEASE_JSON}" + +ASSET_JSON=$(echo "${RELEASE_JSON}" | jq ".assets | map(select(.name | contains(\"${FILE}\")))[0]") + +echo "Asset json:" +echo "${ASSET_JSON}" + +LATEST_ENV_IMG_TIMESTAMP=$(echo "${LATEST_ENV_IMG_JSON}" | jq -r '.updated_at') +LATEST_COMPILER_PACKAGE_TIMESTAMP=$(echo "${ASSET_JSON}" | jq -r '.updated_at') + +echo "Latest env image timestamp: ${LATEST_ENV_IMG_TIMESTAMP}" +echo "Latest compiler package timestamp: ${LATEST_COMPILER_PACKAGE_TIMESTAMP}" + +LATEST_ENV_IMG_EPOCH=$(date -d "${LATEST_ENV_IMG_TIMESTAMP}" +%s) +LATEST_COMPILER_PACKAGE_EPOCH=$(date -d "${LATEST_COMPILER_PACKAGE_TIMESTAMP}" +%s) + +echo "Latest env image epoch: ${LATEST_ENV_IMG_EPOCH}" +echo "Latest compiler package epoch: ${LATEST_COMPILER_PACKAGE_EPOCH}" + +if [[ "${LATEST_COMPILER_PACKAGE_EPOCH}" -gt "${LATEST_ENV_IMG_EPOCH}" ]]; then + echo "Env image out of date, sending rebuild request." + TMP_DOCKER_FILE="$(mktemp)" + sed "s/\(# compiler timestamp: \)\(.*\)/\1${LATEST_COMPILER_PACKAGE_TIMESTAMP}/g" \ + "${ENV_DOCKERFILE}" > "${TMP_DOCKER_FILE}" + cp -f "${TMP_DOCKER_FILE}" "${ENV_DOCKERFILE}" + rm -f "${TMP_DOCKER_FILE}" + echo "LATEST_COMPILER_PACKAGE_TIMESTAMP=${LATEST_COMPILER_PACKAGE_TIMESTAMP}" \ + >> "${GITHUB_ENV_FILE}" + echo "New package timestamp: ${LATEST_COMPILER_PACKAGE_TIMESTAMP}" +else + echo "Image up to date, nothing to do." +fi