From 90fa06f80a72021576a6d9309f03ddca73a60d8d Mon Sep 17 00:00:00 2001 From: Arthur Meyre Date: Tue, 21 Sep 2021 14:57:49 +0200 Subject: [PATCH] chore: less notifications - restructure the way we generate notifications --- .github/workflows/continuous-integration.yaml | 131 ++++++++++++++---- .github/workflows/daily-benchmarks.yaml | 10 +- .github/workflows/package-watcher.yaml | 15 ++ 3 files changed, 130 insertions(+), 26 deletions(-) diff --git a/.github/workflows/continuous-integration.yaml b/.github/workflows/continuous-integration.yaml index 3bcd5111f..c4336cb29 100644 --- a/.github/workflows/continuous-integration.yaml +++ b/.github/workflows/continuous-integration.yaml @@ -41,6 +41,7 @@ jobs: image: ${{ steps.set_image.outputs.image || env.LATEST_IMAGE }} needs-push: ${{ env.BUILD_DOCKER }} force-rebuild-docker: ${{ env.FORCE_REBUILD_DOCKER }} + report: ${{ steps.report.outputs.report || 'Did not run.' }} steps: - uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f @@ -53,14 +54,14 @@ jobs: - name: Should rebuild docker check run : | set +e - echo "${{ steps.files.outputs.all }}" | grep ${ENV_DOCKERFILE} + echo "${{ steps.files.outputs.all }}" | grep "${ENV_DOCKERFILE}" DOCKERFILE_CHANGED=$? if [[ "${DOCKERFILE_CHANGED}" == "0" || "${FORCE_REBUILD_DOCKER}" == "true" ]]; then echo "Should rebuild docker image!" - echo "BUILD_DOCKER=true" >> $GITHUB_ENV + echo "BUILD_DOCKER=true" >> "$GITHUB_ENV" else echo "Docker image up to date." - echo "BUILD_DOCKER=false" >> $GITHUB_ENV + echo "BUILD_DOCKER=false" >> "$GITHUB_ENV" fi - name: Set prefligh Docker image id: set_image @@ -69,7 +70,7 @@ jobs: PREFLIGHT_IMAGE_TAG=$(echo ${{ github.ref }} | sed -e 's/\//-/g') PREFLIGHT_IMAGE="${PREFLIGHT_IMAGE_BASE}-${PREFLIGHT_IMAGE_TAG}" echo "::set-output name=image::${PREFLIGHT_IMAGE}" - echo "PREFLIGHT_IMAGE=${PREFLIGHT_IMAGE}" >> $GITHUB_ENV + echo "PREFLIGHT_IMAGE=${PREFLIGHT_IMAGE}" >> "$GITHUB_ENV" - name: Set up Docker Buildx if: ${{ fromJSON(env.BUILD_DOCKER) }} id: buildx @@ -91,17 +92,24 @@ jobs: push: true tags: "${{ env.PREFLIGHT_IMAGE }}" no-cache: true - - name: Slack Notification + - name: Set notification report + id: report if: ${{ always() }} + run: | + REPORT="Docker image preflight build ${{ env.PREFLIGHT_IMAGE }} finished with \ + status ${{ job.status }}. Rebuilt image: ${{ env.BUILD_DOCKER || 'false' }}." + echo "${REPORT}" + echo "::set-output name=report::${REPORT}" + echo "REPORT=${REPORT}" >> "$GITHUB_ENV" + - name: Slack Notification + if: ${{ always() && !success() }} continue-on-error: true uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7 env: SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "Docker image preflight build ${{ env.PREFLIGHT_IMAGE }} finished with \ - status ${{ job.status }}. Rebuilt image: ${{ env.BUILD_DOCKER || 'false' }}. \ - (${{ env.ACTION_RUN_URL }})" + SLACK_MESSAGE: "${{ env.REPORT }} (${{ env.ACTION_RUN_URL }})" SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} @@ -121,6 +129,9 @@ jobs: matrix: python-version: [3.8] + outputs: + report: ${{ steps.report.outputs.report || 'Did not run.' }} + steps: - name: Checkout Code uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f @@ -205,15 +216,23 @@ jobs: with: path: diff-coverage.txt recreate: true - - name: Slack Notification + - name: Set notification report + id: report if: ${{ always() }} + run: | + REPORT="Build finished with status ${{ job.status }}." + echo "${REPORT}" + echo "::set-output name=report::${REPORT}" + echo "REPORT=${REPORT}" >> "$GITHUB_ENV" + - name: Slack Notification + if: ${{ always() && !success() }} continue-on-error: true uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7 env: SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "Build finished with status ${{ job.status }} (${{ env.ACTION_RUN_URL }})" + SLACK_MESSAGE: "${{ env.REPORT }} (${{ env.ACTION_RUN_URL }})" SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} @@ -223,6 +242,9 @@ jobs: group: ${{ github.ref }} cancel-in-progress: true + outputs: + report: ${{ steps.report.outputs.report || 'Did not run.' }} + runs-on: ubuntu-20.04 if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} @@ -257,16 +279,24 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} DISTRIBUTION_ID: ${{ secrets.AWS_REPO_DOCUMENTATION_DISTRIBUTION_ID }} - - name: Slack Notification + - name: Set notification report + id: report if: ${{ always() }} + run: | + REPORT="Publishing documentation finished with status ${{ job.status }}." + echo "${REPORT}" + echo "::set-output name=report::${REPORT}" + echo "REPORT=${REPORT}" >> "$GITHUB_ENV" + + - name: Slack Notification + if: ${{ always() && !success() }} continue-on-error: true uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7 env: SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "Publishing documentation finished with status ${{ job.status }} \ - (${{ env.ACTION_RUN_URL }})" + SLACK_MESSAGE: "${{ env.REPORT }} (${{ env.ACTION_RUN_URL }})" SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} @@ -278,6 +308,9 @@ jobs: group: ${{ github.ref }} cancel-in-progress: true + outputs: + report: ${{ steps.report.outputs.report || 'Did not run.' }} + name: Push env docker image runs-on: ubuntu-20.04 env: @@ -293,27 +326,36 @@ jobs: password: ${{ secrets.BOT_TOKEN }} - name: Pull preflight image run: | - docker pull ${PREFLIGHT_IMAGE} + docker pull "${PREFLIGHT_IMAGE}" - name: Retag to latest and epoch-sha1 and push run: | EPOCH=$(date +%s) SHA1=$(git rev-parse HEAD) TAGGED_IMAGE="${BASE_IMAGE}:${EPOCH}-${SHA1}" - docker tag ${PREFLIGHT_IMAGE} ${LATEST_IMAGE} - docker tag ${PREFLIGHT_IMAGE} ${TAGGED_IMAGE} - docker push ${LATEST_IMAGE} - docker push ${TAGGED_IMAGE} + docker tag "${PREFLIGHT_IMAGE}" "${LATEST_IMAGE}" + docker tag "${PREFLIGHT_IMAGE}" "${TAGGED_IMAGE}" + docker push "${LATEST_IMAGE}" + docker push "${TAGGED_IMAGE}" + + - name: Set notification report + id: report + if: ${{ always() }} + run: | + REPORT="Pushing docker image ${{ env.BASE_IMAGE }} finished with status \ + ${{ job.status }}." + echo "${REPORT}" + echo "::set-output name=report::${REPORT}" + echo "REPORT=${REPORT}" >> "$GITHUB_ENV" - name: Slack Notification - if: ${{ always() }} + if: ${{ always() && !success() }} continue-on-error: true uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7 env: SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "Pushing docker image ${{ env.BASE_IMAGE }} finished with status \ - ${{ job.status }} (${{ env.ACTION_RUN_URL }})" + SLACK_MESSAGE: "${{ env.REPORT }} (${{ env.ACTION_RUN_URL }})" SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} @@ -325,6 +367,9 @@ jobs: group: ${{ github.ref }} cancel-in-progress: true + outputs: + report: ${{ steps.report.outputs.report || 'Did not run.' }} + name: Prepare docker image release runs-on: ubuntu-20.04 @@ -337,7 +382,7 @@ jobs: run: | GIT_TAG=$(echo "${{ github.ref }}" | sed 's/refs\/tags\///g') RELEASE_IMG_TAG="${RELEASE_IMAGE_BASE}:${GIT_TAG}" - echo "RELEASE_IMG_TAG=${RELEASE_IMG_TAG}" >> $GITHUB_ENV + echo "RELEASE_IMG_TAG=${RELEASE_IMG_TAG}" >> "$GITHUB_ENV" - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 @@ -366,6 +411,40 @@ jobs: -v "$(pwd)"/docker/release_resources:/data \ "${RELEASE_IMG_TAG}" /bin/bash -c "python ./sanity_check.py" docker push "${RELEASE_IMG_TAG}" + - name: Set notification report + id: report + if: ${{ always() }} + run: | + REPORT="Pushing docker image ${{ env.RELEASE_IMG_TAG }} finished with status \ + ${{ job.status }}." + echo "${REPORT}" + echo "::set-output name=report::${REPORT}" + echo "REPORT=${REPORT}" >> "$GITHUB_ENV" + - name: Slack Notification + if: ${{ always() && !success() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7 + env: + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "${{ env.REPORT }} (${{ env.ACTION_RUN_URL }})" + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + + send-report: + if: ${{ always() }} + needs: [ + build-preflight-docker, + build, + publish-docs, + push-docker-image, + package-release, + ] + + name: Send Slack notification + runs-on: ubuntu-20.04 + steps: - name: Slack Notification if: ${{ always() }} continue-on-error: true @@ -374,7 +453,11 @@ jobs: SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "Pushing docker image ${{ env.RELEASE_IMG_TAG }} finished with status \ - ${{ job.status }} (${{ env.ACTION_RUN_URL }})" + SLACK_MESSAGE: "Full run here: ${{ env.ACTION_RUN_URL }}\n\ + - build-preflight-docker: ${{ needs.build-preflight-docker.outputs.report || 'Did not run.' }}\n\n\ + - build: ${{ needs.build.outputs.report || 'Did not run.' }}\n\n\ + - publish-docs: ${{ needs.publish-docs.outputs.report || 'Did not run.' }}\n\n\ + - push-docker-image: ${{ needs.push-docker-image.outputs.report || 'Did not run.' }}\n\n\ + - package-release: ${{ needs.package-release.outputs.report || 'Did not run.' }}" SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} diff --git a/.github/workflows/daily-benchmarks.yaml b/.github/workflows/daily-benchmarks.yaml index 6101fd637..c2295f56a 100644 --- a/.github/workflows/daily-benchmarks.yaml +++ b/.github/workflows/daily-benchmarks.yaml @@ -4,6 +4,9 @@ on: schedule: - cron: '0 22 * * *' # Everyday @ 22:00 +env: + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + jobs: perform: name: Run Benchmarks on EC2 and Publish Results to Progress Tracker @@ -21,7 +24,9 @@ jobs: aws ec2 start-instances --instance-ids ${{ secrets.BENCHMARKS_EC2_INSTANCE_ID }} - name: Wait For The Instance To Get An IP Address - run: timeout 180 bash -c 'until [[ $(aws ec2 describe-instances --instance-ids ${{ secrets.BENCHMARKS_EC2_INSTANCE_ID }} --query 'Reservations[].Instances[].PublicIpAddress' --output text) != "" ]]; do sleep 0.1; done' + run: | + # shellcheck disable=SC2016,2026 + timeout 180 bash -c 'until [[ $(aws ec2 describe-instances --instance-ids ${{ secrets.BENCHMARKS_EC2_INSTANCE_ID }} --query 'Reservations[].Instances[].PublicIpAddress' --output text) != "" ]]; do sleep 0.1; done' - name: Get Public IP Address of EC2 Instance id: public-ip @@ -73,6 +78,7 @@ jobs: SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: 'Publishing benchmarks finished with status ${{ job.status }} (${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})' + SLACK_MESSAGE: "Publishing benchmarks finished with status ${{ job.status }} \ + (${{ env.ACTION_RUN_URL }})" SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} diff --git a/.github/workflows/package-watcher.yaml b/.github/workflows/package-watcher.yaml index f01f7d4c7..ef236b758 100644 --- a/.github/workflows/package-watcher.yaml +++ b/.github/workflows/package-watcher.yaml @@ -7,6 +7,9 @@ on: # Timezone is UTC, so Paris time is +2 during the summer and +1 during winter - cron: '0 6-20 * * 1-5' +env: + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + jobs: check_and_notify_build: name: Check timestamps and notify build @@ -24,3 +27,15 @@ jobs: --token ${{ secrets.BOT_TOKEN }} \ --org-repo ${{ github.repository }} \ --event-type rebuild-env-docker + - name: Send Slack Notification + if: ${{ always() && failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7 + env: + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Package watcher finished with status ${{ job.status }} \ + (${{ env.ACTION_RUN_URL }})" + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}