diff --git a/.github/workflows/publish_docker_images.yml b/.github/workflows/compiler_publish_docker_images.yml similarity index 85% rename from .github/workflows/publish_docker_images.yml rename to .github/workflows/compiler_publish_docker_images.yml index 77a21417f..78c33d767 100644 --- a/.github/workflows/publish_docker_images.yml +++ b/.github/workflows/compiler_publish_docker_images.yml @@ -1,30 +1,8 @@ # Build and publish Docker images for different applications using AWS EC2. -name: Publish Docker Images +name: Compiler - Docker images build & publish on: - workflow_dispatch: - inputs: - instance_id: - description: 'Instance ID' - type: string - instance_image_id: - description: 'Instance AMI ID' - type: string - instance_type: - description: 'Instance product type' - type: string - runner_name: - description: 'Action runner name' - type: string - request_id: - description: 'Slab request ID' - type: string - matrix_item: - description: 'Build matrix item' - type: string - -env: - THIS_FILE: .github/workflows/continuous-integration.yml + workflow_call: jobs: BuildAndPushDockerImages: @@ -94,6 +72,7 @@ jobs: - name: Build Tag and Publish if: ${{ steps.login.conclusion != 'skipped' }} run: | + cd compilers/concrete-compiler docker build -t "${IMAGE}" -f builders/Dockerfile.hpx-env . docker push "${IMAGE}:latest" @@ -121,6 +100,7 @@ jobs: - name: Build Tag and Publish if: ${{ steps.login.conclusion != 'skipped' }} run: | + cd compilers/concrete-compiler docker build -t "${IMAGE}" -f builders/Dockerfile.cuda-env . docker image tag "${IMAGE}" "${IMAGE}:${TAG}" docker push "${IMAGE}:latest" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4fa120dbe..23827e1f6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -20,7 +20,7 @@ jobs: optimizer: ${{ steps.optimizer.outputs.any_changed }} concrete-cpu: ${{ steps.concrete-cpu.outputs.any_changed }} concrete-cuda: ${{ steps.concrete-cuda.outputs.any_changed }} - test-all: ${{ steps.test-all.outputs.test-all }} + push-main: ${{ steps.github.outputs.push-main }} steps: - name: Checkout the repository uses: actions/checkout@v3 @@ -58,16 +58,16 @@ jobs: with: files: backends/concrete-cuda - - name: Set the test-all - id: test-all + - name: Set some github event outputs + id: github if: github.event_name == 'push' && github.ref == 'refs/heads/main' - run: echo "test-all=true" >> "$GITHUB_OUTPUT" + run: echo "push-main=true" >> "$GITHUB_OUTPUT" ################################################# # concrete-python jobs ########################## concrete-python: needs: file-change - if: needs.file-change.outputs.concrete-python == 'true' || needs.file-change.outputs.test-all + if: needs.file-change.outputs.concrete-python == 'true' || needs.file-change.outputs.push-main uses: ./.github/workflows/concrete_python_build_and_test.yml secrets: inherit @@ -78,18 +78,21 @@ jobs: if: needs.file-change.outputs.compiler == 'true' uses: ./.github/workflows/compiler_format_and_linting.yml - compiler-aws-tests: + compiler-cpu-build: needs: file-change - if: needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.test-all - runs-on: ubuntu-latest - steps: - - name: Launch compiler test on AWS using Slab - uses: mshick/add-pr-comment@v2 - with: - allow-repeats: true - message: | - @slab-ci compiler-cpu-build - @slab-ci compiler-gpu-build + if: needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.push-main == 'true' + uses: ./.github/workflows/start_slab.yml + secrets: inherit + with: + command: compiler-cpu-build + + compiler-gpu-build: + needs: file-change + if: needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.push-main == 'true' + uses: ./.github/workflows/start_slab.yml + secrets: inherit + with: + command: compiler-gpu-build compiler-macos-tests: needs: file-change @@ -97,11 +100,19 @@ jobs: uses: ./.github/workflows/compiler_macos_build_and_test.yml secrets: inherit + compiler-publish-docker-images: + needs: file-change + if: needs.file-change.outputs.compiler == 'true' && needs.file-change.outputs.push-main == 'true' + uses: ./.github/workflows/start_slab.yml + secrets: inherit + with: + command: compiler-publish-docker-images + ################################################# # Optimizer jobs ################################ optimizer: needs: file-change - if: needs.file-change.outputs.optimizer == 'true' || needs.file-change.outputs.test-all + if: needs.file-change.outputs.optimizer == 'true' || needs.file-change.outputs.push-main uses: ./.github/workflows/optimizer.yml secrets: inherit @@ -109,7 +120,7 @@ jobs: # ConcreteCPU jobs ############################## concrete-cpu: needs: file-change - if: needs.file-change.outputs.concrete-cpu == 'true' || needs.file-change.outputs.test-all + if: needs.file-change.outputs.concrete-cpu == 'true' || needs.file-change.outputs.push-main uses: ./.github/workflows/concrete_cpu_test.yml secrets: inherit @@ -117,6 +128,6 @@ jobs: # Concrete-cuda jobs ############################ concrete-cuda: needs: file-change - if: needs.file-change.outputs.concrete-cuda == 'true' || needs.file-change.outputs.test-all + if: needs.file-change.outputs.concrete-cuda == 'true' || needs.file-change.outputs.push-main uses: ./.github/workflows/concrete_cuda_test.yml secrets: inherit diff --git a/.github/workflows/start_ml_benchmarks.yml b/.github/workflows/start_ml_benchmarks.yml deleted file mode 100644 index 9c7616b7f..000000000 --- a/.github/workflows/start_ml_benchmarks.yml +++ /dev/null @@ -1,30 +0,0 @@ -# Start application benchmarks job on Slab CI bot. -name: Start ML benchmarks - -on: - workflow_dispatch: - - # Have a weekly benchmark run on main branch to be available on Monday morning (Paris time) -# TODO: uncomment this section once MLBenchmarks are implemented -# schedule: -# # * is a special character in YAML so you have to quote this string -# # At 1:00 every Thursday -# # Timezone is UTC, so Paris time is +2 during the summer and +1 during winter -# - cron: '0 1 * * THU' - -jobs: - start-ml-benchmarks: - steps: - - name: Start AWS job in Slab - shell: bash - # TODO: step result must be correlated to HTTP return code. - run: | - PAYLOAD='{"command": "ml-bench", "git_ref": ${{ github.ref }}, "sha": ${{ github.sha }}}' - SIGNATURE="$(slab/scripts/hmac_calculator.sh ${PAYLOAD} '${{ secrets.JOB_SECRET }}')" - curl -v -k \ - -H "Content-Type: application/json" \ - -H "X-Slab-Repository: ${{ github.repository }}" \ - -H "X-Slab-Command: start_data" \ - -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ - -d @${{ PAYLOAD }} \ - ${{ secrets.SLAB_URL }} diff --git a/.github/workflows/start_aws_builds.yml b/.github/workflows/start_slab.yml similarity index 55% rename from .github/workflows/start_aws_builds.yml rename to .github/workflows/start_slab.yml index ce6c665fe..3d20a96ed 100644 --- a/.github/workflows/start_aws_builds.yml +++ b/.github/workflows/start_slab.yml @@ -1,19 +1,31 @@ -# Start all AWS build jobs on Slab CI bot. -name: Start all AWS builds +# Start job on Slab CI bot given by input command. +name: Start AWS job on: - push: - branches: - - 'main' + workflow_call: + inputs: + command: + required: true + type: string workflow_dispatch: + inputs: + command: + required: true + type: string + +env: + GIT_REF: ${{ github.head_ref }} jobs: - start-builds: - strategy: - matrix: - command: [cpu-build, gpu-build, docker-images-build] + sl: runs-on: ubuntu-latest steps: + - name: Get git ref + # github.head_ref is only available from a Pull Request + if: env.GIT_REF == '' + run: | + echo "GIT_REF=${{ github.ref }}" >> $GITHUB_ENV + - name: Checkout Slab repo uses: actions/checkout@v3 with: @@ -25,9 +37,10 @@ jobs: shell: bash # TODO: step result must be correlated to HTTP return code. run: | - echo -n '{"command": "${{ matrix.command }}", "git_ref": "${{ github.ref }}", "sha": "${{ github.sha }}"}' > command.json + echo -n '{"command": "${{ inputs.command }}", "git_ref": "${{ env.GIT_REF }}", "sha": "${{ github.sha }}"}' > command.json SIGNATURE="$(slab/scripts/hmac_calculator.sh command.json '${{ secrets.JOB_SECRET }}')" curl -v -k \ + --fail-with-body \ -H "Content-Type: application/json" \ -H "X-Slab-Repository: ${{ github.repository }}" \ -H "X-Slab-Command: start_aws" \ diff --git a/ci/slab.toml b/ci/slab.toml index c19c83fcf..262cda103 100644 --- a/ci/slab.toml +++ b/ci/slab.toml @@ -12,6 +12,13 @@ instance_type = "m6i.16xlarge" subnet_id = "subnet-a886b4c1" security_group= ["sg-0bf1c1d79c97bc88f", ] +[profile.m6i-metal] +region = "eu-west-3" +image_id = "ami-0a24aaee029d1295c" +instance_type = "m6i.metal" +subnet_id = "subnet-a886b4c1" +security_group= ["sg-0bf1c1d79c97bc88f", ] + [profile.gpu] region = "us-east-1" image_id = "ami-0c4773f5626d919b6" @@ -44,10 +51,10 @@ profile = "gpu" check_run_name = "Compiler Performances Benchmarks (GPU)" # Trigger Docker images build -[command.docker-images-build] -workflow = "publish_docker_images.yml" -profile = "m6i-old" -check_run_name = "AWS Docker images build & publish (Slab)" +[command.compiler-publish-docker-images] +workflow = "compiler_publish_docker_images.yml" +profile = "m6i-cpu-test" +check_run_name = "Compiler - Docker images build & publish" # Trigger ML benchmarks by running each use cases subset in parallel. [command.ml-bench] diff --git a/compilers/concrete-compiler/builders/Dockerfile.concrete-compiler-env b/compilers/concrete-compiler/builders/Dockerfile.concrete-compiler-env index 130f3511b..8c8fdb11a 100644 --- a/compilers/concrete-compiler/builders/Dockerfile.concrete-compiler-env +++ b/compilers/concrete-compiler/builders/Dockerfile.concrete-compiler-env @@ -14,6 +14,7 @@ RUN ccache -F 0 # Install Rust RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ENV PATH=/root/.cargo/bin:$PATH +RUN rustup install nightly SHELL ["/bin/bash", "-c"] # Install boost ADD https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/boost_1_71_0.tar.gz /boost_1_71_0.tar.gz