diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 7085bad21..e9dd73b4a 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -2,9 +2,21 @@ name: Performance benchmarks on: + push: + branches: + - 'main' workflow_dispatch: + inputs: + ec2-instance-type: + description: 'EC2 instance type' + required: true + default: 'm6i.metal' + type: choice + options: + - m6i.metal + - c6a.metal # Have a weekly benchmark run on main branch to be available on Monday morning (Paris time) -# TODO: uncomment this section once the benchmarks are debugged +# TODO: uncomment this section once MLBenchmarks are implemented # schedule: # # * is a special character in YAML so you have to quote this string # # At 1:00 every Thursday @@ -14,54 +26,80 @@ on: env: CARGO_TERM_COLOR: always RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json - DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler + EC2_INSTANCE_TYPE: ${{ inputs.ec2-instance-type || 'm6i.metal' }} jobs: - StartRunner: + start-runner: name: Start EC2 runner runs-on: ubuntu-20.04 outputs: label: ${{ steps.start-ec2-runner.outputs.label }} ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} steps: + - name: Sets env vars for m6i.metal + if: ${{ env.EC2_INSTANCE_TYPE == 'm6i.metal' }} + run: | + echo "AWS_REGION=eu-west-3" >> $GITHUB_ENV + echo "EC2_IMAGE_ID=ami-0a24aaee029d1295c" >> $GITHUB_ENV + echo "SUBNET_ID=subnet-a886b4c1" >> $GITHUB_ENV + echo "SECURITY_GROUP_ID=sg-0bf1c1d79c97bc88f" >> $GITHUB_ENV + + - name: Sets env vars for c6a.metal + if: ${{ env.EC2_INSTANCE_TYPE == 'c6a.metal' }} + run: | + echo "AWS_REGION=us-east-1" >> $GITHUB_ENV + echo "EC2_IMAGE_ID=ami-0afb83d80b3b060d8" >> $GITHUB_ENV + echo "SUBNET_ID=subnet-da319dd4" >> $GITHUB_ENV + echo "SECURITY_GROUP_ID=sg-0f8b52622a2669491" >> $GITHUB_ENV + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.AWS_IAM_ID }} - aws-secret-access-key: ${{ secrets.AWS_IAM_KEY }} - aws-region: eu-west-3 - aws-resource-tags: [["Name", "compiler-benchmarks-github"]] + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} - name: Start EC2 runner id: start-ec2-runner uses: machulav/ec2-github-runner@v2 with: mode: start github-token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - ec2-image-id: ami-00ac986ef330c077f - ec2-instance-type: m6i.metal - subnet-id: subnet-a886b4c1 - security-group-id: sg-0bf1c1d79c97bc88f + ec2-image-id: ${{ env.EC2_IMAGE_ID }} + ec2-instance-type: ${{ env.EC2_INSTANCE_TYPE }} + subnet-id: ${{ env.SUBNET_ID }} + security-group-id: ${{ env.SECURITY_GROUP_ID }} + aws-resource-tags: > + [ + {"Key": "Name", "Value": "compiler-benchmarks-github"} + ] - RunBenchmarks: + run-benchmarks: name: Execute end-to-end benchmarks in EC2 runs-on: ${{ needs.start-runner.outputs.label }} if: ${{ !cancelled() }} - needs: StartRunner + needs: start-runner steps: # SSH private key is required as some dependencies are from private repos - uses: webfactory/ssh-agent@v0.5.2 with: ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }} - - uses: actions/checkout@v2 + - name: Fetch submodules + uses: actions/checkout@v3 with: submodules: recursive token: ${{ secrets.GH_TOKEN }} + - name: Set up home + # "Install rust" step require root user to have a HOME directory which is not set. + run: | + echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" + - name: Install rust uses: actions-rs/toolchain@v1 with: toolchain: stable + override: true - name: Concrete-Optimizer run: | @@ -79,68 +117,74 @@ jobs: run: | touch keysetcache.timestamp - - name: Build and test compiler - uses: addnab/docker-run-action@v3 - id: build-compiler - with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE_TEST }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}/llvm-project:/llvm-project - -v ${{ github.workspace }}/compiler:/compiler - -v ${{ github.workspace }}/KeySetCache:/tmp/KeySetCache - shell: bash - run: | - set -e - cd /compiler - rm -rf /build - make run-benchmarks + - name: Build compiler and run end-to-end benchmarks + run: | + set -e + cd compiler + make BINDINGS_PYTHON_ENABLED=OFF run-benchmarks - name: Parse results shell: bash run: | - python3 ./ci/benchmark_parser.py benchmarks_results.json ${RESULTS_FILENAME} \ - --series-tags '{"commit_hash": "${{ github.sha }}"}' - gzip -k ${RESULTS_FILENAME} + OPTIMIZER_HASH="$(cd compiler/concrete-optimizer; git rev-parse HEAD)" + python3 ./ci/benchmark_parser.py compiler/benchmarks_results.json ${{ env.RESULTS_FILENAME }} \ + --series-name compiler_end_to_end_benchmarks \ + --series-help "Concrete compiler end-to-end benchmarks timings" \ + --series-tags "{\"compiler_hash\": \"${{ github.sha }}\", \"branch\": \"${{ github.ref_name }}\", \"optimizer_hash\": \"${OPTIMIZER_HASH}\", \"hardware\": \"aws ${{ env.EC2_INSTANCE_TYPE }}\"}" + gzip -k ${{ env.RESULTS_FILENAME }} - name: Upload compressed results artifact uses: actions/upload-artifact@v3 with: name: ${{ github.sha }} - path: ${RESULTS_FILENAME}.gz + path: ${{ env.RESULTS_FILENAME }}.gz + + - name: Checkout Slab repo + uses: actions/checkout@v3 + with: + repository: zama-ai/slab + path: slab + token: ${{ secrets.GH_TOKEN }} - - uses: actions/checkout@v3 - name: Send data to Slab shell: bash run: | echo "Computing HMac on downloaded artifact" - SIGNATURE="$(./ci/hmac_calculator.sh ./${RESULTS_FILENAME} '${{ secrets.JOB_SECRET }}')" + SIGNATURE="$(slab/scripts/hmac_calculator.sh ${{ env.RESULTS_FILENAME }} '${{ secrets.JOB_SECRET }}')" echo "Sending results to Slab..." curl -v -k \ -H "Content-Type: application/json" \ -H "X-Slab-Repository: ${{ github.repository }}" \ -H "X-Slab-Command: plot_data" \ -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ - -d @${RESULTS_FILENAME} \ + -d @${{ env.RESULTS_FILENAME }} \ ${{ secrets.SLAB_URL }} - StopRunner: + stop-runner: name: Stop EC2 runner needs: - - StartRunner - - RunBenchmarks + - start-runner + - run-benchmarks runs-on: ubuntu-20.04 if: ${{ always() && (needs.start-runner.result != 'skipped') }} steps: + - name: Sets AWS region for m6i.metal + if: ${{ env.EC2_INSTANCE_TYPE == 'm6i.metal' }} + run: | + echo "AWS_REGION=eu-west-3" >> $GITHUB_ENV + + - name: Sets AWS region for c6a.metal + if: ${{ env.EC2_INSTANCE_TYPE == 'c6a.metal' }} + run: | + echo "AWS_REGION=us-east-1" >> $GITHUB_ENV + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.AWS_IAM_ID }} - aws-secret-access-key: ${{ secrets.AWS_IAM_KEY }} - aws-region: eu-west-3 + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} - name: Stop EC2 runner uses: machulav/ec2-github-runner@v2 with: diff --git a/compiler/Makefile b/compiler/Makefile index 22c784a14..490c0137a 100644 --- a/compiler/Makefile +++ b/compiler/Makefile @@ -229,7 +229,7 @@ build-benchmarks: build-initialized cmake --build $(BUILD_DIR) --target end_to_end_benchmark run-benchmarks: build-benchmarks - $(BUILD_DIR)/bin/end_to_end_benchmark --benchmark_out=benchmarks_results.json --benchmark_out_format=json + $(BUILD_DIR)/bin/end_to_end_benchmark --benchmark_out=benchmarks_results.json --benchmark_out_format=json --benchmark_repetitions=10 --benchmark_report_aggregates_only=true build-mlbench: build-initialized cmake --build $(BUILD_DIR) --target end_to_end_mlbench