ci(bench): fix end-to-end benchmarks workflow

This includes several fixes and add some functionalities:
 * EC2 instance type can be selected when workflow is triggered manually
 * benchmarks will be run on each push on main branch
 * docker is not used any more due to building issues
 * 10 repetitions are made during the benchmarks then results are aggregated
 * more tags are used to identify benchmarks configuration

.github/workflows/benchmark.yml

@@ -2,9 +2,21 @@
 name: Performance benchmarks
 
 on:
+  push:
+    branches:
+      - 'main'
   workflow_dispatch:
+    inputs:
+      ec2-instance-type:
+        description: 'EC2 instance type'
+        required: true
+        default: 'm6i.metal'
+        type: choice
+        options:
+          - m6i.metal
+          - c6a.metal
   # Have a weekly benchmark run on main branch to be available on Monday morning (Paris time)
-# TODO: uncomment this section once the benchmarks are debugged
+# TODO: uncomment this section once MLBenchmarks are implemented
 #  schedule:
 #    # * is a special character in YAML so you have to quote this string
 #    # At 1:00 every Thursday
@@ -14,54 +26,80 @@ on:
 env:
   CARGO_TERM_COLOR: always
   RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json
-  DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler
+  EC2_INSTANCE_TYPE: ${{ inputs.ec2-instance-type || 'm6i.metal' }}
 
 jobs:
-  StartRunner:
+  start-runner:
     name: Start EC2 runner
     runs-on: ubuntu-20.04
     outputs:
       label: ${{ steps.start-ec2-runner.outputs.label }}
       ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
     steps:
+      - name: Sets env vars for m6i.metal
+        if: ${{ env.EC2_INSTANCE_TYPE == 'm6i.metal' }}
+        run: |
+          echo "AWS_REGION=eu-west-3" >> $GITHUB_ENV
+          echo "EC2_IMAGE_ID=ami-0a24aaee029d1295c" >> $GITHUB_ENV
+          echo "SUBNET_ID=subnet-a886b4c1" >> $GITHUB_ENV
+          echo "SECURITY_GROUP_ID=sg-0bf1c1d79c97bc88f" >> $GITHUB_ENV
+
+      - name: Sets env vars for c6a.metal
+        if: ${{ env.EC2_INSTANCE_TYPE == 'c6a.metal' }}
+        run: |
+          echo "AWS_REGION=us-east-1" >> $GITHUB_ENV
+          echo "EC2_IMAGE_ID=ami-0afb83d80b3b060d8" >> $GITHUB_ENV
+          echo "SUBNET_ID=subnet-da319dd4" >> $GITHUB_ENV
+          echo "SECURITY_GROUP_ID=sg-0f8b52622a2669491" >> $GITHUB_ENV
+
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:
-          aws-access-key-id: ${{ secrets.AWS_IAM_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_IAM_KEY }}
-          aws-region: eu-west-3
-          aws-resource-tags: [["Name", "compiler-benchmarks-github"]]
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
       - name: Start EC2 runner
         id: start-ec2-runner
         uses: machulav/ec2-github-runner@v2
         with:
           mode: start
           github-token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
-          ec2-image-id: ami-00ac986ef330c077f
-          ec2-instance-type: m6i.metal
-          subnet-id: subnet-a886b4c1
-          security-group-id: sg-0bf1c1d79c97bc88f
+          ec2-image-id: ${{ env.EC2_IMAGE_ID }}
+          ec2-instance-type: ${{ env.EC2_INSTANCE_TYPE }}
+          subnet-id: ${{ env.SUBNET_ID }}
+          security-group-id: ${{ env.SECURITY_GROUP_ID }}
+          aws-resource-tags: >
+            [
+              {"Key": "Name", "Value": "compiler-benchmarks-github"}
+            ]
 
-  RunBenchmarks:
+  run-benchmarks:
     name: Execute end-to-end benchmarks in EC2
     runs-on: ${{ needs.start-runner.outputs.label }}
     if: ${{ !cancelled() }}
-    needs: StartRunner
+    needs: start-runner
     steps:
       # SSH private key is required as some dependencies are from private repos
       - uses: webfactory/ssh-agent@v0.5.2
         with:
           ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }}
 
-      - uses: actions/checkout@v2
+      - name: Fetch submodules
+        uses: actions/checkout@v3
         with:
           submodules: recursive
           token: ${{ secrets.GH_TOKEN }}
 
+      - name: Set up home
+        # "Install rust" step require root user to have a HOME directory which is not set.
+        run: |
+          echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}"
+
       - name: Install rust
         uses: actions-rs/toolchain@v1
         with:
           toolchain: stable
+          override: true
 
       - name: Concrete-Optimizer
         run: |
@@ -79,68 +117,74 @@ jobs:
         run: |
           touch keysetcache.timestamp
 
-      - name: Build and test compiler
-        uses: addnab/docker-run-action@v3
-        id: build-compiler
-        with:
-          registry: ghcr.io
-          image: ${{ env.DOCKER_IMAGE_TEST }}
-          username: ${{ secrets.GHCR_LOGIN }}
-          password: ${{ secrets.GHCR_PASSWORD }}
-          options: >-
-            -v ${{ github.workspace }}/llvm-project:/llvm-project
-            -v ${{ github.workspace }}/compiler:/compiler
-            -v ${{ github.workspace }}/KeySetCache:/tmp/KeySetCache
-          shell: bash
-          run: |
-            set -e
-            cd /compiler
-            rm -rf /build
-            make run-benchmarks
+      - name: Build compiler and run end-to-end benchmarks
+        run: |
+          set -e
+          cd compiler
+          make BINDINGS_PYTHON_ENABLED=OFF run-benchmarks
 
       - name: Parse results
         shell: bash
         run: |
-          python3 ./ci/benchmark_parser.py benchmarks_results.json ${RESULTS_FILENAME} \
-          --series-tags '{"commit_hash": "${{ github.sha }}"}'
-          gzip -k ${RESULTS_FILENAME}
+          OPTIMIZER_HASH="$(cd compiler/concrete-optimizer; git rev-parse HEAD)"
+          python3 ./ci/benchmark_parser.py compiler/benchmarks_results.json ${{ env.RESULTS_FILENAME }} \
+          --series-name compiler_end_to_end_benchmarks \
+          --series-help "Concrete compiler end-to-end benchmarks timings" \
+          --series-tags "{\"compiler_hash\": \"${{ github.sha }}\", \"branch\": \"${{ github.ref_name }}\", \"optimizer_hash\": \"${OPTIMIZER_HASH}\", \"hardware\": \"aws ${{ env.EC2_INSTANCE_TYPE }}\"}"
+          gzip -k ${{ env.RESULTS_FILENAME }}
 
       - name: Upload compressed results artifact
         uses: actions/upload-artifact@v3
         with:
           name: ${{ github.sha }}
-          path: ${RESULTS_FILENAME}.gz
+          path: ${{ env.RESULTS_FILENAME }}.gz
+
+      - name: Checkout Slab repo
+        uses: actions/checkout@v3
+        with:
+          repository: zama-ai/slab
+          path: slab
+          token: ${{ secrets.GH_TOKEN }}
 
-      - uses: actions/checkout@v3
       - name: Send data to Slab
         shell: bash
         run: |
           echo "Computing HMac on downloaded artifact"
-          SIGNATURE="$(./ci/hmac_calculator.sh ./${RESULTS_FILENAME} '${{ secrets.JOB_SECRET }}')"
+          SIGNATURE="$(slab/scripts/hmac_calculator.sh ${{ env.RESULTS_FILENAME }} '${{ secrets.JOB_SECRET }}')"
           echo "Sending results to Slab..."
           curl -v -k \
           -H "Content-Type: application/json" \
           -H "X-Slab-Repository: ${{ github.repository }}" \
           -H "X-Slab-Command: plot_data" \
           -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \
-          -d @${RESULTS_FILENAME} \
+          -d @${{ env.RESULTS_FILENAME }} \
           ${{ secrets.SLAB_URL }}
 
 
-  StopRunner:
+  stop-runner:
     name: Stop EC2 runner
     needs:
-      - StartRunner
-      - RunBenchmarks
+      - start-runner
+      - run-benchmarks
     runs-on: ubuntu-20.04
     if: ${{ always() && (needs.start-runner.result != 'skipped') }}
     steps:
+      - name: Sets AWS region for m6i.metal
+        if: ${{ env.EC2_INSTANCE_TYPE == 'm6i.metal' }}
+        run: |
+          echo "AWS_REGION=eu-west-3" >> $GITHUB_ENV
+
+      - name: Sets AWS region for c6a.metal
+        if: ${{ env.EC2_INSTANCE_TYPE == 'c6a.metal' }}
+        run: |
+          echo "AWS_REGION=us-east-1" >> $GITHUB_ENV
+
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:
-          aws-access-key-id: ${{ secrets.AWS_IAM_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_IAM_KEY }}
-          aws-region: eu-west-3
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
       - name: Stop EC2 runner
         uses: machulav/ec2-github-runner@v2
         with:

compiler/Makefile

@@ -229,7 +229,7 @@ build-benchmarks: build-initialized
 	cmake --build $(BUILD_DIR) --target end_to_end_benchmark
 
 run-benchmarks: build-benchmarks
-	$(BUILD_DIR)/bin/end_to_end_benchmark --benchmark_out=benchmarks_results.json --benchmark_out_format=json
+	$(BUILD_DIR)/bin/end_to_end_benchmark --benchmark_out=benchmarks_results.json --benchmark_out_format=json  --benchmark_repetitions=10 --benchmark_report_aggregates_only=true
 
 build-mlbench: build-initialized
 	cmake --build $(BUILD_DIR) --target end_to_end_mlbench