ci(compiler): schedule end-to-end benchmarks

Run end-to-end benchmarks on a weekly basis. Results are parsed and
then sent to Slab CI bot to be later plotted into Grafana.

.github/workflows/benchmark.yml

@@ -0,0 +1,150 @@
+# Run benchmarks on an AWS instance and return parsed results to Slab CI bot.
+name: Performance benchmarks
+
+on:
+  workflow_dispatch:
+  # Have a weekly benchmark run on main branch to be available on Monday morning (Paris time)
+# TODO: uncomment this section once the benchmarks are debugged
+#  schedule:
+#    # * is a special character in YAML so you have to quote this string
+#    # At 1:00 every Thursday
+#    # Timezone is UTC, so Paris time is +2 during the summer and +1 during winter
+#    - cron: '0 1 * * THU'
+
+env:
+  CARGO_TERM_COLOR: always
+  RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json
+  DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler
+
+jobs:
+  StartRunner:
+    name: Start EC2 runner
+    runs-on: ubuntu-20.04
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_IAM_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_IAM_KEY }}
+          aws-region: eu-west-3
+          aws-resource-tags: [["Name", "compiler-benchmarks-github"]]
+      - name: Start EC2 runner
+        id: start-ec2-runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: start
+          github-token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
+          ec2-image-id: ami-00ac986ef330c077f
+          ec2-instance-type: m6i.metal
+          subnet-id: subnet-a886b4c1
+          security-group-id: sg-0bf1c1d79c97bc88f
+
+  RunBenchmarks:
+    name: Execute end-to-end benchmarks in EC2
+    runs-on: ${{ needs.start-runner.outputs.label }}
+    if: ${{ !cancelled() }}
+    needs: StartRunner
+    steps:
+      # SSH private key is required as some dependencies are from private repos
+      - uses: webfactory/ssh-agent@v0.5.2
+        with:
+          ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }}
+
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+          token: ${{ secrets.GH_TOKEN }}
+
+      - name: Install rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+
+      - name: Concrete-Optimizer
+        run: |
+          cd compiler
+          make concrete-optimizer-lib
+
+      - name: Download KeySetCache
+        if: ${{ !contains(github.head_ref, 'newkeysetcache') }}
+        continue-on-error: true
+        run: |
+          cd compiler
+          GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} make keysetcache_ci_populated
+
+      - name: Mark KeySetCache
+        run: |
+          touch keysetcache.timestamp
+
+      - name: Build and test compiler
+        uses: addnab/docker-run-action@v3
+        id: build-compiler
+        with:
+          registry: ghcr.io
+          image: ${{ env.DOCKER_IMAGE_TEST }}
+          username: ${{ secrets.GHCR_LOGIN }}
+          password: ${{ secrets.GHCR_PASSWORD }}
+          options: >-
+            -v ${{ github.workspace }}/llvm-project:/llvm-project
+            -v ${{ github.workspace }}/compiler:/compiler
+            -v ${{ github.workspace }}/KeySetCache:/tmp/KeySetCache
+          shell: bash
+          run: |
+            set -e
+            cd /compiler
+            rm -rf /build
+            make run-benchmarks
+
+      - name: Parse results
+        shell: bash
+        run: |
+          python3 ./ci/benchmark_parser.py benchmarks_results.json ${RESULTS_FILENAME} \
+          --series-tags '{"commit_hash": "${{ github.sha }}"}'
+          gzip -k ${RESULTS_FILENAME}
+
+      - name: Upload compressed results artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ github.sha }}
+          path: ${RESULTS_FILENAME}.gz
+
+      - uses: actions/checkout@v3
+      - name: Send data to Slab
+        shell: bash
+        run: |
+          echo "Computing HMac on downloaded artifact"
+          SIGNATURE="$(./ci/hmac_calculator.sh ./${RESULTS_FILENAME} '${{ secrets.JOB_SECRET }}')"
+          echo "Sending results to Slab..."
+          curl -v -k \
+          -H "Content-Type: application/json" \
+          -H "X-Slab-Repository: ${{ github.repository }}" \
+          -H "X-Slab-Command: plot_data" \
+          -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \
+          -d @${RESULTS_FILENAME} \
+          ${{ secrets.SLAB_URL }}
+
+
+  StopRunner:
+    name: Stop EC2 runner
+    needs:
+      - StartRunner
+      - RunBenchmarks
+    runs-on: ubuntu-20.04
+    if: ${{ always() && (needs.start-runner.result != 'skipped') }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_IAM_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_IAM_KEY }}
+          aws-region: eu-west-3
+      - name: Stop EC2 runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          github-token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
+          label: ${{ needs.start-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}
+          mode: stop

ci/benchmark_parser.py

@@ -0,0 +1,115 @@
+"""
+benchmark_parser
+----------------
+
+Parse benchmark raw results.
+"""
+import argparse
+import pathlib
+import json
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('results_path',
+                    help=('Location of raw benchmark results,'
+                          ' could be either a file or a directory.'
+                          'In a case of a directory, this script will attempt to parse all the'
+                          'files containing a .json extension'))
+parser.add_argument('output_file', help='File storing parsed results')
+parser.add_argument('-n', '--series-name', dest='series_name',
+                    default="concrete_compiler_benchmark_timing",
+                    help='Name of the data series (as stored in Prometheus)')
+parser.add_argument('-e', '--series-help', dest='series_help',
+                    default="Timings of various type of benchmarks in concrete compiler.",
+                    help='Description of the data series (as stored in Prometheus)')
+parser.add_argument('-t', '--series-tags', dest='series_tags',
+                    type=json.loads, default={},
+                    help='Tags to apply to all the points in the data series')
+
+
+def parse_results(raw_results):
+    """
+    Parse raw benchmark results.
+
+    :param raw_results: path to file that contains raw results as :class:`pathlib.Path`
+
+    :return: :class:`list` of data points
+    """
+    result_values = list()
+    raw_results = json.loads(raw_results.read_text())
+    for res in raw_results["benchmarks"]:
+        if not res.get("aggregate_name", None):
+            # Skipping iterations and focus only on aggregated results.
+            continue
+
+        bench_class, action, option_class, application = res["run_name"].split("/")
+
+        for measurement in ("real_time", "cpu_time"):
+            tags = {"bench_class": bench_class,
+                    "action": action,
+                    "option_class": option_class,
+                    "application": application,
+                    "stat": res["aggregate_name"],
+                    "measurement": measurement}
+            result_values.append({"value": res[measurement], "tags": tags})
+
+    return result_values
+
+
+def recursive_parse(directory):
+    """
+    Parse all the benchmark results in a directory. It will attempt to parse all the files having a
+    .json extension at the top-level of this directory.
+
+    :param directory: path to directory that contains raw results as :class:`pathlib.Path`
+
+    :return: :class:`list` of data points
+    """
+    result_values = []
+    for file in directory.glob('*.json'):
+        try:
+            result_values.extend(parse_results(file))
+        except KeyError as err:
+            print(f"Failed to parse '{file.resolve()}': {repr(err)}")
+
+    return result_values
+
+
+def dump_results(parsed_results, filename, series_name,
+                 series_help="", series_tags=None):
+    """
+    Dump parsed results formatted as JSON to file.
+
+    :param parsed_results: :class:`list` of data points
+    :param filename: filename for dump file as :class:`pathlib.Path`
+    :param series_name: name of the data series as :class:`str`
+    :param series_help: description of the data series as :class:`str`
+    :param series_tags: constant tags for the series
+    """
+    filename.parent.mkdir(parents=True, exist_ok=True)
+    series = [
+        {"series_name": series_name,
+         "series_help": series_help,
+         "series_tags": series_tags or dict(),
+         "points": parsed_results},
+    ]
+    filename.write_text(json.dumps(series))
+
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+
+    results_path = pathlib.Path(args.results_path)
+    print("Parsing benchmark results... ")
+    if results_path.is_dir():
+        results = recursive_parse(results_path)
+    else:
+        results = parse_results(results_path)
+    print("Parsing results done")
+
+    output_file = pathlib.Path(args.output_file)
+    print(f"Dump parsed results into '{output_file.resolve()}' ... ", end="")
+    dump_results(results, output_file, args.series_name,
+                 series_help=args.series_help, series_tags=args.series_tags)
+
+    print("Done")

compiler/Makefile

@@ -229,7 +229,7 @@ build-benchmarks: build-initialized
 	cmake --build $(BUILD_DIR) --target end_to_end_benchmark
 
 run-benchmarks: build-benchmarks
-	$(BUILD_DIR)/bin/end_to_end_benchmark
+	$(BUILD_DIR)/bin/end_to_end_benchmark --benchmark_out=benchmarks_results.json --benchmark_out_format=json
 
 build-mlbench: build-initialized
 	cmake --build $(BUILD_DIR) --target end_to_end_mlbench