diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000..7085bad21 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,150 @@ +# Run benchmarks on an AWS instance and return parsed results to Slab CI bot. +name: Performance benchmarks + +on: + workflow_dispatch: + # Have a weekly benchmark run on main branch to be available on Monday morning (Paris time) +# TODO: uncomment this section once the benchmarks are debugged +# schedule: +# # * is a special character in YAML so you have to quote this string +# # At 1:00 every Thursday +# # Timezone is UTC, so Paris time is +2 during the summer and +1 during winter +# - cron: '0 1 * * THU' + +env: + CARGO_TERM_COLOR: always + RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json + DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler + +jobs: + StartRunner: + name: Start EC2 runner + runs-on: ubuntu-20.04 + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_IAM_ID }} + aws-secret-access-key: ${{ secrets.AWS_IAM_KEY }} + aws-region: eu-west-3 + aws-resource-tags: [["Name", "compiler-benchmarks-github"]] + - name: Start EC2 runner + id: start-ec2-runner + uses: machulav/ec2-github-runner@v2 + with: + mode: start + github-token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} + ec2-image-id: ami-00ac986ef330c077f + ec2-instance-type: m6i.metal + subnet-id: subnet-a886b4c1 + security-group-id: sg-0bf1c1d79c97bc88f + + RunBenchmarks: + name: Execute end-to-end benchmarks in EC2 + runs-on: ${{ needs.start-runner.outputs.label }} + if: ${{ !cancelled() }} + needs: StartRunner + steps: + # SSH private key is required as some dependencies are from private repos + - uses: webfactory/ssh-agent@v0.5.2 + with: + ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }} + + - uses: actions/checkout@v2 + with: + submodules: recursive + token: ${{ secrets.GH_TOKEN }} + + - name: Install rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + + - name: Concrete-Optimizer + run: | + cd compiler + make concrete-optimizer-lib + + - name: Download KeySetCache + if: ${{ !contains(github.head_ref, 'newkeysetcache') }} + continue-on-error: true + run: | + cd compiler + GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} make keysetcache_ci_populated + + - name: Mark KeySetCache + run: | + touch keysetcache.timestamp + + - name: Build and test compiler + uses: addnab/docker-run-action@v3 + id: build-compiler + with: + registry: ghcr.io + image: ${{ env.DOCKER_IMAGE_TEST }} + username: ${{ secrets.GHCR_LOGIN }} + password: ${{ secrets.GHCR_PASSWORD }} + options: >- + -v ${{ github.workspace }}/llvm-project:/llvm-project + -v ${{ github.workspace }}/compiler:/compiler + -v ${{ github.workspace }}/KeySetCache:/tmp/KeySetCache + shell: bash + run: | + set -e + cd /compiler + rm -rf /build + make run-benchmarks + + - name: Parse results + shell: bash + run: | + python3 ./ci/benchmark_parser.py benchmarks_results.json ${RESULTS_FILENAME} \ + --series-tags '{"commit_hash": "${{ github.sha }}"}' + gzip -k ${RESULTS_FILENAME} + + - name: Upload compressed results artifact + uses: actions/upload-artifact@v3 + with: + name: ${{ github.sha }} + path: ${RESULTS_FILENAME}.gz + + - uses: actions/checkout@v3 + - name: Send data to Slab + shell: bash + run: | + echo "Computing HMac on downloaded artifact" + SIGNATURE="$(./ci/hmac_calculator.sh ./${RESULTS_FILENAME} '${{ secrets.JOB_SECRET }}')" + echo "Sending results to Slab..." + curl -v -k \ + -H "Content-Type: application/json" \ + -H "X-Slab-Repository: ${{ github.repository }}" \ + -H "X-Slab-Command: plot_data" \ + -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ + -d @${RESULTS_FILENAME} \ + ${{ secrets.SLAB_URL }} + + + StopRunner: + name: Stop EC2 runner + needs: + - StartRunner + - RunBenchmarks + runs-on: ubuntu-20.04 + if: ${{ always() && (needs.start-runner.result != 'skipped') }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_IAM_ID }} + aws-secret-access-key: ${{ secrets.AWS_IAM_KEY }} + aws-region: eu-west-3 + - name: Stop EC2 runner + uses: machulav/ec2-github-runner@v2 + with: + github-token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} + label: ${{ needs.start-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} + mode: stop diff --git a/ci/benchmark_parser.py b/ci/benchmark_parser.py new file mode 100644 index 000000000..b5802d84f --- /dev/null +++ b/ci/benchmark_parser.py @@ -0,0 +1,115 @@ +""" +benchmark_parser +---------------- + +Parse benchmark raw results. +""" +import argparse +import pathlib +import json + + +parser = argparse.ArgumentParser() +parser.add_argument('results_path', + help=('Location of raw benchmark results,' + ' could be either a file or a directory.' + 'In a case of a directory, this script will attempt to parse all the' + 'files containing a .json extension')) +parser.add_argument('output_file', help='File storing parsed results') +parser.add_argument('-n', '--series-name', dest='series_name', + default="concrete_compiler_benchmark_timing", + help='Name of the data series (as stored in Prometheus)') +parser.add_argument('-e', '--series-help', dest='series_help', + default="Timings of various type of benchmarks in concrete compiler.", + help='Description of the data series (as stored in Prometheus)') +parser.add_argument('-t', '--series-tags', dest='series_tags', + type=json.loads, default={}, + help='Tags to apply to all the points in the data series') + + +def parse_results(raw_results): + """ + Parse raw benchmark results. + + :param raw_results: path to file that contains raw results as :class:`pathlib.Path` + + :return: :class:`list` of data points + """ + result_values = list() + raw_results = json.loads(raw_results.read_text()) + for res in raw_results["benchmarks"]: + if not res.get("aggregate_name", None): + # Skipping iterations and focus only on aggregated results. + continue + + bench_class, action, option_class, application = res["run_name"].split("/") + + for measurement in ("real_time", "cpu_time"): + tags = {"bench_class": bench_class, + "action": action, + "option_class": option_class, + "application": application, + "stat": res["aggregate_name"], + "measurement": measurement} + result_values.append({"value": res[measurement], "tags": tags}) + + return result_values + + +def recursive_parse(directory): + """ + Parse all the benchmark results in a directory. It will attempt to parse all the files having a + .json extension at the top-level of this directory. + + :param directory: path to directory that contains raw results as :class:`pathlib.Path` + + :return: :class:`list` of data points + """ + result_values = [] + for file in directory.glob('*.json'): + try: + result_values.extend(parse_results(file)) + except KeyError as err: + print(f"Failed to parse '{file.resolve()}': {repr(err)}") + + return result_values + + +def dump_results(parsed_results, filename, series_name, + series_help="", series_tags=None): + """ + Dump parsed results formatted as JSON to file. + + :param parsed_results: :class:`list` of data points + :param filename: filename for dump file as :class:`pathlib.Path` + :param series_name: name of the data series as :class:`str` + :param series_help: description of the data series as :class:`str` + :param series_tags: constant tags for the series + """ + filename.parent.mkdir(parents=True, exist_ok=True) + series = [ + {"series_name": series_name, + "series_help": series_help, + "series_tags": series_tags or dict(), + "points": parsed_results}, + ] + filename.write_text(json.dumps(series)) + + +if __name__ == "__main__": + args = parser.parse_args() + + results_path = pathlib.Path(args.results_path) + print("Parsing benchmark results... ") + if results_path.is_dir(): + results = recursive_parse(results_path) + else: + results = parse_results(results_path) + print("Parsing results done") + + output_file = pathlib.Path(args.output_file) + print(f"Dump parsed results into '{output_file.resolve()}' ... ", end="") + dump_results(results, output_file, args.series_name, + series_help=args.series_help, series_tags=args.series_tags) + + print("Done") diff --git a/compiler/Makefile b/compiler/Makefile index 3b96fdebb..b01b6fb3a 100644 --- a/compiler/Makefile +++ b/compiler/Makefile @@ -229,7 +229,7 @@ build-benchmarks: build-initialized cmake --build $(BUILD_DIR) --target end_to_end_benchmark run-benchmarks: build-benchmarks - $(BUILD_DIR)/bin/end_to_end_benchmark + $(BUILD_DIR)/bin/end_to_end_benchmark --benchmark_out=benchmarks_results.json --benchmark_out_format=json build-mlbench: build-initialized cmake --build $(BUILD_DIR) --target end_to_end_mlbench