From 8ea446a105fc745e10da0dc5aa1256aea8c382f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Test=C3=A9?= Date: Wed, 23 Nov 2022 11:02:32 +0100 Subject: [PATCH] chore(ci): add benchmark workflow for boolean and shortint These workflows are meant to be triggered by Slab CI bot server. --- .github/workflows/boolean_benchmark.yml | 122 ++++++++++++++++++ .github/workflows/shortint_benchmark.yml | 122 ++++++++++++++++++ ci/benchmark_parser.py | 155 +++++++++++++++++++++++ ci/slab.toml | 10 ++ 4 files changed, 409 insertions(+) create mode 100644 .github/workflows/boolean_benchmark.yml create mode 100644 .github/workflows/shortint_benchmark.yml create mode 100644 ci/benchmark_parser.py diff --git a/.github/workflows/boolean_benchmark.yml b/.github/workflows/boolean_benchmark.yml new file mode 100644 index 000000000..15da4d294 --- /dev/null +++ b/.github/workflows/boolean_benchmark.yml @@ -0,0 +1,122 @@ +# Run boolean benchmarks on an AWS instance and return parsed results to Slab CI bot. +name: Boolean benchmarks + +on: + workflow_dispatch: + inputs: + instance_id: + description: 'Instance ID' + type: string + instance_image_id: + description: 'Instance AMI ID' + type: string + instance_type: + description: 'Instance product type' + type: string + runner_name: + description: 'Action runner name' + type: string + request_id: + description: 'Slab request ID' + type: string + matrix_item: + description: 'Build matrix item' + type: string + + +env: + CARGO_TERM_COLOR: always + RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json + +jobs: + run-boolean-benchmarks: + name: Execute boolean benchmarks in EC2 + runs-on: ${{ github.event.inputs.runner_name }} + if: ${{ !cancelled() }} + steps: + - name: Instance configuration used + run: | + echo "IDs: ${{ inputs.instance_id }}" + echo "AMI: ${{ inputs.instance_image_id }}" + echo "Type: ${{ inputs.instance_type }}" + echo "Request ID: ${{ inputs.request_id }}" + echo "Matrix item: ${{ inputs.matrix_item }}" + + - name: Get benchmark date + run: | + echo "BENCH_DATE=$(date --iso-8601=seconds)" >> "${GITHUB_ENV}" + + - name: Fetch submodules + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.GH_TOKEN }} + + - name: Set up home + # "Install rust" step require root user to have a HOME directory which is not set. + run: | + echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" + + - name: Install rust + uses: actions-rs/toolchain@v1 + with: + toolchain: nightly + override: true + + - name: Run benchmarks + run: | + RUSTFLAGS="-C target-cpu=native" cargo +nightly bench --bench boolean-bench --features=x86_64-unix,boolean,internal-keycache -p tfhe + + - name: Parse results + run: | + COMMIT_DATE="$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})" + COMMIT_HASH="$(git describe --tags --dirty)" + python3 ./ci/benchmark_parser.py target/criterion ${{ env.RESULTS_FILENAME }} \ + --schema tfhe_rs_benchmarks \ + --hardware ${{ inputs.instance_type }} \ + --project-version ${COMMIT_HASH} \ + --branch ${{ github.ref_name }} \ + --commit-date ${COMMIT_DATE} \ + --bench-date "${{ env.BENCH_DATE }}" + + - name: Remove previous raw results + run: | + rm -rf target/criterion + + - name: Run benchmarks with AVX512 + run: | + RUSTFLAGS="-C target-cpu=native" cargo +nightly bench --bench boolean-bench --features=x86_64-unix,boolean,internal-keycache,nightly-avx512 -p tfhe + + - name: Parse AVX512 results + run: | + python3 ./ci/benchmark_parser.py target/criterion ${{ env.RESULTS_FILENAME }} \ + --name-suffix avx512 \ + --append-results + + - name: Upload parsed results artifact + uses: actions/upload-artifact@v3 + with: + name: ${{ github.sha }}_boolean + path: ${{ env.RESULTS_FILENAME }} + + - name: Checkout Slab repo + uses: actions/checkout@v3 + with: + repository: zama-ai/slab + path: slab + token: ${{ secrets.GH_TOKEN }} + + - name: Send data to Slab + shell: bash + run: | + echo "Computing HMac on downloaded artifact" + SIGNATURE="$(slab/scripts/hmac_calculator.sh ${{ env.RESULTS_FILENAME }} '${{ secrets.JOB_SECRET }}')" + echo "Sending results to Slab..." + curl -v -k \ + -H "Content-Type: application/json" \ + -H "X-Slab-Repository: ${{ github.repository }}" \ + -H "X-Slab-Command: store_data" \ + -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ + -d @${{ env.RESULTS_FILENAME }} \ + ${{ secrets.SLAB_URL }} diff --git a/.github/workflows/shortint_benchmark.yml b/.github/workflows/shortint_benchmark.yml new file mode 100644 index 000000000..60b7714e3 --- /dev/null +++ b/.github/workflows/shortint_benchmark.yml @@ -0,0 +1,122 @@ +# Run shortint benchmarks on an AWS instance and return parsed results to Slab CI bot. +name: Shortint benchmarks + +on: + workflow_dispatch: + inputs: + instance_id: + description: 'Instance ID' + type: string + instance_image_id: + description: 'Instance AMI ID' + type: string + instance_type: + description: 'Instance product type' + type: string + runner_name: + description: 'Action runner name' + type: string + request_id: + description: 'Slab request ID' + type: string + matrix_item: + description: 'Build matrix item' + type: string + + +env: + CARGO_TERM_COLOR: always + RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json + +jobs: + run-shortint-benchmarks: + name: Execute shortint benchmarks in EC2 + runs-on: ${{ github.event.inputs.runner_name }} + if: ${{ !cancelled() }} + steps: + - name: Instance configuration used + run: | + echo "IDs: ${{ inputs.instance_id }}" + echo "AMI: ${{ inputs.instance_image_id }}" + echo "Type: ${{ inputs.instance_type }}" + echo "Request ID: ${{ inputs.request_id }}" + echo "Matrix item: ${{ inputs.matrix_item }}" + + - name: Get benchmark date + run: | + echo "BENCH_DATE=$(date --iso-8601=seconds)" >> "${GITHUB_ENV}" + + - name: Fetch submodules + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.GH_TOKEN }} + + - name: Set up home + # "Install rust" step require root user to have a HOME directory which is not set. + run: | + echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" + + - name: Install rust + uses: actions-rs/toolchain@v1 + with: + toolchain: nightly + override: true + + - name: Run benchmarks + run: | + RUSTFLAGS="-C target-cpu=native" cargo +nightly bench --bench shortint-bench --features=x86_64-unix,shortint,internal-keycache -p tfhe + + - name: Parse results + run: | + COMMIT_DATE="$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})" + COMMIT_HASH="$(git describe --tags --dirty)" + python3 ./ci/benchmark_parser.py target/criterion ${{ env.RESULTS_FILENAME }} \ + --schema tfhe_rs_benchmarks \ + --hardware ${{ inputs.instance_type }} \ + --project-version ${COMMIT_HASH} \ + --branch ${{ github.ref_name }} \ + --commit-date ${COMMIT_DATE} \ + --bench-date "${{ env.BENCH_DATE }}" + + - name: Remove previous raw results + run: | + rm -rf target/criterion + + - name: Run benchmarks with AVX512 + run: | + RUSTFLAGS="-C target-cpu=native" cargo +nightly bench --bench shortint-bench --features=x86_64-unix,shortint,internal-keycache,nightly-avx512 -p tfhe + + - name: Parse AVX512 results + run: | + python3 ./ci/benchmark_parser.py target/criterion ${{ env.RESULTS_FILENAME }} \ + --name-suffix avx512 \ + --append-results + + - name: Upload parsed results artifact + uses: actions/upload-artifact@v3 + with: + name: ${{ github.sha }}_shortint + path: ${{ env.RESULTS_FILENAME }} + + - name: Checkout Slab repo + uses: actions/checkout@v3 + with: + repository: zama-ai/slab + path: slab + token: ${{ secrets.GH_TOKEN }} + + - name: Send data to Slab + shell: bash + run: | + echo "Computing HMac on downloaded artifact" + SIGNATURE="$(slab/scripts/hmac_calculator.sh ${{ env.RESULTS_FILENAME }} '${{ secrets.JOB_SECRET }}')" + echo "Sending results to Slab..." + curl -v -k \ + -H "Content-Type: application/json" \ + -H "X-Slab-Repository: ${{ github.repository }}" \ + -H "X-Slab-Command: store_data" \ + -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ + -d @${{ env.RESULTS_FILENAME }} \ + ${{ secrets.SLAB_URL }} diff --git a/ci/benchmark_parser.py b/ci/benchmark_parser.py new file mode 100644 index 000000000..4e538d869 --- /dev/null +++ b/ci/benchmark_parser.py @@ -0,0 +1,155 @@ +""" +benchmark_parser +---------------- + +Parse criterion benchmark results. +""" +import argparse +import pathlib +import json +import sys + + +parser = argparse.ArgumentParser() +parser.add_argument('results_dir', + help='Location of criterion benchmark results directory') +parser.add_argument('output_file', help='File storing parsed results') +parser.add_argument('-s', '--schema', dest='schema', + help='Name of the database schema used to store results') +parser.add_argument('-w', '--hardware', dest='hardware', + help='Hardware reference used to perform benchmark') +parser.add_argument('-V', '--project-version', dest='project_version', + help='Commit hash reference') +parser.add_argument('-b', '--branch', dest='branch', + help='Git branch name on which benchmark was performed') +parser.add_argument('--commit-date', dest='commit_date', + help='Timestamp of commit hash used in project_version') +parser.add_argument('--bench-date', dest='bench_date', + help='Timestamp when benchmark was run') +parser.add_argument('--name-suffix', dest='name_suffix', default='', + help='Suffix to append to each of the result test names') +parser.add_argument('--append-results', dest='append_results', action='store_true', + help='Append parsed results to an existing file') + + +def recursive_parse(directory, name_suffix=""): + """ + Parse all the benchmark results in a directory. It will attempt to parse all the files having a + .json extension at the top-level of this directory. + + :param directory: path to directory that contains raw results as :class:`pathlib.Path` + :param name_suffix: a :class:`str` suffix to apply to each test name found + + :return: :class:`list` of data points + """ + excluded_directories = ["child_generate", "fork", "parent_generate", "report"] + result_values = list() + for dire in directory.iterdir(): + if dire.name in excluded_directories or not dire.is_dir(): + continue + for subdir in dire.iterdir(): + if subdir.name != "new": + continue + + test_name = parse_benchmark_file(subdir) + for stat_name, value in parse_estimate_file(subdir).items(): + test_name_parts = list(filter(None, [test_name, stat_name, name_suffix])) + result_values.append({"value": value, "test": "_".join(test_name_parts)}) + + return result_values + + +def parse_benchmark_file(directory): + """ + Parse file containing details of the parameters used for a benchmark. + + :param directory: directory where a benchmark case results are located as :class:`pathlib.Path` + + :return: name of the test as :class:`str` + """ + raw_results = _parse_file_to_json(directory, "benchmark.json") + return raw_results["full_id"].replace(" ", "_") + + +def parse_estimate_file(directory): + """ + Parse file containing timing results for a benchmark. + + :param directory: directory where a benchmark case results are located as :class:`pathlib.Path` + + :return: :class:`dict` of data points + """ + raw_results = _parse_file_to_json(directory, "estimates.json") + return { + stat_name: raw_results[stat_name]["point_estimate"] + for stat_name in ("mean", "std_dev") + } + + +def _parse_file_to_json(directory, filename): + result_file = directory.joinpath(filename) + return json.loads(result_file.read_text()) + + +def dump_results(parsed_results, filename, input_args): + """ + Dump parsed results formatted as JSON to file. + + :param parsed_results: :class:`list` of data points + :param filename: filename for dump file as :class:`pathlib.Path` + :param input_args: CLI input arguments + """ + if input_args.append_results: + parsed_content = json.loads(filename.read_text()) + parsed_content["points"].extend(parsed_results) + filename.write_text(json.dumps(parsed_content)) + else: + filename.parent.mkdir(parents=True, exist_ok=True) + series = { + "schema": input_args.schema, + "hardware": input_args.hardware, + "project_version": input_args.project_version, + "branch": input_args.branch, + "insert_date": input_args.bench_date, + "commit_date": input_args.commit_date, + "points": parsed_results, + } + filename.write_text(json.dumps(series)) + + +def check_mandatory_args(input_args): + """ + Check for availability of required input arguments, the program will exit if one of them is + not present. If `append_results` flag is set, all the required arguments will be ignored. + + :param input_args: CLI input arguments + """ + if input_args.append_results: + return + + missing_args = list() + for arg_name in vars(input_args): + if arg_name in ["results_dir", "output_file", "name_suffix", "append_results"]: + continue + if not getattr(input_args, arg_name): + missing_args.append(arg_name) + + if missing_args: + for arg_name in missing_args: + print(f"Missing required argument: --{arg_name.replace('_', '-')}") + sys.exit(1) + + +if __name__ == "__main__": + args = parser.parse_args() + check_mandatory_args(args) + + print("Parsing benchmark results... ") + results = recursive_parse(pathlib.Path(args.results_dir), args.name_suffix) + print("Parsing results done") + + output_file = pathlib.Path(args.output_file) + print(f"Dump parsed results into '{output_file.resolve()}' ... ", end="") + dump_results(results, output_file, args) + + print("Done") diff --git a/ci/slab.toml b/ci/slab.toml index ea095dfaa..c30fbca80 100644 --- a/ci/slab.toml +++ b/ci/slab.toml @@ -19,3 +19,13 @@ check_run_name = "Shortint CPU AWS Tests" workflow = "aws_tfhe_tests_w_gpu.yml" profile = "gpu" check_run_name = "AWS tests GPU (Slab)" + +[command.shortint_bench] +workflow = "shortint_benchmark.yml" +profile = "cpu-big" +check_run_name = "Shortint CPU AWS Benchmarks" + +[command.boolean_bench] +workflow = "boolean_benchmark.yml" +profile = "cpu-big" +check_run_name = "Boolean CPU AWS Benchmarks"