diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml deleted file mode 100644 index dc11054d2..000000000 --- a/.github/workflows/benchmark.yml +++ /dev/null @@ -1,256 +0,0 @@ -# Run benchmarks on an AWS instance and return parsed results to Slab CI bot. -name: Performance benchmarks - -on: - push: - branches: - - 'main' - - 'test-bench-ci' - workflow_dispatch: - inputs: - backend: - description: 'Backend type' - required: true - default: 'cpu' - type: choice - options: - - cpu - - gpu - benchmark-name: - description: 'Benchmark name' - required: true - default: 'standard' - type: choice - options: - - standard - - application - ec2-instance-type: - description: 'EC2 instance type' - required: true - default: 'm6i.metal' - type: choice - options: - - m6i.metal - - c6a.metal - -env: - CARGO_TERM_COLOR: always - RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json - EC2_INSTANCE_TYPE: ${{ inputs.ec2-instance-type || 'm6i.metal' }} - CUDA_PATH: /usr/local/cuda-11.3 - GCC_VERSION: 8 - BACKEND: ${{ inputs.backend || 'cpu' }} - BENCHMARK_NAME: ${{ inputs.benchmark-name || 'standard' }} - -jobs: - start-runner: - name: Start EC2 runner - runs-on: ubuntu-20.04 - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Sets env vars for m6i.metal - if: ${{ env.EC2_INSTANCE_TYPE == 'm6i.metal' }} - run: | - echo "AWS_REGION=eu-west-3" >> $GITHUB_ENV - echo "EC2_IMAGE_ID=ami-0a24aaee029d1295c" >> $GITHUB_ENV - echo "SUBNET_ID=subnet-a886b4c1" >> $GITHUB_ENV - echo "SECURITY_GROUP_ID=sg-0bf1c1d79c97bc88f" >> $GITHUB_ENV - - - name: Sets env vars for c6a.metal - if: ${{ env.EC2_INSTANCE_TYPE == 'c6a.metal' }} - run: | - echo "AWS_REGION=us-east-1" >> $GITHUB_ENV - echo "EC2_IMAGE_ID=ami-0afb83d80b3b060d8" >> $GITHUB_ENV - echo "SUBNET_ID=subnet-da319dd4" >> $GITHUB_ENV - echo "SECURITY_GROUP_ID=sg-0f8b52622a2669491" >> $GITHUB_ENV - - - name: Sets env vars for p3.2xlarge - if: ${{ env.BACKEND == 'gpu' }} - run: | - echo "AWS_REGION=us-east-1" >> $GITHUB_ENV - echo "EC2_INSTANCE_TYPE=p3.2xlarge" >> $GITHUB_ENV - echo "EC2_IMAGE_ID=ami-03deb184ab492226b" >> $GITHUB_ENV - echo "SUBNET_ID=subnet-8123c9e7" >> $GITHUB_ENV - echo "SECURITY_GROUP_ID=sg-0f8b52622a2669491" >> $GITHUB_ENV - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_IAM_ID }} - aws-secret-access-key: ${{ secrets.AWS_IAM_KEY }} - aws-region: ${{ env.AWS_REGION }} - - name: Start EC2 runner - id: start-ec2-runner - uses: machulav/ec2-github-runner@v2 - with: - mode: start - github-token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - ec2-image-id: ${{ env.EC2_IMAGE_ID }} - ec2-instance-type: ${{ env.EC2_INSTANCE_TYPE }} - subnet-id: ${{ env.SUBNET_ID }} - security-group-id: ${{ env.SECURITY_GROUP_ID }} - aws-resource-tags: > - [ - {"Key": "Name", "Value": "compiler-benchmarks-github"} - ] - - run-benchmarks: - name: Execute end-to-end benchmarks in EC2 - runs-on: ${{ needs.start-runner.outputs.label }} - if: ${{ !cancelled() }} - needs: start-runner - steps: - - name: Get benchmark date - run: | - echo "BENCH_DATE=$(date --iso-8601=seconds)" >> "${GITHUB_ENV}" - - # SSH private key is required as some dependencies are from private repos - - uses: webfactory/ssh-agent@v0.5.2 - with: - ssh-private-key: ${{ secrets.CONCRETE_CI_SSH_PRIVATE }} - - - name: Fetch submodules - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: recursive - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Set up home - # "Install rust" step require root user to have a HOME directory which is not set. - run: | - echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - - - name: Export specific variables (CPU) - if: ${{ env.BACKEND == 'cpu' }} - run: | - echo "CUDA_SUPPORT=OFF" >> "${GITHUB_ENV}" - echo "BENCHMARK_TARGET=run-cpu-benchmarks" >> "${GITHUB_ENV}" - - - name: Export specific variables (GPU) - if: ${{ env.BACKEND == 'gpu' }} - run: | - echo "CUDA_SUPPORT=ON" >> "${GITHUB_ENV}" - echo "BENCHMARK_TARGET=run-gpu-benchmarks" >> "${GITHUB_ENV}" - echo "CUDA_PATH=$CUDA_PATH" >> "${GITHUB_ENV}" - echo "$CUDA_PATH/bin" >> "${GITHUB_PATH}" - echo "LD_LIBRARY_PATH=$CUDA_PATH/lib:$LD_LIBRARY_PATH" >> "${GITHUB_ENV}" - echo "CC=/usr/bin/gcc-${{ env.GCC_VERSION }}" >> "${GITHUB_ENV}" - echo "CXX=/usr/bin/g++-${{ env.GCC_VERSION }}" >> "${GITHUB_ENV}" - echo "CUDAHOSTCXX=/usr/bin/g++-${{ env.GCC_VERSION }}" >> "${GITHUB_ENV}" - echo "CUDACXX=$CUDA_PATH/bin/nvcc" >> "${GITHUB_ENV}" - - - name: Setup environment variable for benchmark target - if: ${{ env.BENCHMARK_NAME != 'standard' }} - run: | - echo "BENCHMARK_TARGET=${{ env.BENCHMARK_TARGET }}-${{ env.BENCHMARK_NAME }}" >> "${GITHUB_ENV}" - - - name: Install rust - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - - - name: Build compiler benchmarks - run: | - set -e - cd compiler - make BINDINGS_PYTHON_ENABLED=OFF CUDA_SUPPORT=${{ env.CUDA_SUPPORT }} build-benchmarks - - - name: Download KeySetCache - if: ${{ !contains(github.head_ref, 'newkeysetcache') }} - continue-on-error: true - run: | - cd compiler - GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} make keysetcache_ci_populated - - - name: Run end-to-end benchmarks - run: | - set -e - cd compiler - make ${{ env.BENCHMARK_TARGET }} - - - name: Upload raw results artifact - uses: actions/upload-artifact@v3 - with: - name: ${{ github.sha }}_raw - path: compiler/benchmarks_results.json - - - name: Parse results - shell: bash - run: | - COMMIT_DATE="$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})" - COMMIT_HASH="$(git describe --tags --dirty)" - python3 ./ci/benchmark_parser.py compiler/benchmarks_results.json ${{ env.RESULTS_FILENAME }} \ - --database compiler_benchmarks \ - --hardware ${{ env.EC2_INSTANCE_TYPE }} \ - --project-version ${COMMIT_HASH} \ - --branch ${{ github.ref_name }} \ - --commit-date ${COMMIT_DATE} \ - --bench-date "${{ env.BENCH_DATE }}" - - - name: Upload parsed results artifact - uses: actions/upload-artifact@v3 - with: - name: ${{ github.sha }} - path: ${{ env.RESULTS_FILENAME }} - - - name: Checkout Slab repo - uses: actions/checkout@v3 - with: - repository: zama-ai/slab - path: slab - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Send data to Slab - shell: bash - run: | - echo "Computing HMac on downloaded artifact" - SIGNATURE="$(slab/scripts/hmac_calculator.sh ${{ env.RESULTS_FILENAME }} '${{ secrets.JOB_SECRET }}')" - echo "Sending results to Slab..." - curl -v -k \ - -H "Content-Type: application/json" \ - -H "X-Slab-Repository: ${{ github.repository }}" \ - -H "X-Slab-Command: store_data" \ - -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ - -d @${{ env.RESULTS_FILENAME }} \ - ${{ secrets.SLAB_URL }} - - stop-runner: - name: Stop EC2 runner - needs: - - start-runner - - run-benchmarks - runs-on: ubuntu-20.04 - if: ${{ always() && (needs.start-runner.result != 'skipped') }} - steps: - - name: Sets AWS region for m6i.metal - if: ${{ env.EC2_INSTANCE_TYPE == 'm6i.metal' }} - run: | - echo "AWS_REGION=eu-west-3" >> $GITHUB_ENV - - - name: Sets AWS region for c6a.metal - if: ${{ env.EC2_INSTANCE_TYPE == 'c6a.metal' }} - run: | - echo "AWS_REGION=us-east-1" >> $GITHUB_ENV - - - name: Sets AWS region for p3.2xlarge - if: ${{ env.BACKEND == 'gpu' }} - run: | - echo "AWS_REGION=us-east-1" >> $GITHUB_ENV - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_IAM_ID }} - aws-secret-access-key: ${{ secrets.AWS_IAM_KEY }} - aws-region: ${{ env.AWS_REGION }} - - name: Stop EC2 runner - uses: machulav/ec2-github-runner@v2 - with: - github-token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - label: ${{ needs.start-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} - mode: stop diff --git a/.github/workflows/compiler_benchmark.yml b/.github/workflows/compiler_benchmark.yml new file mode 100644 index 000000000..b4b677d3d --- /dev/null +++ b/.github/workflows/compiler_benchmark.yml @@ -0,0 +1,151 @@ +# Run benchmarks on an AWS instance for compiler and return parsed results to Slab CI bot. +name: Compiler - Performance benchmarks + +on: + workflow_dispatch: + inputs: + instance_id: + description: 'Instance ID' + type: string + instance_image_id: + description: 'Instance AMI ID' + type: string + instance_type: + description: 'Instance product type' + type: string + runner_name: + description: 'Action runner name' + type: string + request_id: + description: 'Slab request ID' + type: string + +env: + CARGO_TERM_COLOR: always + RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json + CUDA_PATH: /usr/local/cuda-11.3 + GCC_VERSION: 8 + +jobs: + run-benchmarks: + name: Execute end-to-end benchmarks in EC2 + runs-on: ${{ github.event.inputs.runner_name }} + if: ${{ !cancelled() }} + steps: + - name: Instance configuration used + run: | + echo "IDs: ${{ inputs.instance_id }}" + echo "AMI: ${{ inputs.instance_image_id }}" + echo "Type: ${{ inputs.instance_type }}" + echo "Request ID: ${{ inputs.request_id }}" + + - name: Get benchmark date + run: | + echo "BENCH_DATE=$(date --iso-8601=seconds)" >> "${GITHUB_ENV}" + + # SSH private key is required as some dependencies are from private repos + - uses: webfactory/ssh-agent@v0.5.2 + with: + ssh-private-key: ${{ secrets.CONCRETE_CI_SSH_PRIVATE }} + + - name: Fetch submodules + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} + + - name: Set up home + # "Install rust" step require root user to have a HOME directory which is not set. + run: | + echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" + + - name: Export specific variables (CPU) + if: ${{ !startswith(inputs.instance_type, 'p3.') }} + run: | + echo "CUDA_SUPPORT=OFF" >> "${GITHUB_ENV}" + echo "BENCHMARK_TARGET=run-cpu-benchmarks" >> "${GITHUB_ENV}" + + - name: Export specific variables (GPU) + if: ${{ startswith(inputs.instance_type, 'p3.') }} + run: | + echo "CUDA_SUPPORT=ON" >> "${GITHUB_ENV}" + echo "BENCHMARK_TARGET=run-gpu-benchmarks" >> "${GITHUB_ENV}" + echo "CUDA_PATH=$CUDA_PATH" >> "${GITHUB_ENV}" + echo "$CUDA_PATH/bin" >> "${GITHUB_PATH}" + echo "LD_LIBRARY_PATH=$CUDA_PATH/lib:$LD_LIBRARY_PATH" >> "${GITHUB_ENV}" + echo "CC=/usr/bin/gcc-${{ env.GCC_VERSION }}" >> "${GITHUB_ENV}" + echo "CXX=/usr/bin/g++-${{ env.GCC_VERSION }}" >> "${GITHUB_ENV}" + echo "CUDAHOSTCXX=/usr/bin/g++-${{ env.GCC_VERSION }}" >> "${GITHUB_ENV}" + echo "CUDACXX=$CUDA_PATH/bin/nvcc" >> "${GITHUB_ENV}" + + - name: Install rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + + - name: Build compiler benchmarks + run: | + set -e + cd compilers/concrete-compiler/compiler + make BINDINGS_PYTHON_ENABLED=OFF CUDA_SUPPORT=${{ env.CUDA_SUPPORT }} build-benchmarks + + - name: Download KeySetCache + if: ${{ !contains(github.head_ref, 'newkeysetcache') }} + continue-on-error: true + run: | + cd compilers/concrete-compiler/compiler + GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} make keysetcache_ci_populated + + - name: Run end-to-end benchmarks + run: | + set -e + cd compilers/concrete-compiler/compiler + make ${{ env.BENCHMARK_TARGET }} + + - name: Upload raw results artifact + uses: actions/upload-artifact@v3 + with: + name: compiler_${{ github.sha }}_raw + path: compilers/concrete-compiler/compiler/benchmarks_results.json + + - name: Parse results + shell: bash + run: | + COMMIT_DATE="$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})" + COMMIT_HASH="$(git describe --tags --dirty)" + python3 ./ci/benchmark_parser.py compilers/concrete-compiler/compiler/benchmarks_results.json ${{ env.RESULTS_FILENAME }} \ + --database compiler_benchmarks \ + --hardware ${{ inputs.instance_type }} \ + --project-version ${COMMIT_HASH} \ + --branch ${{ github.ref_name }} \ + --commit-date ${COMMIT_DATE} \ + --bench-date "${{ env.BENCH_DATE }}" + + - name: Upload parsed results artifact + uses: actions/upload-artifact@v3 + with: + name: compiler_${{ github.sha }} + path: ${{ env.RESULTS_FILENAME }} + + - name: Checkout Slab repo + uses: actions/checkout@v3 + with: + repository: zama-ai/slab + path: slab + token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} + + - name: Send data to Slab + shell: bash + run: | + echo "Computing HMac on downloaded artifact" + SIGNATURE="$(slab/scripts/hmac_calculator.sh ${{ env.RESULTS_FILENAME }} '${{ secrets.JOB_SECRET }}')" + echo "Sending results to Slab..." + curl -v -k \ + -H "Content-Type: application/json" \ + -H "X-Slab-Repository: ${{ github.repository }}" \ + -H "X-Slab-Command: store_data" \ + -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ + -d @${{ env.RESULTS_FILENAME }} \ + ${{ secrets.SLAB_URL }} diff --git a/.github/workflows/compiler_start_aws_benchmarks.yml b/.github/workflows/compiler_start_aws_benchmarks.yml new file mode 100644 index 000000000..fcdad4aee --- /dev/null +++ b/.github/workflows/compiler_start_aws_benchmarks.yml @@ -0,0 +1,36 @@ +# Trigger benchmarks jobs on Slab CI bot. +name: Compiler - Trigger AWS benchmarks + +on: + push: + branches: + - 'main' + workflow_dispatch: + +jobs: + trigger-benchmarks: + strategy: + matrix: + command: [compiler-cpu-benchmark, compiler-gpu-benchmark] + runs-on: ubuntu-latest + steps: + - name: Checkout Slab repo + uses: actions/checkout@v3 + with: + repository: zama-ai/slab + path: slab + token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} + + - name: Start AWS job in Slab + shell: bash + # TODO: step result must be correlated to HTTP return code. + run: | + echo -n '{"command": "${{ matrix.command }}", "git_ref": "${{ github.ref }}", "sha": "${{ github.sha }}"}' > command.json + SIGNATURE="$(slab/scripts/hmac_calculator.sh command.json '${{ secrets.JOB_SECRET }}')" + curl -v -k \ + -H "Content-Type: application/json" \ + -H "X-Slab-Repository: ${{ github.repository }}" \ + -H "X-Slab-Command: start_aws" \ + -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ + -d @command.json \ + ${{ secrets.SLAB_URL }} diff --git a/ci/slab.toml b/ci/slab.toml index 07b1e5a32..d440c064d 100644 --- a/ci/slab.toml +++ b/ci/slab.toml @@ -33,6 +33,16 @@ workflow = "compiler_build_and_test_gpu.yml" profile = "gpu" check_run_name = "Compiler Build and Test (GPU)" +[command.compiler-cpu-benchmark] +workflow = "compiler_benchmark.yml" +profile = "m6i-old" +check_run_name = "Compiler Performances Benchmarks (CPU)" + +[command.compiler-gpu-benchmark] +workflow = "compiler_benchmark.yml" +profile = "gpu" +check_run_name = "Compiler Performances Benchmarks (GPU)" + # Trigger Docker images build [command.docker-images-build] workflow = "publish_docker_images.yml"