From bb728959ec0c3fabb282042f955869caf755f72c Mon Sep 17 00:00:00 2001 From: Quentin Bourgerie Date: Tue, 7 Mar 2023 15:04:50 +0100 Subject: [PATCH] chore: Integrate concrete-optimizer CI --- .github/workflows/compiler.yml | 79 ------------------- .../compiler_macos_build_and_test.yml | 2 +- .github/workflows/llvm-compatibility.yml | 76 ------------------ .github/workflows/main.yml | 10 ++- .github/workflows/optimizer.yml | 58 ++++++++------ .../{setup => optimizer_setup}/action.yml | 0 .github/workflows/start_aws_builds.yml | 2 +- compilers/concrete-optimizer/Cargo.toml | 2 + .../concrete-optimizer/optimizer | 0 9 files changed, 49 insertions(+), 180 deletions(-) delete mode 100644 .github/workflows/compiler.yml delete mode 100644 .github/workflows/llvm-compatibility.yml rename .github/workflows/{setup => optimizer_setup}/action.yml (100%) rename optimizer => compilers/concrete-optimizer/optimizer (100%) diff --git a/.github/workflows/compiler.yml b/.github/workflows/compiler.yml deleted file mode 100644 index 1ecea9c5d..000000000 --- a/.github/workflows/compiler.yml +++ /dev/null @@ -1,79 +0,0 @@ -name: Continuous integration - compiler - -on: - push: - branches: [ main, testci ] - pull_request: - -env: - CARGO_TERM_COLOR: always - -jobs: - conditions: - runs-on: ubuntu-20.04 - outputs: - proceed: ${{ steps.interface-changes.outputs.any_changed == 'true' || github.ref == 'refs/heads/main' }} - steps: - - name: Code checkout - uses: actions/checkout@v3 - - - name: Detect changes on API to compiler - uses: tj-actions/changed-files@v34 - id: interface-changes - with: - files: | - concrete-optimizer-cpp/src/cpp/* - - - run: echo ${{ steps.interface-changes.outputs.any_changed }} - - check-compiler-compatibility: - runs-on: ubuntu-20.04 - needs: conditions - if: ${{ needs.conditions.outputs.proceed == 'true' }} - steps: - # A SSH private key is required as some dependencies are from private repos - - name: Set ssh keys - uses: webfactory/ssh-agent@v0.6.0 - with: - ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }} - - - name: Compiler checkout - uses: actions/checkout@v3 - with: - submodules: recursive - repository: zama-ai/concrete-compiler-internal - ssh-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }} - - - name: Override optimizer for compiler - uses: actions/checkout@v3 - with: - path: compiler/concrete-optimizer - - - name: Install rust - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - - name: Download cargo cache - uses: Swatinem/rust-cache@v2 - - - name: Concrete-Optimizer - run: | - cd compiler - make concrete-optimizer-lib - - - name: Build compiler - uses: addnab/docker-run-action@v3 - with: - registry: ghcr.io - image: ghcr.io/zama-ai/concrete-compiler - username: zama-bot - password: ${{ secrets.ZAMA_BOT_TOKEN }} - options: >- - -v ${{ github.workspace }}/compiler:/compiler - shell: bash - run: | - set -e - cd /compiler - export PYTHONPATH="" - make CCACHE=ON BUILD_DIR=/build concretecompiler diff --git a/.github/workflows/compiler_macos_build_and_test.yml b/.github/workflows/compiler_macos_build_and_test.yml index 37b6cbe51..5607b747b 100644 --- a/.github/workflows/compiler_macos_build_and_test.yml +++ b/.github/workflows/compiler_macos_build_and_test.yml @@ -1,5 +1,5 @@ # Perform a build on MacOS platform with M1 chip. -name: MacOsBuild +name: Compiler - Build and Test (MacOS) on: workflow_call: diff --git a/.github/workflows/llvm-compatibility.yml b/.github/workflows/llvm-compatibility.yml deleted file mode 100644 index 9a98b9a46..000000000 --- a/.github/workflows/llvm-compatibility.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: LLVM Compatibility - -on: - workflow_dispatch: - schedule: - - cron: '0 00 * * 1' # Every Monday @ 00:00 UTC - -jobs: - build_test: - name: Build & test the Docker image with latest LLVM - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - with: - submodules: recursive - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Rebase LLVM - run: | - cd llvm-project - git remote add base git@github.com:llvm/llvm-project.git - git fetch base - git rebase base/main - - - name: Log LLVM commit - run: echo "LLVM commit" && cd ${{ github.workspace }}/llvm-project && git log -1 - - - name: login - run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io - - - name: Build - run: docker image build --no-cache -t compiler-latest-llvm -f builders/Dockerfile.concrete-compiler-env . - - - name: Test compiler with latest LLVM - uses: addnab/docker-run-action@v3 - with: - image: compiler-latest-llvm - run: | - cd /compiler - pip install pytest - make BUILD_DIR=/build run-tests - - - name: Update Custom LLVM - uses: ad-m/github-push-action@master - with: - github_token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - branch: main - force: true - repository: zama-ai/concrete-compiler-internal-llvm-project - - - name: Update LLVM submodule - id: update-llvm - run: | - git submodule update --remote - echo "::set-output name=commit::$(cd llvm-project && git rev-parse --short HEAD)" - - - name: Commit latest LLVM version - uses: EndBug/add-and-commit@v7 - with: - add: ./llvm-project - default_author: github_actions - message: "chore: update LLVM to ${{ steps.update-llvm.outputs.commit }}" - - - name: Send Slack Notification - if: ${{ always() }} - continue-on-error: true - uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7 - env: - SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} - SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png - SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "Compatibility check with latest LLVM finished with status ${{ job.status }} \ - (${{ env.ACTION_RUN_URL }})" - SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 15d1d755e..f89c2ea78 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,4 +1,4 @@ -# Trigger an AWS build each time commits are pushed to a pull request. +# This job is the main jobs will dispatch build and test for every modules of our mono repo. name: Main on: @@ -58,3 +58,11 @@ jobs: if: needs.file-change.outputs.compiler == 'true' uses: ./.github/workflows/compiler_macos_build_and_test.yml secrets: inherit + +################################################# +# Optimizer jobs ################################ + optimizer: + needs: file-change + if: needs.file-change.outputs.optimizer == 'true' + uses: ./.github/workflows/optimizer.yml + secrets: inherit diff --git a/.github/workflows/optimizer.yml b/.github/workflows/optimizer.yml index 849fb813c..dc56b1488 100644 --- a/.github/workflows/optimizer.yml +++ b/.github/workflows/optimizer.yml @@ -1,14 +1,17 @@ -name: Continuous integration +name: Optimizer - Tests on: - push: - branches: [ main, testci ] - pull_request: + workflow_call: + secrets: + CONCRETE_CI_SSH_PRIVATE: + required: true + CONCRETE_ACTIONS_TOKEN: + required: true env: CARGO_TERM_COLOR: always jobs: - check-concrete-optimizer: + tests: strategy: matrix: os: [ubuntu-20.04, macos-11] @@ -19,37 +22,46 @@ jobs: - uses: actions/checkout@v3 - name: "Setup" - uses: ./.github/workflows/setup + uses: ./.github/workflows/optimizer_setup with: - ssh_private_key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }} + ssh_private_key: ${{ secrets.CONCRETE_CI_SSH_PRIVATE }} - name: Formatting - run: cargo fmt --check + run: | + cd compilers/concrete-optimizer + cargo fmt --check - name: Build - run: cargo build --release --all-targets + run: | + cd compilers/concrete-optimizer + cargo build --release --all-targets - name: Lint - run: cargo clippy --release --all-targets + run: | + cd compilers/concrete-optimizer + cargo clippy --release --all-targets - name: Tests if: matrix.os == 'ubuntu-20.04' run: | + cd compilers/concrete-optimizer cargo test --release --no-fail-fast --all-targets make -C concrete-optimizer-cpp test-ci - benchmark-concrete-optimizer: + benchmarks: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v3 - name: "Setup" - uses: ./.github/workflows/setup + uses: ./.github/workflows/optimizer_setup with: - ssh_private_key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }} + ssh_private_key: ${{ secrets.CONCRETE_CI_SSH_PRIVATE }} - name: Run benchmark - run: cargo bench -p v0-parameters -- --output-format bencher | tee bench_result.txt + run: | + cd compilers/concrete-optimizer + cargo bench -p v0-parameters -- --output-format bencher | tee bench_result.txt - name: Download PR base benchmark data if: ${{ github.event_name == 'pull_request' }} @@ -69,7 +81,7 @@ jobs: uses: benchmark-action/github-action-benchmark@v1 with: tool: 'cargo' - output-file-path: bench_result.txt + output-file-path: ./compilers/concrete-optimizer/bench_result.txt external-data-json-path: ./benchmark/benchmark-data.json # GitHub API token to make a commit comment github-token: ${{ secrets.GITHUB_TOKEN }} @@ -79,12 +91,10 @@ jobs: - name: Upload benchmark data uses: actions/upload-artifact@v3 with: - path: ./benchmark + path: ./compilers/concrete-optimizer/benchmark name: ${{ runner.os }}-benchmark - - - nightly-check-concrete-optimizer: + nightly-check: runs-on: ubuntu-20.04 env: RUSTFLAGS: -D warnings @@ -100,10 +110,14 @@ jobs: - name: Set ssh keys uses: webfactory/ssh-agent@v0.6.0 with: - ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }} + ssh-private-key: ${{ secrets.CONCRETE_CI_SSH_PRIVATE }} - name: Formatting - run: cargo fmt --check + run: | + cd compilers/concrete-optimizer + cargo fmt --check - name: Lint - run: cargo clippy --release --all-targets + run: | + cd compilers/concrete-optimizer + cargo clippy --release --all-targets diff --git a/.github/workflows/setup/action.yml b/.github/workflows/optimizer_setup/action.yml similarity index 100% rename from .github/workflows/setup/action.yml rename to .github/workflows/optimizer_setup/action.yml diff --git a/.github/workflows/start_aws_builds.yml b/.github/workflows/start_aws_builds.yml index a63e936cc..ce6c665fe 100644 --- a/.github/workflows/start_aws_builds.yml +++ b/.github/workflows/start_aws_builds.yml @@ -3,7 +3,7 @@ name: Start all AWS builds on: push: - branch: + branches: - 'main' workflow_dispatch: diff --git a/compilers/concrete-optimizer/Cargo.toml b/compilers/concrete-optimizer/Cargo.toml index 72399dcee..9a347dc77 100644 --- a/compilers/concrete-optimizer/Cargo.toml +++ b/compilers/concrete-optimizer/Cargo.toml @@ -9,6 +9,8 @@ members = [ [profile.test] + + overflow-checks = true [profile.release] diff --git a/optimizer b/compilers/concrete-optimizer/optimizer similarity index 100% rename from optimizer rename to compilers/concrete-optimizer/optimizer