chore: Integrate concrete-optimizer CI

.github/workflows/compiler.yml

@@ -1,79 +0,0 @@
-name: Continuous integration - compiler
-
-on:
-  push:
-    branches: [ main, testci ]
-  pull_request:
-
-env:
-  CARGO_TERM_COLOR: always
-
-jobs:
-  conditions:
-    runs-on: ubuntu-20.04
-    outputs:
-      proceed: ${{ steps.interface-changes.outputs.any_changed == 'true' || github.ref == 'refs/heads/main' }}
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v3
-
-      - name: Detect changes on API to compiler
-        uses: tj-actions/changed-files@v34
-        id: interface-changes
-        with:
-          files: |
-            concrete-optimizer-cpp/src/cpp/*
-
-      - run: echo ${{ steps.interface-changes.outputs.any_changed }}
-
-  check-compiler-compatibility:
-    runs-on: ubuntu-20.04
-    needs: conditions
-    if: ${{ needs.conditions.outputs.proceed == 'true' }}
-    steps:
-      # A SSH private key is required as some dependencies are from private repos
-      - name: Set ssh keys
-        uses: webfactory/ssh-agent@v0.6.0
-        with:
-          ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }}
-
-      - name: Compiler checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: recursive
-          repository: zama-ai/concrete-compiler-internal
-          ssh-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }}
-
-      - name: Override optimizer for compiler
-        uses: actions/checkout@v3
-        with:
-          path: compiler/concrete-optimizer
-
-      - name: Install rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-
-      - name: Download cargo cache
-        uses: Swatinem/rust-cache@v2
-
-      - name: Concrete-Optimizer
-        run: |
-          cd compiler
-          make concrete-optimizer-lib
-
-      - name: Build compiler
-        uses: addnab/docker-run-action@v3
-        with:
-          registry: ghcr.io
-          image: ghcr.io/zama-ai/concrete-compiler
-          username: zama-bot
-          password: ${{ secrets.ZAMA_BOT_TOKEN }}
-          options: >-
-            -v ${{ github.workspace }}/compiler:/compiler
-          shell: bash
-          run: |
-            set -e
-            cd /compiler
-            export PYTHONPATH=""
-            make CCACHE=ON BUILD_DIR=/build concretecompiler

.github/workflows/compiler_macos_build_and_test.yml

@@ -1,5 +1,5 @@
 # Perform a build on MacOS platform with M1 chip.
-name: MacOsBuild
+name: Compiler - Build and Test (MacOS)
 
 on:
   workflow_call:

.github/workflows/llvm-compatibility.yml

@@ -1,76 +0,0 @@
-name: LLVM Compatibility
-
-on:
-  workflow_dispatch:
-  schedule:
-    - cron: '0 00 * * 1' # Every Monday @ 00:00 UTC
-
-jobs:
-  build_test:
-    name: Build & test the Docker image with latest LLVM
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: recursive
-          token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
-
-      - name: Rebase LLVM
-        run: |
-          cd llvm-project
-          git remote add base git@github.com:llvm/llvm-project.git
-          git fetch base
-          git rebase base/main
-
-      - name: Log LLVM commit
-        run: echo "LLVM commit" && cd ${{ github.workspace }}/llvm-project && git log -1
-
-      - name: login
-        run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io
-
-      - name: Build
-        run: docker image build --no-cache -t compiler-latest-llvm -f builders/Dockerfile.concrete-compiler-env .
-
-      - name: Test compiler with latest LLVM
-        uses: addnab/docker-run-action@v3
-        with:
-          image: compiler-latest-llvm
-          run: |
-            cd /compiler
-            pip install pytest
-            make BUILD_DIR=/build run-tests
-
-      - name: Update Custom LLVM
-        uses: ad-m/github-push-action@master
-        with:
-          github_token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
-          branch: main
-          force: true
-          repository: zama-ai/concrete-compiler-internal-llvm-project
-
-      - name: Update LLVM submodule
-        id: update-llvm
-        run: |
-          git submodule update --remote
-          echo "::set-output name=commit::$(cd llvm-project && git rev-parse --short HEAD)"
-
-      - name: Commit latest LLVM version
-        uses: EndBug/add-and-commit@v7
-        with:
-          add: ./llvm-project
-          default_author: github_actions
-          message: "chore: update LLVM to ${{ steps.update-llvm.outputs.commit }}"
-
-      - name: Send Slack Notification
-        if: ${{ always() }}
-        continue-on-error: true
-        uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7
-        env:
-          SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
-          SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
-          SLACK_COLOR: ${{ job.status }}
-          SLACK_MESSAGE: "Compatibility check with latest LLVM finished with status ${{ job.status }} \
-            (${{ env.ACTION_RUN_URL }})"
-          SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
-          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}

.github/workflows/main.yml

@@ -1,4 +1,4 @@
-# Trigger an AWS build each time commits are pushed to a pull request.
+# This job is the main jobs will dispatch build and test for every modules of our mono repo.
 name: Main
 
 on:
@@ -58,3 +58,11 @@ jobs:
     if: needs.file-change.outputs.compiler == 'true'
     uses: ./.github/workflows/compiler_macos_build_and_test.yml
     secrets: inherit
+
+#################################################
+# Optimizer jobs ################################
+  optimizer:
+    needs: file-change
+    if: needs.file-change.outputs.optimizer == 'true'
+    uses: ./.github/workflows/optimizer.yml
+    secrets: inherit

.github/workflows/optimizer.yml

@@ -1,14 +1,17 @@
-name: Continuous integration
+name: Optimizer - Tests
 
 on:
-  push:
-    branches: [ main, testci ]
-  pull_request:
+  workflow_call:
+    secrets:
+      CONCRETE_CI_SSH_PRIVATE:
+        required: true
+      CONCRETE_ACTIONS_TOKEN:
+        required: true
 
 env:
   CARGO_TERM_COLOR: always
 jobs:
-  check-concrete-optimizer:
+  tests:
     strategy:
       matrix:
         os: [ubuntu-20.04, macos-11]
@@ -19,37 +22,46 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: "Setup"
-        uses: ./.github/workflows/setup
+        uses: ./.github/workflows/optimizer_setup
         with:
-          ssh_private_key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }}
+          ssh_private_key: ${{ secrets.CONCRETE_CI_SSH_PRIVATE }}
 
       - name: Formatting
-        run: cargo fmt --check
+        run: |
+          cd compilers/concrete-optimizer
+          cargo fmt --check
 
       - name: Build
-        run: cargo build --release --all-targets
+        run: |
+          cd compilers/concrete-optimizer
+          cargo build --release --all-targets
 
       - name: Lint
-        run: cargo clippy --release --all-targets
+        run: |
+          cd compilers/concrete-optimizer
+          cargo clippy --release --all-targets
 
       - name: Tests
         if: matrix.os == 'ubuntu-20.04'
         run: |
+          cd compilers/concrete-optimizer
           cargo test --release --no-fail-fast --all-targets
           make -C concrete-optimizer-cpp test-ci
       
-  benchmark-concrete-optimizer:
+  benchmarks:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3
 
       - name: "Setup"
-        uses: ./.github/workflows/setup
+        uses: ./.github/workflows/optimizer_setup
         with:
-          ssh_private_key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }}
+          ssh_private_key: ${{ secrets.CONCRETE_CI_SSH_PRIVATE }}
 
       - name: Run benchmark
-        run: cargo bench -p v0-parameters -- --output-format bencher | tee bench_result.txt
+        run: |
+          cd compilers/concrete-optimizer
+          cargo bench -p v0-parameters -- --output-format bencher | tee bench_result.txt
 
       - name: Download PR base benchmark data
         if: ${{ github.event_name == 'pull_request' }}
@@ -69,7 +81,7 @@ jobs:
         uses: benchmark-action/github-action-benchmark@v1
         with:
           tool: 'cargo'
-          output-file-path: bench_result.txt
+          output-file-path: ./compilers/concrete-optimizer/bench_result.txt
           external-data-json-path: ./benchmark/benchmark-data.json
           # GitHub API token to make a commit comment
           github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -79,12 +91,10 @@ jobs:
       - name: Upload benchmark data
         uses: actions/upload-artifact@v3
         with:
-          path: ./benchmark
+          path: ./compilers/concrete-optimizer/benchmark
           name: ${{ runner.os }}-benchmark
-          
-          
 
-  nightly-check-concrete-optimizer:
+  nightly-check:
     runs-on: ubuntu-20.04
     env:
       RUSTFLAGS: -D warnings
@@ -100,10 +110,14 @@ jobs:
       - name: Set ssh keys
         uses: webfactory/ssh-agent@v0.6.0
         with:
-          ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }}
+          ssh-private-key: ${{ secrets.CONCRETE_CI_SSH_PRIVATE }}
       - name: Formatting
-        run: cargo fmt --check
+        run: |
+          cd compilers/concrete-optimizer
+          cargo fmt --check
 
       - name: Lint
-        run: cargo clippy --release --all-targets
+        run: |
+          cd compilers/concrete-optimizer
+          cargo clippy --release --all-targets
 

.github/workflows/start_aws_builds.yml

@@ -3,7 +3,7 @@ name: Start all AWS builds
 
 on:
   push:
-    branch:
+    branches:
       - 'main'
   workflow_dispatch: