chore(ci): add ci workflow for concrete-rust tests

2026-01-08 04:23:57 -05:00 · 2025-05-15 11:34:52 +02:00
parent 6948d91cc6
commit 79618159ff
8 changed files with 277 additions and 17 deletions
--- a/.github/workflows/concrete_rust_test_macos.yml
+++ b/.github/workflows/concrete_rust_test_macos.yml
@@ -0,0 +1,90 @@
+name: concrete-rust tests macos
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - .github/workflows/concrete_rust_test_macos.yml
+      - frontends/concrete-rust/**
+  push:
+    branches:
+      - 'main'
+      - 'release/*'
+
+concurrency:
+  group: concrete_rust_tests_macos_${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+env:
+  ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+  SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
+  SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
+  SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+
+jobs:
+  concrete-rust-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        machine: ["aws-mac1-metal", "aws-mac2-metal"]
+    runs-on: ${{ matrix.machine }}
+    outputs:
+      slack_message: ${{ steps.prepare_slack_notif.outputs.slack_message }}
+      slack_color: ${{ steps.prepare_slack_notif.outputs.slack_color }}
+    env:
+      concrete-rust-dir: ${{ github.workspace }}/frontends/concrete-rust
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: recursive
+          fetch-depth: 0
+          persist-credentials: 'false'
+      - name: Install build dependencies
+        run: |
+          brew install ninja ccache
+      - name: Setup rust toolchain for concrete-cpu
+        uses: ./.github/actions/setup_rust_toolchain_for_concrete_cpu
+      - name: Install rust
+        uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1.0.7
+        with:
+          toolchain: 1.84.0
+          override: true
+      - name: Build rust bindings
+        run: |
+          set -e
+          cd ${{ env.concrete-compiler-dir }}
+          rm -rf .venv
+          python3 -m venv .venv
+          source .venv/bin/activate
+          pip install -r lib/Bindings/Python/requirements_dev.txt
+          cargo install cxxbridge-cmd
+          ccache -z
+          make Python3_EXECUTABLE=$(which python) concrete-rust
+          ccache -s
+      - name: Run tests
+        run: |
+          set -e
+          cd ${{ env.concrete-rust-dir }}
+          make clean
+          make clear_locks
+          make test
+      - name: Prepare Slack Notification
+        id: prepare_slack_notif
+        if: ${{ failure() }}
+        continue-on-error: true
+        run: |
+          echo "slack_message=concrete-rust-test (${{matrix.machine}}) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" >> "$GITHUB_OUTPUT"
+          echo "slack_color=${{ job.status }}" >> "$GITHUB_OUTPUT"
+
+  slack-notif-macos:
+    needs: ["concrete-rust-test"]
+    runs-on: "ubuntu-latest"
+    if: always()
+    steps:
+      - name: Slack Notification
+        # we want to check that prepare_slack_notif was run
+        if: ${{ needs.concrete-rust-test.outputs.slack_color != '' }}
+        uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990
+        env:
+          SLACK_COLOR: ${{ needs.concrete-rust-test.outputs.slack_color }}
+          SLACK_MESSAGE: ${{ needs.concrete-rust-test.outputs.slack_message }}
--- a/.github/workflows/concrete_rust_tests_linux.yml
+++ b/.github/workflows/concrete_rust_tests_linux.yml
@@ -0,0 +1,120 @@
+name: concrete-rust tests linux-cpu
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - .github/workflows/concrete_rust_tests_linux.yml
+      - frontends/concrete-rust/**
+  push:
+    branches:
+      - 'main'
+      - 'release/*'
+
+env:
+  DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete/compiler-ci
+  ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+  SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
+  SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
+  SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+
+concurrency:
+  group: concrete_rust_tests_linux_${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+jobs:
+  setup-instance:
+    runs-on: ubuntu-latest
+    outputs:
+      runner-name: ${{ steps.start-instance.outputs.label }}
+    steps:
+      - name: Start instance
+        id: start-instance
+        uses: zama-ai/slab-github-runner@f26b8d611b2e695158fb0a6980834f0612f65ef8 # v1.4.0
+        with:
+          mode: start
+          github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
+          slab-url: ${{ secrets.SLAB_BASE_URL }}
+          job-secret: ${{ secrets.JOB_SECRET }}
+          backend: aws
+          profile: cpu-test
+
+  build-and-test-rust:
+    needs: setup-instance
+    runs-on: ${{ needs.setup-instance.outputs.runner-name }}
+    steps:
+      - name: Checkout concrete
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: recursive
+          fetch-depth: 0
+
+      - name: Create concrete build directory
+        run: mkdir build
+
+      - name: Build rust bindings
+        uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3
+        with:
+          registry: ghcr.io
+          image: ${{ env.DOCKER_IMAGE_TEST }}
+          username: ${{ secrets.GHCR_LOGIN }}
+          password: ${{ secrets.GHCR_PASSWORD }}
+          options: >-
+            -v ${{ github.workspace }}:/concrete
+            -v ${{ github.workspace }}/build:/build
+          shell: bash
+          run: |
+            set -e
+            ccache -z
+            ccache -p
+            cd /concrete/compilers/concrete-compiler/compiler
+            make BUILD_DIR=/build DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$(which python) concrete-rust
+            echo "Debug: ccache statistics (after the build):"
+            ccache -s
+
+      - name: Run tests
+        uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3
+        with:
+          registry: ghcr.io
+          image: ${{ env.DOCKER_IMAGE_TEST }}
+          username: ${{ secrets.GHCR_LOGIN }}
+          password: ${{ secrets.GHCR_PASSWORD }}
+          options: >-
+            -v ${{ github.workspace }}:/concrete
+            -v ${{ github.workspace }}/build:/build
+          shell: bash
+          run: |
+            set -e
+            cd /concrete/frontends/concrete-rust
+            RUSTFLAGS="-Clink-args=-Wl,/usr/lib64/libstdc++.so.6" make COMPILER_BUILD_DIRECTORY=/build test
+
+      - name: Slack Notification
+        if: ${{ failure() && github.ref == 'refs/heads/main' }}
+        continue-on-error: true
+        uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990
+        env:
+          SLACK_COLOR: ${{ job.status }}
+          SLACK_MESSAGE: "build-and-test-rust finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
+
+  teardown-instance:
+    if: ${{ always() && needs.setup-instance.result != 'skipped' }}
+    needs: [ setup-instance, build-and-test-rust]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Stop instance
+        id: stop-instance
+        uses: zama-ai/slab-github-runner@f26b8d611b2e695158fb0a6980834f0612f65ef8 # v1.4.0
+        with:
+          mode: stop
+          github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
+          slab-url: ${{ secrets.SLAB_BASE_URL }}
+          job-secret: ${{ secrets.JOB_SECRET }}
+          label: ${{ needs.setup-instance.outputs.runner-name }}
+
+      - name: Slack Notification
+        if: ${{ failure() }}
+        continue-on-error: true
+        uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990
+        env:
+          SLACK_COLOR: ${{ job.status }}
+          SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
--- a/compilers/concrete-compiler/compiler/lib/Bindings/Python/requirements_dev.txt
+++ b/compilers/concrete-compiler/compiler/lib/Bindings/Python/requirements_dev.txt
@@ -3,4 +3,5 @@ pylint==3.3.3
 mypy==1.11.2
 numpy>=1.23,<2.0
 jsonpickle>=3.0.3
+pybind11>=2.13
 pybind11-stubgen>=2.5
--- a/frontends/concrete-rust/Makefile
+++ b/frontends/concrete-rust/Makefile
@@ -1,10 +1,12 @@
 CARGO_SUBDIRS := concrete concrete-macro test concrete-keygen
+COMPILER_BUILD_DIRECTORY := ../../../compilers/concrete-compiler/compiler/build
+
+.PHONY: test clean run pcc

-.PHONY: test clean
 test:
 	@for dir in $(CARGO_SUBDIRS); do \
-	   echo "Running cargo test in $$dir..."; \
-	   COMPILER_BUILD_DIRECTORY=../../../compilers/concrete-compiler/compiler/build cargo test --all-features --manifest-path $$dir/Cargo.toml -- --nocapture || exit 1; \
+		echo "Running cargo test in $$dir..."; \
+		COMPILER_BUILD_DIRECTORY=$(COMPILER_BUILD_DIRECTORY) cargo test --all-features --manifest-path $$dir/Cargo.toml -- --nocapture || exit 1; \
 	done

 clean:
@@ -13,9 +15,21 @@ clean:
 		rm -rf $$dir/target || exit 1; \
 	done

+clear_locks :
+	@for dir in $(CARGO_SUBDIRS); do \
+		echo "Cleaning lock in $$dir..."; \
+		rm -rf $$dir/Cargo.lock || exit 1; \
+	done
+
 run:
 	cd test && \
- 	COMPILER_BUILD_DIRECTORY=../../../compilers/concrete-compiler/compiler/build cargo run
+		COMPILER_BUILD_DIRECTORY=$(COMPILER_BUILD_DIRECTORY) cargo run
+
+pcc:
+	@for dir in $(CARGO_SUBDIRS); do \
+		echo "Running pcc in $$dir..."; \
+		COMPILER_BUILD_DIRECTORY=$(COMPILER_BUILD_DIRECTORY) cargo check --all-features --manifest-path $$dir/Cargo.toml || exit 1; \
+	done

 format:
 	@for dir in $(CARGO_SUBDIRS); do \
@@ -24,5 +38,6 @@ format:
 	done

 regen_test_zips:
-	python test/python/test.py
-	python test/python/test_tfhers.py
+	cd test/src && \
+	python ../python/test.py && \
+	python ../python/test_tfhers.py
--- a/frontends/concrete-rust/concrete-keygen/Cargo.toml
+++ b/frontends/concrete-rust/concrete-keygen/Cargo.toml
@@ -37,7 +37,7 @@ capnpc = "0.20.1"
 capnp = "0.20.3"

 clap = { version = "4.5.16", features = ["derive"], optional = true }
-zip = { version = "2.6.1", optional = true }
+zip = { version = "3.0", optional = true }

 wasm-bindgen = { version = "0.2", features = ["serde-serialize"], optional = true }
 wasm-bindgen-futures = { version = "0.4.50", optional = true }
--- a/frontends/concrete-rust/concrete-macro/Cargo.toml
+++ b/frontends/concrete-rust/concrete-macro/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "concrete-macro"
-version = "2.10.1"
+version = "0.0.0"
 edition = "2021"
 readme = "../../../../../../../README.md"
 keywords = ["fully", "homomorphic", "encryption", "fhe", "cryptography"]
@@ -22,4 +22,4 @@ zip = "2.2.2"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 itertools = "0.14.0"
-concrete = {  version = "2.10.1", path = "../concrete", features = ["compiler"]}
+concrete = {  version = "=0.0.0", path = "../concrete", features = ["compiler"]}
--- a/frontends/concrete-rust/concrete/Cargo.toml
+++ b/frontends/concrete-rust/concrete/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "concrete"
-version = "2.10.1"
+version = "0.0.0"
 edition = "2021"
 links = "concrete"
 readme = "../../../../../../../README.md"
@@ -21,7 +21,7 @@ proc-macro2 = {version = "1.0", optional = true}
 tfhe = {version = "1.1", features = ["integer"], optional = true}

 [build-dependencies]
-zip = "2.6"
+zip = "2.4"
 curl = "0.4.47"
 cxx-build = "1.0"

--- a/frontends/concrete-rust/concrete/build.rs
+++ b/frontends/concrete-rust/concrete/build.rs
@@ -72,13 +72,27 @@ fn install_artifacts(out_dir: &Path) {
 }

 fn copy_local_artifacts(out_dir: &Path, mut lib_dir: PathBuf) {
-    println!("cargo:warning=\"COMPILER_BUILD_DIRECTORY detected: building from local artifacts\"");
+    println!(
+        "cargo:warning=\"COMPILER_BUILD_DIRECTORY detected: building from local artifacts in {}\"",
+        lib_dir.display()
+    );
    lib_dir.push("lib");
-    lib_dir = lib_dir.canonicalize().unwrap();
+    lib_dir = lib_dir
+        .canonicalize()
+        .map_err(|e| format!("Failed to canonicalize {}: {e}", lib_dir.display()))
+        .unwrap();
    do_with_lock(&out_dir.join(INSTALL_LOCK), || {
        for art in ARTIFACTS {
            println!("cargo::rerun-if-changed={}", &lib_dir.join(art).display());
-            std::fs::copy(&lib_dir.join(art), &out_dir.join(art)).unwrap();
+            std::fs::copy(&lib_dir.join(art), &out_dir.join(art))
+                .map_err(|e| {
+                    format!(
+                        "Failed to copy {} to {}: {e}",
+                        lib_dir.join(art).display(),
+                        out_dir.join(art).display()
+                    )
+                })
+                .unwrap();
        }
    });
 }
@@ -115,10 +129,18 @@ fn fetch_artifacts(out_dir: &Path) {
                .write(true)
                .create(true)
                .open(out_dir.join(path.file_name().unwrap()))
+                .map_err(|e| {
+                    format!(
+                        "Failed to open file {}: {e}",
+                        out_dir.join(path.file_name().unwrap()).display()
+                    )
+                })
                .unwrap();
            std::io::copy(&mut content, &mut file).unwrap();
        }
-        std::fs::File::create(out_dir.join(INSTALLED_FILE)).unwrap();
+        std::fs::File::create(out_dir.join(INSTALLED_FILE))
+            .map_err(|e| format!("Failed to create INSTALLED_FILE: {e}"))
+            .unwrap();
    });
 }

@@ -132,13 +154,25 @@ fn symlink_outdir(concrete_dir: &Path, out_dir: &Path) {
            "Failed to delete original out_dir {}",
            out_dir.display()
        );
-        std::os::unix::fs::symlink(concrete_dir, &out_dir).unwrap();
+        std::os::unix::fs::symlink(concrete_dir, &out_dir)
+            .map_err(|e| {
+                format!(
+                    "Failed to symlink {} to {}: {e}",
+                    concrete_dir.display(),
+                    out_dir.display()
+                )
+            })
+            .unwrap();
    }
 }

 fn main() {
    let out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap());
-    let target_dir = out_dir.join("../../../..").canonicalize().unwrap();
+    let target_dir = out_dir
+        .join("../../../..")
+        .canonicalize()
+        .map_err(|e| format!("Failed to get target dir: {e}"))
+        .unwrap();
    let concrete_dir = target_dir.join("concrete");

    symlink_outdir(&concrete_dir, &out_dir);