From 79618159ff3cb332dfa6dca4c3db1203c89973b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexandre=20P=C3=A9r=C3=A9?= Date: Thu, 15 May 2025 11:34:52 +0200 Subject: [PATCH] chore(ci): add ci workflow for concrete-rust tests --- .../workflows/concrete_rust_test_macos.yml | 90 +++++++++++++ .../workflows/concrete_rust_tests_linux.yml | 120 ++++++++++++++++++ .../lib/Bindings/Python/requirements_dev.txt | 1 + frontends/concrete-rust/Makefile | 27 +++- .../concrete-rust/concrete-keygen/Cargo.toml | 2 +- .../concrete-rust/concrete-macro/Cargo.toml | 4 +- frontends/concrete-rust/concrete/Cargo.toml | 4 +- frontends/concrete-rust/concrete/build.rs | 46 ++++++- 8 files changed, 277 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/concrete_rust_test_macos.yml create mode 100644 .github/workflows/concrete_rust_tests_linux.yml diff --git a/.github/workflows/concrete_rust_test_macos.yml b/.github/workflows/concrete_rust_test_macos.yml new file mode 100644 index 000000000..17e7a8030 --- /dev/null +++ b/.github/workflows/concrete_rust_test_macos.yml @@ -0,0 +1,90 @@ +name: concrete-rust tests macos +on: + workflow_dispatch: + pull_request: + paths: + - .github/workflows/concrete_rust_test_macos.yml + - frontends/concrete-rust/** + push: + branches: + - 'main' + - 'release/*' + +concurrency: + group: concrete_rust_tests_macos_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +env: + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + +jobs: + concrete-rust-test: + strategy: + fail-fast: false + matrix: + machine: ["aws-mac1-metal", "aws-mac2-metal"] + runs-on: ${{ matrix.machine }} + outputs: + slack_message: ${{ steps.prepare_slack_notif.outputs.slack_message }} + slack_color: ${{ steps.prepare_slack_notif.outputs.slack_color }} + env: + concrete-rust-dir: ${{ github.workspace }}/frontends/concrete-rust + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: recursive + fetch-depth: 0 + persist-credentials: 'false' + - name: Install build dependencies + run: | + brew install ninja ccache + - name: Setup rust toolchain for concrete-cpu + uses: ./.github/actions/setup_rust_toolchain_for_concrete_cpu + - name: Install rust + uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1.0.7 + with: + toolchain: 1.84.0 + override: true + - name: Build rust bindings + run: | + set -e + cd ${{ env.concrete-compiler-dir }} + rm -rf .venv + python3 -m venv .venv + source .venv/bin/activate + pip install -r lib/Bindings/Python/requirements_dev.txt + cargo install cxxbridge-cmd + ccache -z + make Python3_EXECUTABLE=$(which python) concrete-rust + ccache -s + - name: Run tests + run: | + set -e + cd ${{ env.concrete-rust-dir }} + make clean + make clear_locks + make test + - name: Prepare Slack Notification + id: prepare_slack_notif + if: ${{ failure() }} + continue-on-error: true + run: | + echo "slack_message=concrete-rust-test (${{matrix.machine}}) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" >> "$GITHUB_OUTPUT" + echo "slack_color=${{ job.status }}" >> "$GITHUB_OUTPUT" + + slack-notif-macos: + needs: ["concrete-rust-test"] + runs-on: "ubuntu-latest" + if: always() + steps: + - name: Slack Notification + # we want to check that prepare_slack_notif was run + if: ${{ needs.concrete-rust-test.outputs.slack_color != '' }} + uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 + env: + SLACK_COLOR: ${{ needs.concrete-rust-test.outputs.slack_color }} + SLACK_MESSAGE: ${{ needs.concrete-rust-test.outputs.slack_message }} diff --git a/.github/workflows/concrete_rust_tests_linux.yml b/.github/workflows/concrete_rust_tests_linux.yml new file mode 100644 index 000000000..791103803 --- /dev/null +++ b/.github/workflows/concrete_rust_tests_linux.yml @@ -0,0 +1,120 @@ +name: concrete-rust tests linux-cpu + +on: + workflow_dispatch: + pull_request: + paths: + - .github/workflows/concrete_rust_tests_linux.yml + - frontends/concrete-rust/** + push: + branches: + - 'main' + - 'release/*' + +env: + DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete/compiler-ci + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + +concurrency: + group: concrete_rust_tests_linux_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + setup-instance: + runs-on: ubuntu-latest + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} + steps: + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@f26b8d611b2e695158fb0a6980834f0612f65ef8 # v1.4.0 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: cpu-test + + build-and-test-rust: + needs: setup-instance + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + steps: + - name: Checkout concrete + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: recursive + fetch-depth: 0 + + - name: Create concrete build directory + run: mkdir build + + - name: Build rust bindings + uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 + with: + registry: ghcr.io + image: ${{ env.DOCKER_IMAGE_TEST }} + username: ${{ secrets.GHCR_LOGIN }} + password: ${{ secrets.GHCR_PASSWORD }} + options: >- + -v ${{ github.workspace }}:/concrete + -v ${{ github.workspace }}/build:/build + shell: bash + run: | + set -e + ccache -z + ccache -p + cd /concrete/compilers/concrete-compiler/compiler + make BUILD_DIR=/build DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$(which python) concrete-rust + echo "Debug: ccache statistics (after the build):" + ccache -s + + - name: Run tests + uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 + with: + registry: ghcr.io + image: ${{ env.DOCKER_IMAGE_TEST }} + username: ${{ secrets.GHCR_LOGIN }} + password: ${{ secrets.GHCR_PASSWORD }} + options: >- + -v ${{ github.workspace }}:/concrete + -v ${{ github.workspace }}/build:/build + shell: bash + run: | + set -e + cd /concrete/frontends/concrete-rust + RUSTFLAGS="-Clink-args=-Wl,/usr/lib64/libstdc++.so.6" make COMPILER_BUILD_DIRECTORY=/build test + + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "build-and-test-rust finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + teardown-instance: + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + needs: [ setup-instance, build-and-test-rust] + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@f26b8d611b2e695158fb0a6980834f0612f65ef8 # v1.4.0 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Python/requirements_dev.txt b/compilers/concrete-compiler/compiler/lib/Bindings/Python/requirements_dev.txt index ba9c7ea11..ea96c42f2 100644 --- a/compilers/concrete-compiler/compiler/lib/Bindings/Python/requirements_dev.txt +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Python/requirements_dev.txt @@ -3,4 +3,5 @@ pylint==3.3.3 mypy==1.11.2 numpy>=1.23,<2.0 jsonpickle>=3.0.3 +pybind11>=2.13 pybind11-stubgen>=2.5 diff --git a/frontends/concrete-rust/Makefile b/frontends/concrete-rust/Makefile index ce4661212..2d24eee78 100644 --- a/frontends/concrete-rust/Makefile +++ b/frontends/concrete-rust/Makefile @@ -1,10 +1,12 @@ CARGO_SUBDIRS := concrete concrete-macro test concrete-keygen +COMPILER_BUILD_DIRECTORY := ../../../compilers/concrete-compiler/compiler/build + +.PHONY: test clean run pcc -.PHONY: test clean test: @for dir in $(CARGO_SUBDIRS); do \ - echo "Running cargo test in $$dir..."; \ - COMPILER_BUILD_DIRECTORY=../../../compilers/concrete-compiler/compiler/build cargo test --all-features --manifest-path $$dir/Cargo.toml -- --nocapture || exit 1; \ + echo "Running cargo test in $$dir..."; \ + COMPILER_BUILD_DIRECTORY=$(COMPILER_BUILD_DIRECTORY) cargo test --all-features --manifest-path $$dir/Cargo.toml -- --nocapture || exit 1; \ done clean: @@ -13,9 +15,21 @@ clean: rm -rf $$dir/target || exit 1; \ done +clear_locks : + @for dir in $(CARGO_SUBDIRS); do \ + echo "Cleaning lock in $$dir..."; \ + rm -rf $$dir/Cargo.lock || exit 1; \ + done + run: cd test && \ - COMPILER_BUILD_DIRECTORY=../../../compilers/concrete-compiler/compiler/build cargo run + COMPILER_BUILD_DIRECTORY=$(COMPILER_BUILD_DIRECTORY) cargo run + +pcc: + @for dir in $(CARGO_SUBDIRS); do \ + echo "Running pcc in $$dir..."; \ + COMPILER_BUILD_DIRECTORY=$(COMPILER_BUILD_DIRECTORY) cargo check --all-features --manifest-path $$dir/Cargo.toml || exit 1; \ + done format: @for dir in $(CARGO_SUBDIRS); do \ @@ -24,5 +38,6 @@ format: done regen_test_zips: - python test/python/test.py - python test/python/test_tfhers.py + cd test/src && \ + python ../python/test.py && \ + python ../python/test_tfhers.py diff --git a/frontends/concrete-rust/concrete-keygen/Cargo.toml b/frontends/concrete-rust/concrete-keygen/Cargo.toml index 4cf0ea3f2..43a136e8b 100644 --- a/frontends/concrete-rust/concrete-keygen/Cargo.toml +++ b/frontends/concrete-rust/concrete-keygen/Cargo.toml @@ -37,7 +37,7 @@ capnpc = "0.20.1" capnp = "0.20.3" clap = { version = "4.5.16", features = ["derive"], optional = true } -zip = { version = "2.6.1", optional = true } +zip = { version = "3.0", optional = true } wasm-bindgen = { version = "0.2", features = ["serde-serialize"], optional = true } wasm-bindgen-futures = { version = "0.4.50", optional = true } diff --git a/frontends/concrete-rust/concrete-macro/Cargo.toml b/frontends/concrete-rust/concrete-macro/Cargo.toml index bddecd27e..c3c622105 100644 --- a/frontends/concrete-rust/concrete-macro/Cargo.toml +++ b/frontends/concrete-rust/concrete-macro/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "concrete-macro" -version = "2.10.1" +version = "0.0.0" edition = "2021" readme = "../../../../../../../README.md" keywords = ["fully", "homomorphic", "encryption", "fhe", "cryptography"] @@ -22,4 +22,4 @@ zip = "2.2.2" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" itertools = "0.14.0" -concrete = { version = "2.10.1", path = "../concrete", features = ["compiler"]} +concrete = { version = "=0.0.0", path = "../concrete", features = ["compiler"]} diff --git a/frontends/concrete-rust/concrete/Cargo.toml b/frontends/concrete-rust/concrete/Cargo.toml index 02bcabf8c..868c045c9 100644 --- a/frontends/concrete-rust/concrete/Cargo.toml +++ b/frontends/concrete-rust/concrete/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "concrete" -version = "2.10.1" +version = "0.0.0" edition = "2021" links = "concrete" readme = "../../../../../../../README.md" @@ -21,7 +21,7 @@ proc-macro2 = {version = "1.0", optional = true} tfhe = {version = "1.1", features = ["integer"], optional = true} [build-dependencies] -zip = "2.6" +zip = "2.4" curl = "0.4.47" cxx-build = "1.0" diff --git a/frontends/concrete-rust/concrete/build.rs b/frontends/concrete-rust/concrete/build.rs index c9c0ec6df..1abde1d4b 100644 --- a/frontends/concrete-rust/concrete/build.rs +++ b/frontends/concrete-rust/concrete/build.rs @@ -72,13 +72,27 @@ fn install_artifacts(out_dir: &Path) { } fn copy_local_artifacts(out_dir: &Path, mut lib_dir: PathBuf) { - println!("cargo:warning=\"COMPILER_BUILD_DIRECTORY detected: building from local artifacts\""); + println!( + "cargo:warning=\"COMPILER_BUILD_DIRECTORY detected: building from local artifacts in {}\"", + lib_dir.display() + ); lib_dir.push("lib"); - lib_dir = lib_dir.canonicalize().unwrap(); + lib_dir = lib_dir + .canonicalize() + .map_err(|e| format!("Failed to canonicalize {}: {e}", lib_dir.display())) + .unwrap(); do_with_lock(&out_dir.join(INSTALL_LOCK), || { for art in ARTIFACTS { println!("cargo::rerun-if-changed={}", &lib_dir.join(art).display()); - std::fs::copy(&lib_dir.join(art), &out_dir.join(art)).unwrap(); + std::fs::copy(&lib_dir.join(art), &out_dir.join(art)) + .map_err(|e| { + format!( + "Failed to copy {} to {}: {e}", + lib_dir.join(art).display(), + out_dir.join(art).display() + ) + }) + .unwrap(); } }); } @@ -115,10 +129,18 @@ fn fetch_artifacts(out_dir: &Path) { .write(true) .create(true) .open(out_dir.join(path.file_name().unwrap())) + .map_err(|e| { + format!( + "Failed to open file {}: {e}", + out_dir.join(path.file_name().unwrap()).display() + ) + }) .unwrap(); std::io::copy(&mut content, &mut file).unwrap(); } - std::fs::File::create(out_dir.join(INSTALLED_FILE)).unwrap(); + std::fs::File::create(out_dir.join(INSTALLED_FILE)) + .map_err(|e| format!("Failed to create INSTALLED_FILE: {e}")) + .unwrap(); }); } @@ -132,13 +154,25 @@ fn symlink_outdir(concrete_dir: &Path, out_dir: &Path) { "Failed to delete original out_dir {}", out_dir.display() ); - std::os::unix::fs::symlink(concrete_dir, &out_dir).unwrap(); + std::os::unix::fs::symlink(concrete_dir, &out_dir) + .map_err(|e| { + format!( + "Failed to symlink {} to {}: {e}", + concrete_dir.display(), + out_dir.display() + ) + }) + .unwrap(); } } fn main() { let out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap()); - let target_dir = out_dir.join("../../../..").canonicalize().unwrap(); + let target_dir = out_dir + .join("../../../..") + .canonicalize() + .map_err(|e| format!("Failed to get target dir: {e}")) + .unwrap(); let concrete_dir = target_dir.join("concrete"); symlink_outdir(&concrete_dir, &out_dir);