From 2fd4d772d0a2466c91edc3b20b77b408136610c5 Mon Sep 17 00:00:00 2001 From: Quentin Bourgerie Date: Wed, 2 Apr 2025 11:12:01 +0200 Subject: [PATCH 01/17] chore(ci): Use zsh for building wheel in macos release workflow --- .github/workflows/concrete_python_release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/concrete_python_release.yml b/.github/workflows/concrete_python_release.yml index 62601ecd0..82e7cbac1 100644 --- a/.github/workflows/concrete_python_release.yml +++ b/.github/workflows/concrete_python_release.yml @@ -239,6 +239,7 @@ jobs: # add new version cat frontends/concrete-python/version.txt >> frontends/concrete-python/concrete/fhe/version.py - name: Build wheel + shell: zsh run: | CONCRETE_PYTHON=$(pwd)/frontends/concrete-python CONCRETE_COMPILER=$(pwd)/compilers/concrete-compiler/compiler From dab23c625cebb35be5e0d870046cb378d682faf4 Mon Sep 17 00:00:00 2001 From: Quentin Bourgerie Date: Wed, 2 Apr 2025 11:20:03 +0200 Subject: [PATCH 02/17] chore(ci): Revert use zsh for building wheel in macos release workflow This reverts commit 80f60818fe269d1c4f4d7a24c6ccefb3228750f2. --- .github/workflows/concrete_python_release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/concrete_python_release.yml b/.github/workflows/concrete_python_release.yml index 82e7cbac1..62601ecd0 100644 --- a/.github/workflows/concrete_python_release.yml +++ b/.github/workflows/concrete_python_release.yml @@ -239,7 +239,6 @@ jobs: # add new version cat frontends/concrete-python/version.txt >> frontends/concrete-python/concrete/fhe/version.py - name: Build wheel - shell: zsh run: | CONCRETE_PYTHON=$(pwd)/frontends/concrete-python CONCRETE_COMPILER=$(pwd)/compilers/concrete-compiler/compiler From f5d812e2d92fb9d282fd0221e1d418792171b5e2 Mon Sep 17 00:00:00 2001 From: Quentin Bourgerie Date: Wed, 2 Apr 2025 11:53:56 +0200 Subject: [PATCH 03/17] chore(ci): Hack for python3.8 in mac2 release --- .github/workflows/concrete_python_release.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/concrete_python_release.yml b/.github/workflows/concrete_python_release.yml index 62601ecd0..5e82113bb 100644 --- a/.github/workflows/concrete_python_release.yml +++ b/.github/workflows/concrete_python_release.yml @@ -240,6 +240,9 @@ jobs: cat frontends/concrete-python/version.txt >> frontends/concrete-python/concrete/fhe/version.py - name: Build wheel run: | + # Hack for mac2 host... + export PATH=$PATH:/Users/ec2-user/.cargo/bin:/Users/ec2-user/.pyenv/versions/3.8.20/bin + CONCRETE_PYTHON=$(pwd)/frontends/concrete-python CONCRETE_COMPILER=$(pwd)/compilers/concrete-compiler/compiler export COMPILER_BUILD_DIRECTORY=$CONCRETE_COMPILER/build From d843d85461973f3bec507aa146250fda9c69115b Mon Sep 17 00:00:00 2001 From: Quentin Bourgerie Date: Thu, 3 Apr 2025 11:43:31 +0200 Subject: [PATCH 04/17] chore(ci): Update twine in finalize release --- .github/workflows/concrete_python_finalize_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/concrete_python_finalize_release.yml b/.github/workflows/concrete_python_finalize_release.yml index a45268d0d..b9f8686db 100644 --- a/.github/workflows/concrete_python_finalize_release.yml +++ b/.github/workflows/concrete_python_finalize_release.yml @@ -26,7 +26,7 @@ jobs: ls -la ./wheels/ - name: Push wheels to public PyPI (public) run: | - pip install twine==4.0.2 + pip install twine==6.0.0 twine upload wheels/concrete_python-${{ inputs.version }}*.whl \ -u "${{ secrets.PUBLIC_PYPI_USER }}" \ -p "${{ secrets.PUBLIC_PYPI_PASSWORD }}" \ From 97dfae8f2d1615e3af70d6459c75b69bb6bb35b8 Mon Sep 17 00:00:00 2001 From: Quentin Bourgerie Date: Fri, 4 Apr 2025 15:55:45 +0200 Subject: [PATCH 05/17] chore(ci): Add binaries as release artifacts --- .github/workflows/concrete_python_release.yml | 77 ++++++++++++++++++- 1 file changed, 74 insertions(+), 3 deletions(-) diff --git a/.github/workflows/concrete_python_release.yml b/.github/workflows/concrete_python_release.yml index 5e82113bb..63ccbcdc4 100644 --- a/.github/workflows/concrete_python_release.yml +++ b/.github/workflows/concrete_python_release.yml @@ -136,13 +136,23 @@ jobs: make PYTHON=$PYTHON venv source .venv/bin/activate + ccache -z + cd /concrete/compilers/concrete-compiler/compiler - make BUILD_DIR=/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=${{ env.DATAFLOW_EXECUTION_ENABLED }} Python3_EXECUTABLE=$(which python) \ - CUDA_SUPPORT=${{ env.CUDA_SUPPORT }} TIMING_ENABLED=${{ env.TIMING_ENABLED }} CUDA_PATH=${{ env.CUDA_PATH }} python-bindings + make BUILD_DIR=/build \ + CCACHE=ON \ + DATAFLOW_EXECUTION_ENABLED=${{ env.DATAFLOW_EXECUTION_ENABLED }} \ + Python3_EXECUTABLE=$(which python) \ + CUDA_SUPPORT=${{ env.CUDA_SUPPORT }} \ + TIMING_ENABLED=${{ env.TIMING_ENABLED }} \ + CUDA_PATH=${{ env.CUDA_PATH }} \ + python-bindings echo "Debug: ccache statistics (after the build):" ccache -s + + cd /concrete/frontends/concrete-python export COMPILER_BUILD_DIRECTORY="/build" @@ -155,6 +165,38 @@ jobs: name: ${{ format('{0}-wheel-{1}-linux-x86', matrix.hw, matrix.python-version) }} path: frontends/concrete-python/dist/*manylinux*.whl retention-days: 3 + - name: Build concrete rust + # Note: That arbitrary if to not build in every python version + if: ${{ matrix.python-version == '3.10' }} + uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 + with: + registry: ghcr.io + image: ${{ env.DOCKER_IMAGE_TEST }} + username: ${{ secrets.GHCR_LOGIN }} + password: ${{ secrets.GHCR_PASSWORD }} + options: >- + -v ${{ github.workspace }}:/concrete + -v ${{ github.workspace }}/build:/build + shell: bash + run: | + cargo install cxxbridge-cmd + cd /concrete/compilers/concrete-compiler/compiler + make BUILD_DIR=/build \ + CCACHE=ON \ + DATAFLOW_EXECUTION_ENABLED=${{ env.DATAFLOW_EXECUTION_ENABLED }} \ + Python3_EXECUTABLE=$(which python) \ + CUDA_SUPPORT=${{ env.CUDA_SUPPORT }} \ + TIMING_ENABLED=${{ env.TIMING_ENABLED }} \ + CUDA_PATH=${{ env.CUDA_PATH }} \ + concrete-rust + - name: Upload binary libraries + # Note: That arbitrary if to not build in every python version + if: ${{ matrix.python-version == '3.10' }} + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: ${{ format('{0}-binaries-linux-x86_64', matrix.hw) }} + path: ${{ github.workspace }}/build/lib/libConcrete*.so + retention-days: 3 - name: Slack Notification if: ${{ failure() }} continue-on-error: true @@ -280,6 +322,28 @@ jobs: name: ${{ format('cpu-wheel-{0}-{1}', matrix.python-version, matrix.runs-on) }} path: frontends/concrete-python/dist/*macos*.whl retention-days: 3 + - name: Build concrete rust + # Note: That arbitrary if to not build in every python version + if: ${{ matrix.python-version == '3.10' }} + run: | + cargo install cxxbridge-cmd + cd compilers/concrete-compiler/compiler + make concrete-rust + - name: Upload binary libraries mac2 + # Note: That arbitrary if to not build in every python version + if: ${{ matrix.python-version == '3.10' && matrix.runs-on == 'aws-mac2-metal' }} + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: cpu-binaries-macosx_arm64 + path: compilers/concrete-compiler/compiler/build/lib/libConcrete*.dylib + retention-days: 3 + - name: Upload binary libraries mac1 + if: ${{ matrix.python-version == '3.10' && matrix.runs-on == 'aws-mac1-metal' }} + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: cpu-binaries-macosx_x86_64 + path: compilers/concrete-compiler/compiler/build/lib/libConcrete*.dylib + retention-days: 3 - name: Slack Notification if: ${{ failure() }} continue-on-error: true @@ -359,6 +423,9 @@ jobs: path: wheels pattern: '${{ matrix.hw }}-wheel-*' merge-multiple: true + - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + pattern: '${{ matrix.hw }}-binaries-*' - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: pattern: '${{ matrix.hw }}*.intoto.jsonl' @@ -370,10 +437,14 @@ jobs: HW=${{ matrix.hw }} export TAG echo "${TAG}" + # Create zip of binaries folders + for i in ./*binaries*; do + zip "${i}.zip" "${i}"/* + done gh release create --draft --repo ${{ github.repository }} \ --verify-tag "${TAG}" \ --title "${TAG} - ${HW^^}" \ - wheels/* ./*.intoto.jsonl/* + wheels/* ./*.intoto.jsonl/* *binaries*.zip env: GH_TOKEN: ${{ github.token }} - name: Upload wheels to S3 From c3f742d9cddd7e7dcc9b856aaa2077bbeb0a393a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexandre=20P=C3=A9r=C3=A9?= Date: Mon, 24 Feb 2025 15:32:49 +0100 Subject: [PATCH 06/17] feat(frontend): introduce concrete rust adapter --- .../workflows/concrete_compiler_benchmark.yml | 2 + .../workflows/concrete_compiler_test_cpu.yml | 4 +- .../workflows/concrete_compiler_test_gpu.yml | 1 - .../concrete_compiler_test_macos_cpu.yml | 1 + ci/slab.toml | 2 +- compilers/concrete-compiler/compiler/Makefile | 12 +- .../cmake/modules/FetchHpxLibrary.cmake | 17 +- .../concretelang/ClientLib/ClientLib.h | 2 +- .../concretelang/Common/Transformers.h | 9 + .../include/concretelang/Common/Values.h | 6 +- .../include/concretelang/Runtime/simulation.h | 9 - .../include/concretelang/Runtime/utils.h | 21 - .../include/concretelang/Runtime/wrappers.h | 6 +- .../concretelang/ServerLib/ServerLib.h | 11 +- .../concretelang/Support/CompilerEngine.h | 2 + .../concretelang/Support/V0Parameters.h | 4 +- .../lib/Bindings/Python/CMakeLists.txt | 12 +- .../lib/Bindings/Python/CompilerAPIModule.cpp | 31 + .../Python/concrete/compiler/__init__.py | 28 + .../lib/Bindings/Python/requirements_dev.txt | 2 + .../compiler/lib/Bindings/Rust/CMakeLists.txt | 21 +- .../compiler/lib/Bindings/Rust/Cargo.toml | 4 +- .../compiler/lib/Bindings/Rust/NOTES.md | 41 + .../Bindings/Rust/concrete-macro/Cargo.toml | 24 + .../lib/Bindings/Rust/concrete-macro/build.rs | 16 + .../Rust/concrete-macro/src/configuration.rs | 248 +++++ .../concrete-macro/src/fast_path_hasher.rs | 30 + .../Rust/concrete-macro/src/generation.rs | 232 +++++ .../Bindings/Rust/concrete-macro/src/lib.rs | 247 +++++ .../Bindings/Rust/concrete-macro/src/unzip.rs | 36 + .../lib/Bindings/Rust/concrete-sys/Cargo.toml | 13 - .../lib/Bindings/Rust/concrete-sys/build.rs | 33 - .../Bindings/Rust/concrete-sys/src/.gitignore | 2 - .../Bindings/Rust/concrete-sys/src/lib.cpp | 57 - .../lib/Bindings/Rust/concrete-sys/src/lib.h | 40 - .../lib/Bindings/Rust/concrete-sys/src/lib.rs | 64 -- .../{concrete-sys => concrete}/.gitignore | 0 .../lib/Bindings/Rust/concrete/Cargo.toml | 31 + .../lib/Bindings/Rust/concrete/build.rs | 158 +++ .../lib/Bindings/Rust/concrete/src/.gitignore | 1 + .../Rust/{concrete-sys => concrete}/src/cxx.h | 2 - .../lib/Bindings/Rust/concrete/src/ffi.h | 581 +++++++++++ .../lib/Bindings/Rust/concrete/src/ffi.rs | 940 +++++++++++++++++ .../lib/Bindings/Rust/concrete/src/lib.rs | 27 + .../Bindings/Rust/concrete/src/protocol.rs | 977 ++++++++++++++++++ .../lib/Bindings/Rust/rust-toolchain.toml | 2 + .../lib/Bindings/Rust/test/Cargo.toml | 8 + .../lib/Bindings/Rust/test/src/lib.rs | 36 + .../lib/Bindings/Rust/test/src/main.rs | 22 + .../lib/Bindings/Rust/test/src/test.zip | Bin 0 -> 1696 bytes .../compiler/lib/ClientLib/ClientLib.cpp | 3 +- .../compiler/lib/Common/CMakeLists.txt | 6 +- .../compiler/lib/Common/Transformers.cpp | 28 +- .../compiler/lib/Common/Values.cpp | 44 +- .../compiler/lib/Runtime/CMakeLists.txt | 21 +- .../compiler/lib/Runtime/GPUDFG.cpp | 1 + .../compiler/lib/Runtime/simulation.cpp | 7 - .../compiler/lib/Runtime/utils.cpp | 15 - .../compiler/lib/Runtime/wrappers.cpp | 4 +- .../compiler/lib/ServerLib/ServerLib.cpp | 46 +- .../compiler/lib/Support/Pipeline.cpp | 4 +- compilers/concrete-optimizer/Cargo.lock | 2 + .../concrete-optimizer-cpp/Cargo.toml | 2 + .../src/concrete-optimizer.rs | 44 + .../src/cpp/concrete-optimizer.cpp | 40 + .../src/cpp/concrete-optimizer.hpp | 8 + .../multi_parameters/optimize/restriction.rs | 7 + docker/Dockerfile.concrete-compiler-env | 4 + 68 files changed, 4007 insertions(+), 354 deletions(-) delete mode 100644 compilers/concrete-compiler/compiler/include/concretelang/Runtime/utils.h create mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/NOTES.md create mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-macro/Cargo.toml create mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-macro/build.rs create mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-macro/src/configuration.rs create mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-macro/src/fast_path_hasher.rs create mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-macro/src/generation.rs create mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-macro/src/lib.rs create mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-macro/src/unzip.rs delete mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/Cargo.toml delete mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/build.rs delete mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/src/.gitignore delete mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/src/lib.cpp delete mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/src/lib.h delete mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/src/lib.rs rename compilers/concrete-compiler/compiler/lib/Bindings/Rust/{concrete-sys => concrete}/.gitignore (100%) create mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/Cargo.toml create mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/build.rs create mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/src/.gitignore rename compilers/concrete-compiler/compiler/lib/Bindings/Rust/{concrete-sys => concrete}/src/cxx.h (99%) create mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/src/ffi.h create mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/src/ffi.rs create mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/src/lib.rs create mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/src/protocol.rs create mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/rust-toolchain.toml create mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/test/Cargo.toml create mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/test/src/lib.rs create mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/test/src/main.rs create mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/test/src/test.zip delete mode 100644 compilers/concrete-compiler/compiler/lib/Runtime/utils.cpp diff --git a/.github/workflows/concrete_compiler_benchmark.yml b/.github/workflows/concrete_compiler_benchmark.yml index 6fe5a9a77..94db33901 100644 --- a/.github/workflows/concrete_compiler_benchmark.yml +++ b/.github/workflows/concrete_compiler_benchmark.yml @@ -73,6 +73,8 @@ jobs: set -e git config --global --add safe.directory '*' cd compilers/concrete-compiler/compiler + sudo apt install -y cmake + cmake --version make BINDINGS_PYTHON_ENABLED=OFF build-benchmarks - name: Run compiler benchmarks run: | diff --git a/.github/workflows/concrete_compiler_test_cpu.yml b/.github/workflows/concrete_compiler_test_cpu.yml index 3231b3373..5c7c31d3c 100644 --- a/.github/workflows/concrete_compiler_test_cpu.yml +++ b/.github/workflows/concrete_compiler_test_cpu.yml @@ -119,8 +119,6 @@ jobs: rm -rf /build/* ccache -z ccache -p - cargo install cxxbridge-cmd - dnf -y install libatomic make DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build all - name: Ccache stats if: always() @@ -176,7 +174,7 @@ jobs: set -e cd /concrete/compilers/concrete-compiler/compiler mkdir -p /tmp/concrete_compiler/gpu_tests/ - pip install pytest + pip install pytest jsonpickle sed "s/pytest/python -m pytest/g" -i Makefile make MINIMAL_TESTS=${{ env.MINIMAL_TESTS }} DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build run-tests - name: Slack Notification diff --git a/.github/workflows/concrete_compiler_test_gpu.yml b/.github/workflows/concrete_compiler_test_gpu.yml index 17096c533..64b0e7003 100644 --- a/.github/workflows/concrete_compiler_test_gpu.yml +++ b/.github/workflows/concrete_compiler_test_gpu.yml @@ -74,7 +74,6 @@ jobs: rm -rf /build/* ccache -z ccache -p - cargo install cxxbridge-cmd mkdir -p /tmp/concrete_compiler/gpu_tests/ make BINDINGS_PYTHON_ENABLED=OFF Python3_EXECUTABLE=$PYTHON_EXEC CUDA_SUPPORT=ON CUDA_PATH=${{ env.CUDA_PATH }} run-end-to-end-tests-gpu - name: Ccache stats diff --git a/.github/workflows/concrete_compiler_test_macos_cpu.yml b/.github/workflows/concrete_compiler_test_macos_cpu.yml index 7d1cf928c..0b9e18efb 100644 --- a/.github/workflows/concrete_compiler_test_macos_cpu.yml +++ b/.github/workflows/concrete_compiler_test_macos_cpu.yml @@ -61,6 +61,7 @@ jobs: echo "Debug: ccache statistics (prior to the build):" ccache -s cargo install cxxbridge-cmd + rm -rf build/* make Python3_EXECUTABLE="${PYTHON_EXEC}" all echo "Debug: ccache statistics (after the build):" ccache -s diff --git a/ci/slab.toml b/ci/slab.toml index aa62a7d07..41985791a 100644 --- a/ci/slab.toml +++ b/ci/slab.toml @@ -12,7 +12,7 @@ instance_type = "hpc7a.96xlarge" [backend.aws.gpu-test] region = "us-east-1" image_id = "ami-0257c6ad39f902b5e" -instance_type = "p3.2xlarge" +instance_type = "p3.8xlarge" subnet_id = "subnet-8123c9e7" security_group= ["sg-017afab1f328af917", ] diff --git a/compilers/concrete-compiler/compiler/Makefile b/compilers/concrete-compiler/compiler/Makefile index 19e30505a..417508fdd 100644 --- a/compilers/concrete-compiler/compiler/Makefile +++ b/compilers/concrete-compiler/compiler/Makefile @@ -113,7 +113,7 @@ ifeq ($(BUILD_TYPE),Debug) LIBOMP_LINK_TO_LIBSTDCXX_OPT=-DLIBOMP_USE_STDCPPLIB=ON endif -all: concretecompiler runtime python-bindings build-tests build-benchmarks doc concrete-sys +all: concretecompiler runtime python-bindings build-tests build-benchmarks doc concrete-rust # COMPILER ##################################################### @@ -130,7 +130,7 @@ $(BUILD_DIR)/configured.stamp: -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \ -DLLVM_ENABLE_ASSERTIONS=ON \ -DLLVM_ENABLE_RTTI=ON \ - -DLLVM_LINK_LLVM_DYLIB=ON \ + -DLLVM_LINK_LLVM_DYLIB=OFF \ -DMLIR_ENABLE_BINDINGS_PYTHON=$(BINDINGS_PYTHON_ENABLED) \ -DCONCRETELANG_BINDINGS_PYTHON_ENABLED=$(BINDINGS_PYTHON_ENABLED) \ -DCONCRETELANG_DATAFLOW_EXECUTION_ENABLED=$(DATAFLOW_EXECUTION_ENABLED) \ @@ -173,8 +173,8 @@ clientlib: build-initialized serverlib: build-initialized cmake --build $(BUILD_DIR) --target ConcretelangServerLib -concrete-sys: build-initialized - cmake --build $(BUILD_DIR) --target ConcreteSys +concrete-rust: build-initialized + cmake --build $(BUILD_DIR) --target ConcreteRust GITHUB_URL=https://api.github.com/repos/zama-ai/concrete-compiler-internal GITHUB_URL_LIST_ARTIFACTS="${GITHUB_URL}/actions/artifacts?name=${KEYSETCACHENAME}&per_page=1" @@ -234,9 +234,9 @@ run-python-tests: python-bindings concretecompiler runtime ## rust-tests -run-rust-tests: concrete-sys +run-rust-tests: concrete-rust $(eval _abs_build_dir = $(realpath $(BUILD_DIR))) - cd lib/Bindings/Rust && COMPILER_BUILD_DIRECTORY=$(_abs_build_dir) cargo test + cd lib/Bindings/Rust && COMPILER_BUILD_DIRECTORY=$(_abs_build_dir) cargo +nightly test test-compiler-file-output: concretecompiler pytest -vs tests/test_compiler_file_output diff --git a/compilers/concrete-compiler/compiler/cmake/modules/FetchHpxLibrary.cmake b/compilers/concrete-compiler/compiler/cmake/modules/FetchHpxLibrary.cmake index 411d1593f..01ede1546 100644 --- a/compilers/concrete-compiler/compiler/cmake/modules/FetchHpxLibrary.cmake +++ b/compilers/concrete-compiler/compiler/cmake/modules/FetchHpxLibrary.cmake @@ -4,8 +4,7 @@ function(fetch_hpx_library) FetchContent_Declare( HPX GIT_REPOSITORY https://github.com/STEllAR-GROUP/hpx.git - GIT_TAG stable # release 1.10 + fixes for compilation with fopenmp with gcc and clang :,) - GIT_SHALLOW TRUE + GIT_TAG 9c11915378e3c3901e49a5c3a67d6ee476fce3d1 GIT_PROGRESS TRUE ) set(HPX_WITH_FETCH_ASIO ON CACHE BOOL INTERNAL) @@ -14,7 +13,7 @@ function(fetch_hpx_library) set(HPX_WITH_MALLOC system CACHE STRING INTERNAL) set(HPX_WITH_EXAMPLES OFF CACHE BOOL INTERNAL) set(HPX_WITH_TESTS OFF CACHE BOOL INTERNAL) - set(HPX_WITH_STATIC_LINKING ON CACHE BOOL INTERNAL) + set(HPX_WITH_STATIC_LINKING OFF CACHE BOOL INTERNAL) set(HPX_WITH_PKGCONFIG OFF CACHE BOOL INTERNAL) set(HPX_WITH_MAX_CPU_COUNT "" CACHE STRING INTERNAL) set(HPX_WITH_CXX_STANDARD 17 CACHE STRING INTERNAL) @@ -25,4 +24,16 @@ function(fetch_hpx_library) remove_definitions("-Werror ") remove_definitions("-Wfatal-errors") FetchContent_MakeAvailable(HPX) + add_custom_target( + HPXLibs + ALL + COMMAND ${CMAKE_COMMAND} -E copy + ${hpx_BINARY_DIR}/lib/libhpx* + ${CMAKE_BINARY_DIR}/lib + ) + add_dependencies(HPXLibs + HPX::hpx + HPX::iostreams_component + HPX::component + ) endfunction() diff --git a/compilers/concrete-compiler/compiler/include/concretelang/ClientLib/ClientLib.h b/compilers/concrete-compiler/compiler/include/concretelang/ClientLib/ClientLib.h index 8d74364ef..313b0cb80 100644 --- a/compilers/concrete-compiler/compiler/include/concretelang/ClientLib/ClientLib.h +++ b/compilers/concrete-compiler/compiler/include/concretelang/ClientLib/ClientLib.h @@ -105,7 +105,7 @@ public: std::shared_ptr csprng); /// Returns a reference to the named client circuit if it exists. - Result getClientCircuit(std::string circuitName); + Result getClientCircuit(std::string circuitName) const; private: ClientProgram() = default; diff --git a/compilers/concrete-compiler/compiler/include/concretelang/Common/Transformers.h b/compilers/concrete-compiler/compiler/include/concretelang/Common/Transformers.h index e6c18ba02..3a0d401b8 100644 --- a/compilers/concrete-compiler/compiler/include/concretelang/Common/Transformers.h +++ b/compilers/concrete-compiler/compiler/include/concretelang/Common/Transformers.h @@ -19,6 +19,15 @@ using concretelang::values::Tensor; using concretelang::values::TransportValue; using concretelang::values::Value; +/// \brief simulate the encryption of a value by adding noise +/// +/// \param message encoded message to encrypt +/// \param lwe_dim +/// \param csprng used to generate noise during encryption +/// \return noisy plaintext +uint64_t sim_encrypt_lwe_u64(uint64_t message, uint32_t lwe_dim, + Csprng *encrypt_csprng); + namespace concretelang { namespace transformers { diff --git a/compilers/concrete-compiler/compiler/include/concretelang/Common/Values.h b/compilers/concrete-compiler/compiler/include/concretelang/Common/Values.h index 1d59ba84f..aff5b92e7 100644 --- a/compilers/concrete-compiler/compiler/include/concretelang/Common/Values.h +++ b/compilers/concrete-compiler/compiler/include/concretelang/Common/Values.h @@ -174,7 +174,7 @@ struct Value { Message intoProtoShape() const; - std::vector getDimensions() const; + const std::vector &getDimensions() const; size_t getLength() const; @@ -189,6 +189,10 @@ struct Value { return std::get>(inner); } + template const Tensor &getTensorRef() const { + return std::get>(inner); + } + template Tensor *getTensorPtr() { if (!hasElementType()) { return nullptr; diff --git a/compilers/concrete-compiler/compiler/include/concretelang/Runtime/simulation.h b/compilers/concrete-compiler/compiler/include/concretelang/Runtime/simulation.h index 72e5600c7..a781fbcfe 100644 --- a/compilers/concrete-compiler/compiler/include/concretelang/Runtime/simulation.h +++ b/compilers/concrete-compiler/compiler/include/concretelang/Runtime/simulation.h @@ -12,15 +12,6 @@ extern "C" { -/// \brief simulate the encryption of a value by adding noise -/// -/// \param message encoded message to encrypt -/// \param lwe_dim -/// \param csprng used to generate noise during encryption -/// \return noisy plaintext -uint64_t sim_encrypt_lwe_u64(uint64_t message, uint32_t lwe_dim, - Csprng *encrypt_csprng); - /// \brief simulate the negation of a noisy plaintext /// /// \param plaintext noisy plaintext diff --git a/compilers/concrete-compiler/compiler/include/concretelang/Runtime/utils.h b/compilers/concrete-compiler/compiler/include/concretelang/Runtime/utils.h deleted file mode 100644 index 71c1fb39b..000000000 --- a/compilers/concrete-compiler/compiler/include/concretelang/Runtime/utils.h +++ /dev/null @@ -1,21 +0,0 @@ -// Part of the Concrete Compiler Project, under the BSD3 License with Zama -// Exceptions. See -// https://github.com/zama-ai/concrete/blob/main/LICENSE.txt -// for license information. - -#ifndef CONCRETELANG_RUNTIME_UTILS_H -#define CONCRETELANG_RUNTIME_UTILS_H - -#include "llvm/Support/TargetSelect.h" - -namespace mlir { -namespace concretelang { - -// Mainly a wrapper to some LLVM functions. The reason to have this wrapper is -// to avoid linking conflicts between the python binary extension, and LLVM. -void LLVMInitializeNativeTarget(); - -} // namespace concretelang -} // namespace mlir - -#endif diff --git a/compilers/concrete-compiler/compiler/include/concretelang/Runtime/wrappers.h b/compilers/concrete-compiler/compiler/include/concretelang/Runtime/wrappers.h index 568d81fa8..c23467481 100644 --- a/compilers/concrete-compiler/compiler/include/concretelang/Runtime/wrappers.h +++ b/compilers/concrete-compiler/compiler/include/concretelang/Runtime/wrappers.h @@ -6,7 +6,11 @@ #ifndef CONCRETELANG_RUNTIME_WRAPPERS_H #define CONCRETELANG_RUNTIME_WRAPPERS_H -#include "concretelang/Runtime/context.h" +#include "concrete-cpu.h" +#include +#include +#include +#include extern "C" { diff --git a/compilers/concrete-compiler/compiler/include/concretelang/ServerLib/ServerLib.h b/compilers/concrete-compiler/compiler/include/concretelang/ServerLib/ServerLib.h index 476b0bae7..134d20a91 100644 --- a/compilers/concrete-compiler/compiler/include/concretelang/ServerLib/ServerLib.h +++ b/compilers/concrete-compiler/compiler/include/concretelang/ServerLib/ServerLib.h @@ -46,13 +46,18 @@ class ServerCircuit { friend class ServerProgram; public: + static Result + fromFnPtr(const Message &circuitInfo, + void (*func)(void *...), bool useSimulation); + /// Call the circuit with public arguments. - Result> call(const ServerKeyset &serverKeyset, - std::vector &args); + Result> + call(const ServerKeyset &serverKeyset, + const std::vector &args); /// Simulate the circuit with public arguments. Result> - simulate(std::vector &args); + simulate(const std::vector &args); /// Returns the name of this circuit. std::string getName(); diff --git a/compilers/concrete-compiler/compiler/include/concretelang/Support/CompilerEngine.h b/compilers/concrete-compiler/compiler/include/concretelang/Support/CompilerEngine.h index 65a0e669d..8f75e60ed 100644 --- a/compilers/concrete-compiler/compiler/include/concretelang/Support/CompilerEngine.h +++ b/compilers/concrete-compiler/compiler/include/concretelang/Support/CompilerEngine.h @@ -183,6 +183,8 @@ class Library { std::string runtimeLibraryPath; bool cleanUp; mlir::concretelang::ProgramCompilationFeedback compilationFeedback; + +public: std::optional> programInfo; public: diff --git a/compilers/concrete-compiler/compiler/include/concretelang/Support/V0Parameters.h b/compilers/concrete-compiler/compiler/include/concretelang/Support/V0Parameters.h index ec685495a..05bbd78a5 100644 --- a/compilers/concrete-compiler/compiler/include/concretelang/Support/V0Parameters.h +++ b/compilers/concrete-compiler/compiler/include/concretelang/Support/V0Parameters.h @@ -93,10 +93,10 @@ enum Strategy { V0 = 0, /// DAG_MONO is a strategy that used the optimizer dag but resolve with a /// unique set of keyswitch and boostrap key - DAG_MONO, + DAG_MONO = 1, /// DAG_MULTI is a strategy that used the optimizer dag but resolve with a /// multiple set of keyswitch and boostrap key - DAG_MULTI + DAG_MULTI = 2 }; std::string const StrategyLabel[] = {"V0", "dag-mono", "dag-multi"}; diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Python/CMakeLists.txt b/compilers/concrete-compiler/compiler/lib/Bindings/Python/CMakeLists.txt index 3c3ec9802..61a392293 100644 --- a/compilers/concrete-compiler/compiler/lib/Bindings/Python/CMakeLists.txt +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Python/CMakeLists.txt @@ -19,12 +19,6 @@ declare_mlir_python_extension( ConcretelangModule.cpp FHEModule.cpp CompilerAPIModule.cpp - PRIVATE_LINK_LIBS - ConcretelangSupport - ConcretelangCommon - ConcretelangRuntime - ConcretelangClientLib - ConcretelangServerLib EMBED_CAPI_LINK_LIBS MLIRCAPIRegisterEverything) @@ -116,6 +110,12 @@ add_mlir_python_common_capi_library( target_include_directories(ConcretelangBindingsPythonCAPI PUBLIC ${CONCRETE_CPU_INCLUDE_DIR}) +target_link_libraries( + ConcretelangBindingsPythonCAPI + PUBLIC ConcretelangRuntime + PRIVATE $ $ + $) + # Bundle the MLIR python sources into our package. The MLIR API is position independent, so we explicitly output it to # the mlir/ folder as a temporary measure. It will eventually migrate under the concretelang/ folder and be accessible # under the unified "import concretelang..." namespace. diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Python/CompilerAPIModule.cpp b/compilers/concrete-compiler/compiler/lib/Bindings/Python/CompilerAPIModule.cpp index fcd5469ae..660835d17 100644 --- a/compilers/concrete-compiler/compiler/lib/Bindings/Python/CompilerAPIModule.cpp +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Python/CompilerAPIModule.cpp @@ -313,6 +313,21 @@ void mlir::concretelang::python::populateCompilerAPISubmodule( [](concrete_optimizer::restriction::RangeRestriction &restriction, uint64_t v) { restriction.ks_base_log.push_back(v); }, "Add an available ks base log to the restriction") + .def( + "from_json", + [](std::string input) + -> concrete_optimizer::restriction::RangeRestriction { + return concrete_optimizer::restriction::range_restriction_from_json( + input); + }, + "Create a RangeRestriction from a json string.") + .def( + "to_json", + [](concrete_optimizer::restriction::RangeRestriction &restriction) + -> std::string { + return std::string(restriction.range_restriction_to_json()); + }, + "Serialize a RangeRestriction to a json string.") .doc() = "Allow to restrict the optimizer parameter search space to a " "set of values."; @@ -321,6 +336,22 @@ void mlir::concretelang::python::populateCompilerAPISubmodule( // ------------------------------------------------------------------------------// pybind11::class_( m, "KeysetRestriction") + .def( + "from_json", + [](std::string input) + -> concrete_optimizer::restriction::KeysetRestriction { + return concrete_optimizer::restriction:: + keyset_restriction_from_json(input); + }, + "Create a KeysetRestriction from a json string.") + .def( + "to_json", + [](concrete_optimizer::restriction::KeysetRestriction &restriction) + -> std::string { + return std::string(restriction.keyset_restriction_to_json()); + }, + "Serialize a KeysetRestriction to a json string.") + .doc() = "Allow to restrict the optimizer search space to be compatible " "with a keyset."; diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Python/concrete/compiler/__init__.py b/compilers/concrete-compiler/compiler/lib/Bindings/Python/concrete/compiler/__init__.py index 533dfb480..7e2907fd0 100644 --- a/compilers/concrete-compiler/compiler/lib/Bindings/Python/concrete/compiler/__init__.py +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Python/concrete/compiler/__init__.py @@ -3,7 +3,9 @@ """Compiler submodule.""" import atexit +import json from typing import Union +import jsonpickle # pylint: disable=no-name-in-module,import-error from mlir._mlir_libs._concretelang._compiler import ( @@ -33,6 +35,8 @@ from mlir._mlir_libs._concretelang._compiler import ( Library, ProgramCompilationFeedback, CircuitCompilationFeedback, + KeysetRestriction, + RangeRestriction, terminate_df_parallelization as _terminate_df_parallelization, init_df_parallelization as _init_df_parallelization, check_gpu_runtime_enabled as _check_gpu_runtime_enabled, @@ -98,3 +102,27 @@ def round_trip(mlir_str: str) -> str: if not isinstance(mlir_str, str): raise TypeError(f"mlir_str must be of type str, not {type(mlir_str)}") return _round_trip(mlir_str) + + +@jsonpickle.handlers.register(RangeRestriction) +class RangeRestrictionHandler(jsonpickle.handlers.BaseHandler): + """Handler to serialize and deserialize range restrictions""" + + def flatten(self, obj, data): + data["serialized"] = json.loads(obj.to_json()) + return data + + def restore(self, obj): + return RangeRestriction.from_json(json.dumps(obj["serialized"])) + + +@jsonpickle.handlers.register(KeysetRestriction) +class KeysetRestrictionHandler(jsonpickle.handlers.BaseHandler): + """Handler to serialize and deserialize keyset restrictions""" + + def flatten(self, obj, data): + data["serialized"] = json.loads(obj.to_json()) + return data + + def restore(self, obj): + return KeysetRestriction.from_json(json.dumps(obj["serialized"])) diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Python/requirements_dev.txt b/compilers/concrete-compiler/compiler/lib/Bindings/Python/requirements_dev.txt index 97925029b..ba9c7ea11 100644 --- a/compilers/concrete-compiler/compiler/lib/Bindings/Python/requirements_dev.txt +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Python/requirements_dev.txt @@ -2,3 +2,5 @@ black==24.4.0 pylint==3.3.3 mypy==1.11.2 numpy>=1.23,<2.0 +jsonpickle>=3.0.3 +pybind11-stubgen>=2.5 diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/CMakeLists.txt b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/CMakeLists.txt index a943ed272..b40fa0ce2 100644 --- a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/CMakeLists.txt +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/CMakeLists.txt @@ -1,18 +1,25 @@ add_compile_options(-fexceptions) add_custom_command( - OUTPUT concrete-sys/src/lib.rs.h concrete-sys/src/lib.rs.cc - COMMAND cxxbridge ${CMAKE_CURRENT_SOURCE_DIR}/concrete-sys/src/lib.rs --header > concrete-sys/src/lib.rs.h - COMMAND cxxbridge ${CMAKE_CURRENT_SOURCE_DIR}/concrete-sys/src/lib.rs > concrete-sys/src/lib.rs.cc) + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/concrete/src/ffi.rs ${CMAKE_CURRENT_SOURCE_DIR}/concrete/src/ffi.h + OUTPUT concrete/src/ffi.rs.cc ${CMAKE_CURRENT_SOURCE_DIR}/concrete/src/cxx.h + COMMAND cxxbridge ${CMAKE_CURRENT_SOURCE_DIR}/concrete/src/ffi.rs > concrete/src/ffi.rs.cc + COMMAND cxxbridge --header > ${CMAKE_CURRENT_SOURCE_DIR}/concrete/src/cxx.h) -add_library(ConcreteSys SHARED concrete-sys/src/lib.cpp concrete-sys/src/lib.rs.cc) +add_library(ConcreteRust SHARED concrete/src/ffi.h concrete/src/ffi.rs.cc) -target_include_directories(ConcreteSys PRIVATE concrete-sys/src) +target_include_directories(ConcreteRust PRIVATE concrete/src) + +if(LINUX) + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-Bsymbolic") +endif() target_link_libraries( - ConcreteSys + ConcreteRust PRIVATE ConcretelangSupport ConcretelangCommon + ConcretelangClientLib + ConcretelangServerLib ConcretelangRuntimeStatic LLVMSupport capnp @@ -24,5 +31,5 @@ if(APPLE) if(NOT SECURITY_FRAMEWORK) message(FATAL_ERROR "Security framework not found") endif() - target_link_libraries(ConcreteSys LINK_PUBLIC ${SECURITY_FRAMEWORK}) + target_link_libraries(ConcreteRust LINK_PUBLIC ${SECURITY_FRAMEWORK}) endif() diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/Cargo.toml b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/Cargo.toml index 6d67cb65a..20142f1e7 100644 --- a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/Cargo.toml +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/Cargo.toml @@ -1,5 +1,3 @@ [workspace] - -members = ["concrete-sys"] - resolver = "2" +members = ["concrete", "concrete-macro", "test"] diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/NOTES.md b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/NOTES.md new file mode 100644 index 000000000..e863f8ec3 --- /dev/null +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/NOTES.md @@ -0,0 +1,41 @@ +# Developer notes + +## `Cargo` vs `CMake` + +Cargo/Rust and Cmake/C++ projects follows an almost opposite philosophy: ++ Cargo: dependency management is handled internally and the linking is mostly static (actually, it is more than that, it is bundled). ++ Cmake: dependency management is left to the environment and linking is mostly dynamic (no portable solution for bundled static libs). + +Cargo promotes a static (bundled) linking model to manage the dependencies of a Rust project. Specifically, the default behavior for rust libraries, is to statically bundle all the dependencies inside of the resulting `.rlib` file. At the cost of increasing the weights of the intermediate artifacts (a rather small price to pay given the cost of storage nowadays), cargo solves a lots of link-time and run-time burdens with this approach. All the dependencies of a project are pulled and built automatically, and it is rarely needed to tweak `LD_LIBRARY_PATH` to run an application. Also, resulting binaries are portable. + +Cmake promotes a dynamic linking model, in which shared object pre-built binaries are distributed via distribution package managers, and dependencies are resolved at run-time thanks to a dynamic linker. Some recent features of cmake partially help, but there is no real dependency management, no support for building bundled static libraries (static libraries containing all dependencies), and so on. + +As of now, it seems essentially impossible to reliably build an artifact for all the versions of the concrete compiler that can be statically linked into a rust library. For this reason, the concrete compiler artifacts coming from the Cmake world can only be shared objects dynamically linked to rust libraries (with all the book-keeping it brings at run-time to ensure all dependencies are available). + +## `ConcreteSys` and `concrete-sys` + +`concrete-sys` is a thin binding providing the _compilation_ features of concrete. Ideally, following the cargo philosophy, it should be up to cargo to pull the concrete sources and build the necessary artifacts to be linked with `concrete-sys`. In practice, this is not doable given how long it takes to build `llvm`. + +For this reason we pre-build, with CMake, a target specifically for the `concrete-sys` crate, which will then be pulled from the `github` repository at build-time. This Cmake library target is referred to as `ConcreteSys` (CamelCase) and is defined in the workspace level `CMakeLists.txt` file. + +In order for `concrete-sys` (the rust crate) to bind against `ConcreteSys` (the cmake library target), we need to define an interface which can be compiled on the CMake/C++ side, and used on the cargo/rust side. This interface is defined in three file: ++ `concrete-sys/lib.h` and `concrete-sys/lib.rs` which kind of mirror each others for the cpp and the rust side of the interface. ++ `concrete-sys/lib.cpp` which contains the sources of the interface, and are compiled by cmake to the `ConcreteSys` target. + +In practice, other files automatically generated by `CXX` (the rust-cpp binding tool we use), need to be included when compiling the interface. Those are automatically generated with the `cxx-bridge` command by CMake. + +Assuming the `libConcreteSys.so` artifact is built, we rely on the `concrete-sys/build.rs` to bring it to the `target` folder (either from a local build directory, or from the distributed binaries). + +## Non-transitive link-args for `concrete-sys` users. + +`concrete-sys` is built as an `rlib`, and contains a dynamic link dependency on `libConcreteSys.so` file. + +On macos, this means that for the crates depending on `concrete-sys`, they will contain a dependency to `@rpath/libConcreteSys.dylib`. The `@rpath` is a variable that can be set by the linker to help the dynamic linker locate the proper dynamic library. The dynamic linker will try the usual system-wide paths, and the different `@rpath` specified in the binary (`@rpath` can be a list of directory). Those can be set by adding link args such as `-Wl,-rpath,path_to_dylib`. + +In our case, since the `libConcreteSys.so` is not distributed by system package-managers but by cargo which will pull it to the `target` dir, it will not be possible for the dynamic linker to find it in the usual system locations (`/usr/lib/`, `/usr/local/lib` etc). We have to set the `@rpath` properly when linking the final users of the `concrete-sys` crate, for the dynamic linker to find the shared object. + +Unfortunately, `rustc-link-args` are not transitively propagated to dependent crates (See https://github.com/rust-lang/cargo/issues/9554). This means that emitting `-Wl,-rpath,path_to_dylib` from the `concrete-sys/build.rs` script won't set the `@rpath` in the user crates. The link arg must be provided when linking the final user of the `concrete-sys` crate. + +In our case, `concrete-macro` is the main user of the `concrete-sys` crate. Since it contains procedural macros (which are under the hood compiled to `dylib`), we _must_ provide the proper link args when building it. This is achieved in two steps: ++ In the `concrete-sys/build.rs` script, the `libConcreteSys.dylib` artifact is pulled to the root of the target dir (either `target/debug` or `target/release`). ++ In the `concrete-macro/build.rs` script, link args setting the `@rpath` to the target root are issued. diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-macro/Cargo.toml b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-macro/Cargo.toml new file mode 100644 index 000000000..9934089dc --- /dev/null +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-macro/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "concrete-macro" +version = "2.10.1-rc1" +edition = "2021" +readme = "../../../../../../../README.md" +keywords = ["fully", "homomorphic", "encryption", "fhe", "cryptography"] +homepage = "https://zama.ai/" +documentation = "https://docs.zama.ai/concrete" +repository = "https://github.com/zama-ai/concrete" +license = "BSD-3-Clause-Clear" +description = "Concrete is an open-source FHE Compiler that simplifies the use of fully homomorphic encryption (FHE)." +build = "build.rs" + +[lib] +proc-macro = true + +[dependencies] +syn = "2.0" +quote = "1.0" +proc-macro2 = "1.0" +zip = "2.2.2" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +concrete = { version = "2.10.1-rc1", path = "../concrete", features = ["compiler"]} diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-macro/build.rs b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-macro/build.rs new file mode 100644 index 000000000..da167214c --- /dev/null +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-macro/build.rs @@ -0,0 +1,16 @@ +use std::path::PathBuf; + +fn main() { + let out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap()); + let target_dir = out_dir.join("../../../..").canonicalize().unwrap(); + let concrete_dir = target_dir.join("concrete"); + println!( + "cargo:rustc-env=CONCRETE_BUILD_DIR={}", + concrete_dir.display() + ); + #[cfg(target_os = "macos")] + { + println!("cargo::rustc-link-arg=-rpath"); + println!("cargo::rustc-link-arg={}", concrete_dir.display()); + } +} diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-macro/src/configuration.rs b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-macro/src/configuration.rs new file mode 100644 index 000000000..9446fce09 --- /dev/null +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-macro/src/configuration.rs @@ -0,0 +1,248 @@ +use serde::{Deserialize, Deserializer}; +use serde_json::Value; + +struct PythonPickledObject { + py_object: String, + py_serialized: String, +} + +impl<'de> Deserialize<'de> for PythonPickledObject { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let json = Value::deserialize(deserializer)?; + let Value::Object(obj) = json else { + return Err(::custom("Missing object")); + }; + let Some(Value::String(py_object)) = obj.get("py/object") else { + return Err(::custom( + "Missing field \"py/object\"", + )); + }; + let Some(py_serialized) = obj.get("serialized") else { + return Err(::custom( + "Missing field \"serialized\"", + )); + }; + Ok(PythonPickledObject { + py_object: py_object.clone(), + py_serialized: py_serialized.to_string(), + }) + } +} + +struct PythonPickledEnum { + py_type: String, + py_tuple: Value, +} + +impl<'de> Deserialize<'de> for PythonPickledEnum { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let json = Value::deserialize(deserializer)?; + let Value::Object(obj) = json else { + return Err(::custom("Missing object")); + }; + let Some(py_reduce) = obj.get("py/reduce") else { + return Err(::custom( + "Missing field \"py/reduce\"", + )); + }; + let Value::Array(arr) = py_reduce else { + return Err(::custom("Missing array")); + }; + if arr.len() != 2 { + return Err(::custom( + "Unexpected py_reduce array length", + )); + } + let Some(Value::Object(py_type_obj)) = arr.get(0) else { + return Err(::custom("Missing object")); + }; + let Some(Value::String(py_type)) = py_type_obj.get("py/type") else { + return Err(::custom( + "Missing \"py/type\" field.", + )); + }; + let Some(Value::Object(py_tuple_obj)) = arr.get(1) else { + return Err(::custom("Missing object")); + }; + let Some(py_tuple_value) = py_tuple_obj.get("py/tuple") else { + return Err(::custom( + "Missing \"py/tuple\" field.", + )); + }; + let Value::Array(py_tuple_arr) = py_tuple_value else { + return Err(::custom( + "\"py/tuple\" is not an array.", + )); + }; + if py_tuple_arr.len() != 1 { + return Err(::custom( + "Unexpected py_tuple array length", + )); + } + let py_tuple = py_tuple_arr.get(0).unwrap(); + Ok(PythonPickledEnum { + py_type: py_type.clone(), + py_tuple: py_tuple.clone(), + }) + } +} + +#[derive(Debug)] +pub enum ParameterSelectionStrategy { + V0, + Mono, + Multi, +} + +impl<'de> Deserialize<'de> for ParameterSelectionStrategy { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let python_enum = PythonPickledEnum::deserialize(deserializer)?; + if python_enum.py_type.split(".").last() != Some("ParameterSelectionStrategy") { + return Err(::custom( + "Unexpected py/type.", + )); + } + let Value::String(variant) = python_enum.py_tuple else { + return Err(::custom( + "Unexpected py/tuple.", + )); + }; + match variant.as_str() { + "v0" => Ok(ParameterSelectionStrategy::V0), + "mono" => Ok(ParameterSelectionStrategy::Mono), + "multi" => Ok(ParameterSelectionStrategy::Multi), + _ => Err(::custom( + "Unexpected variant.", + )), + } + } +} + +#[derive(Debug)] +pub enum MultiParameterStrategy { + Precision, + PrecisionAndNorm2, +} + +impl<'de> Deserialize<'de> for MultiParameterStrategy { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let python_enum = PythonPickledEnum::deserialize(deserializer)?; + if python_enum.py_type.split(".").last() != Some("MultiParameterStrategy") { + return Err(::custom( + "Unexpected py/type.", + )); + } + let Value::String(variant) = python_enum.py_tuple else { + return Err(::custom( + "Unexpected py/tuple.", + )); + }; + match variant.as_str() { + "precision" => Ok(MultiParameterStrategy::Precision), + "precision_and_norm2" => Ok(MultiParameterStrategy::PrecisionAndNorm2), + _ => Err(::custom( + "Unexpected variant.", + )), + } + } +} + +#[derive(Debug)] +pub enum SecurityLevel { + Security128Bits, + Security132Bits, +} + +impl<'de> Deserialize<'de> for SecurityLevel { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let python_enum = PythonPickledEnum::deserialize(deserializer)?; + if python_enum.py_type.split(".").last() != Some("SecurityLevel") { + return Err(::custom( + "Unexpected py/type.", + )); + } + let Value::Number(variant) = python_enum.py_tuple else { + return Err(::custom( + "Unexpected py/tuple.", + )); + }; + match variant.as_u64() { + Some(128) => Ok(SecurityLevel::Security128Bits), + Some(132) => Ok(SecurityLevel::Security132Bits), + _ => Err(::custom( + "Unexpected variant.", + )), + } + } +} + +#[derive(Debug)] +pub struct RangeRestriction(pub String); + +impl<'de> Deserialize<'de> for RangeRestriction { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let python_object = PythonPickledObject::deserialize(deserializer)?; + if python_object.py_object.split(".").last() != Some("RangeRestriction") { + return Err(::custom( + "Unexpected py/object.", + )); + } + Ok(RangeRestriction(python_object.py_serialized)) + } +} + +#[derive(Debug)] +pub struct KeysetRestriction(pub String); + +impl<'de> Deserialize<'de> for KeysetRestriction { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let python_object = PythonPickledObject::deserialize(deserializer)?; + if python_object.py_object.split(".").last() != Some("KeysetRestriction") { + return Err(::custom( + "Unexpected py/object.", + )); + } + Ok(KeysetRestriction(python_object.py_serialized)) + } +} + +#[derive(Deserialize, Debug)] +pub struct Configuration { + pub show_optimizer: Option, + pub loop_parallelize: bool, + pub dataflow_parallelize: bool, + pub auto_parallelize: bool, + pub compress_evaluation_keys: bool, + pub compress_input_ciphertexts: bool, + pub p_error: Option, + pub global_p_error: Option, + pub parameter_selection_strategy: ParameterSelectionStrategy, + pub multi_parameter_strategy: MultiParameterStrategy, + pub enable_tlu_fusing: bool, + pub detect_overflow_in_simulation: bool, + pub composable: bool, + pub range_restriction: Option, + pub keyset_restriction: Option, + pub security_level: SecurityLevel, +} diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-macro/src/fast_path_hasher.rs b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-macro/src/fast_path_hasher.rs new file mode 100644 index 000000000..d8b08f122 --- /dev/null +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-macro/src/fast_path_hasher.rs @@ -0,0 +1,30 @@ +use std::{hash::{Hash, Hasher}, os::unix::fs::MetadataExt, path::PathBuf}; + +pub struct FastPathHasher { + path: PathBuf, + ctime: i64, + mtime: i64, +} + +impl FastPathHasher { + pub fn from_pathbuf(path: &PathBuf) -> FastPathHasher { + let path = path.canonicalize().unwrap(); + let metadata = path.metadata().unwrap(); + FastPathHasher { + ctime: metadata.ctime(), + mtime: metadata.mtime(), + path, + } + } +} + +impl Hash for FastPathHasher { + fn hash(&self, state: &mut H) + where + H: Hasher, + { + self.path.hash(state); + self.ctime.hash(state); + self.mtime.hash(state); + } +} diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-macro/src/generation.rs b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-macro/src/generation.rs new file mode 100644 index 000000000..51aad87c5 --- /dev/null +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-macro/src/generation.rs @@ -0,0 +1,232 @@ +use concrete::protocol::{CircuitInfo, ProgramInfo}; +use quote::{format_ident, quote}; + +pub fn generate_unsafe_binding(pi: &ProgramInfo) -> proc_macro2::TokenStream { + let func_defs = pi + .circuits + .iter() + .map(|circuit| { + let name = format_ident!("_mlir_concrete_{}", circuit.name); + quote! { + pub fn #name(arg: *mut std::ffi::c_void, ...) -> *mut std::ffi::c_void; + } + }) + .collect::>(); + quote! { + extern "C" { + #(#func_defs)* + } + } +} + +pub fn generate_infos(pi: &ProgramInfo) -> proc_macro2::TokenStream { + quote! { + pub static PROGRAM_INFO: std::sync::LazyLock<::concrete::protocol::ProgramInfo> = std::sync::LazyLock::new(|| { + #pi + }); + } +} + +pub fn generate_keyset() -> proc_macro2::TokenStream { + quote! { + pub fn new_keyset( + secret_csprng: std::pin::Pin<&mut ::concrete::common::SecretCsprng>, + encryption_csprng: std::pin::Pin<&mut ::concrete::common::EncryptionCsprng> + ) -> ::concrete::UniquePtr<::concrete::common::Keyset> { + ::concrete::common::Keyset::new( + &PROGRAM_INFO.keyset, + secret_csprng, + encryption_csprng + ) + } + } +} + +pub(crate) fn generate_client(program_info: &ProgramInfo) -> proc_macro2::TokenStream { + let client_functions = program_info + .circuits + .iter() + .map(|ci| generate_client_function(ci)); + quote! { + pub mod client { + #(#client_functions)* + } + } +} + +fn generate_client_function_prepare_inputs(circuit_info: &CircuitInfo) -> proc_macro2::TokenStream { + let ith = (0..circuit_info.inputs.len()).collect::>(); + let input_idents = circuit_info.inputs.iter().enumerate().map(|(ith, _)| format_ident!("arg_{ith}")).collect::>(); + let input_types = circuit_info.inputs.iter().map(|gi| { + match (gi.rawInfo.integerPrecision, gi.rawInfo.isSigned) { + (8, true) => quote! {::concrete::common::Tensor}, + (8, false) => quote! {::concrete::common::Tensor}, + (16, true) => quote! {::concrete::common::Tensor}, + (16, false) => quote! {::concrete::common::Tensor}, + (32, true) => quote! {::concrete::common::Tensor}, + (32, false) => quote! {::concrete::common::Tensor}, + (64, true) => quote! {::concrete::common::Tensor}, + (64, false) => quote! {::concrete::common::Tensor}, + _ => unreachable!(), + } + }).collect::>(); + let output_types = circuit_info.inputs.iter().map(|_| { + quote! {::concrete::UniquePtr<::concrete::common::TransportValue>} + }).collect::>(); + + quote!{ + pub fn prepare_inputs(&mut self, #(#input_idents: #input_types),*) -> (#(#output_types),*) { + ( + #( + self.0.pin_mut().prepare_input(::concrete::common::Value::from_tensor(#input_idents), #ith) + ),* + ) + } + } +} + +fn generate_client_function_process_outputs(circuit_info: &CircuitInfo) -> proc_macro2::TokenStream { + let ith = (0..circuit_info.outputs.len()).collect::>(); + let input_idents = circuit_info.outputs.iter().enumerate().map(|(ith, _)| format_ident!("res_{ith}")).collect::>(); + let input_types = circuit_info.outputs.iter().map(|_| { + quote! {::concrete::UniquePtr<::concrete::common::TransportValue>} + }).collect::>(); + let output_types = circuit_info.outputs.iter().map(|gi| { + match (gi.rawInfo.integerPrecision, gi.rawInfo.isSigned) { + (8, true) => quote! {::concrete::common::Tensor}, + (8, false) => quote! {::concrete::common::Tensor}, + (16, true) => quote! {::concrete::common::Tensor}, + (16, false) => quote! {::concrete::common::Tensor}, + (32, true) => quote! {::concrete::common::Tensor}, + (32, false) => quote! {::concrete::common::Tensor}, + (64, true) => quote! {::concrete::common::Tensor}, + (64, false) => quote! {::concrete::common::Tensor}, + _ => unreachable!(), + } + }).collect::>(); + let output_unwrap = circuit_info.outputs.iter().map(|gi| { + match (gi.rawInfo.integerPrecision, gi.rawInfo.isSigned) { + (8, true) => quote! {get_tensor::().unwrap()}, + (8, false) => quote! {get_tensor::().unwrap()}, + (16, true) => quote! {get_tensor::().unwrap()}, + (16, false) => quote! {get_tensor::().unwrap()}, + (32, true) => quote! {get_tensor::().unwrap()}, + (32, false) => quote! {get_tensor::().unwrap()}, + (64, true) => quote! {get_tensor::().unwrap()}, + (64, false) => quote! {get_tensor::().unwrap()}, + _ => unreachable!(), + } + }).collect::>(); + + + quote!{ + pub fn process_outputs(&mut self, #(#input_idents: #input_types),*) -> (#(#output_types),*) { + ( + #( + self.0.pin_mut().process_output(#input_idents, #ith).#output_unwrap + ),* + ) + } + } +} + +fn generate_client_function(circuit_info: &CircuitInfo) -> proc_macro2::TokenStream { + let function_identifier = format_ident!("{}", circuit_info.name); + + let prepare_inputs = generate_client_function_prepare_inputs(circuit_info); + let process_outputs = generate_client_function_process_outputs(circuit_info); + + quote! { + pub mod #function_identifier{ + pub static CIRCUIT_INFO: std::sync::LazyLock<::concrete::protocol::CircuitInfo> = std::sync::LazyLock::new(|| { + #circuit_info + }); + + pub struct ClientFunction(::concrete::UniquePtr<::concrete::client::ClientFunction>); + + impl ClientFunction{ + pub fn new( + client_keyset: &::concrete::common::ClientKeyset, + encryption_csprng: ::concrete::UniquePtr<::concrete::common::EncryptionCsprng> + ) -> Self { + ClientFunction( + ::concrete::client::ClientFunction::new_encrypted( + & CIRCUIT_INFO, + client_keyset, + encryption_csprng + ) + ) + } + + #prepare_inputs + + #process_outputs + } + } + } +} + +pub(crate) fn generate_server(program_info: &ProgramInfo) -> proc_macro2::TokenStream { + let server_functions = program_info + .circuits + .iter() + .map(|ci| generate_server_function(ci)); + quote! { + pub mod server { + #(#server_functions)* + } + } +} + +fn generate_server_function(circuit_info: &CircuitInfo) -> proc_macro2::TokenStream { + + let function_identifier = format_ident!("{}", circuit_info.name); + let binding_identifier = format_ident!("_mlir_concrete_{}", circuit_info.name); + let invoke = generate_server_function_invoke(circuit_info); + + quote! { + pub mod #function_identifier{ + pub static CIRCUIT_INFO: std::sync::LazyLock<::concrete::protocol::CircuitInfo> = std::sync::LazyLock::new(|| { + #circuit_info + }); + + pub struct ServerFunction(::concrete::UniquePtr<::concrete::server::ServerFunction>); + + impl ServerFunction{ + pub fn new() -> Self { + ServerFunction( + ::concrete::server::ServerFunction::new( + & CIRCUIT_INFO, + super::super::_binding::#binding_identifier as *mut ::concrete::c_void, + false + ) + ) + } + + #invoke + } + } + } + +} + +fn generate_server_function_invoke(circuit_info: &CircuitInfo) -> proc_macro2::TokenStream { + let args_idents = (0..circuit_info.inputs.len()).map(|a| format_ident!("arg_{a}")).collect::>(); + let args_types = (0..circuit_info.inputs.len()).map(|_| quote!{::concrete::UniquePtr<::concrete::common::TransportValue>}).collect::>(); + let results_idents = (0..circuit_info.outputs.len()).map(|a| format_ident!("res_{a}")).collect::>(); + let results_types = (0..circuit_info.outputs.len()).map(|_| quote!{::concrete::UniquePtr<::concrete::common::TransportValue>}).collect::>(); + let output_len = circuit_info.outputs.len(); + + quote!{ + pub fn invoke(&mut self, server_keyset: &::concrete::common::ServerKeyset, #(#args_idents: #args_types),*) -> (#(#results_types),*) { + let inputs = vec![ + #(#args_idents),* + ]; + let output = self.0.pin_mut().call(server_keyset, inputs); + let [#(#results_idents),*] = <[::concrete::UniquePtr<::concrete::common::TransportValue>; #output_len]>::try_from(output).unwrap(); + ( + #(#results_idents),* + ) + } + } +} diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-macro/src/lib.rs b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-macro/src/lib.rs new file mode 100644 index 000000000..a476fc054 --- /dev/null +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-macro/src/lib.rs @@ -0,0 +1,247 @@ +#![allow(stable_features)] +#![feature(file_lock)] +#[allow(unused)] +use quote::quote; +use concrete::{compiler, protocol::ProgramInfo}; +use configuration::Configuration; +use proc_macro::{ + TokenStream, {self}, +}; +use std::{fs::read_to_string, path::PathBuf}; +use std::hash::{DefaultHasher, Hash, Hasher}; +use syn::LitStr; + +const CONCRETE_BUILD_DIR: &'static str = env!("CONCRETE_BUILD_DIR"); +const PATH_STATIC_LIB: &'static str = "staticlib.a"; +const PATH_PROGRAM_INFO: &'static str = "program_info.concrete.params.json"; +const PATH_CIRCUIT: &'static str = "circuit.mlir"; +const PATH_COMPOSITION_RULES: &'static str = "composition_rules.json"; +const PATH_SIMULATED: &'static str = "is_simulated"; +const PATH_CONFIGURATION: &'static str = "configuration.json"; +const DEFAULT_GLOBAL_P_ERROR: Option = Some(0.00001); +const DEFAULT_P_ERROR: Option = None; + +mod configuration; +mod fast_path_hasher; +mod unzip; +mod generation; + +#[proc_macro] +pub fn from_concrete_python_export_zip(input: TokenStream) -> TokenStream { + let pt: Result = syn::parse(input); + + let Ok(path_litteral) = pt else { + panic!("Ununwraped input. Expected path string litteral."); + }; + + let path = PathBuf::from(path_litteral.value()); + if !path.is_relative() { + panic!("Found absolute artifact path. Artifacts paths are resolved relative to the CARGO_MANIFEST_DIR directory."); + } + if std::env::var("CARGO_MANIFEST_DIR").is_err() { + panic!("CARGO_MANIFEST_DIR environment variable not set (usually set by cargo). Something is wrong."); + } + let zip_path = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()).join(path); + if !zip_path.exists() { + panic!("Input path must point to an existing export zip (relative to CARGO_MANIFEST_DIR): File {} not found.", zip_path.display()); + }; + let concrete_build_dir = PathBuf::from(CONCRETE_BUILD_DIR); + + let mut s = DefaultHasher::new(); + fast_path_hasher::FastPathHasher::from_pathbuf(&zip_path).hash(&mut s); + let hash_val = s.finish(); + + if !concrete_build_dir.exists() { + let _ = std::fs::create_dir(&concrete_build_dir); + } + + let lock_file = std::fs::OpenOptions::new() + .write(true) + .create(true) + .open(concrete_build_dir.join(format!("{hash_val}.lock"))) + .expect("Failed to open lock file."); + + lock_file.lock().expect("Failed to acquire lock on the lock file"); + + let concrete_hash_dir = concrete_build_dir.join(format!("{hash_val}")); + if !concrete_hash_dir.exists() { + unzip::unzip(&zip_path, &concrete_hash_dir); + } + + let circuit_path = concrete_hash_dir.join(PATH_CIRCUIT); + let simulated_path = concrete_hash_dir.join(PATH_SIMULATED); + let config_path = concrete_hash_dir.join(PATH_CONFIGURATION); + let composition_rules_path = concrete_hash_dir.join(PATH_COMPOSITION_RULES); + let output_lib_path = concrete_hash_dir.join(PATH_STATIC_LIB); + + if !output_lib_path.exists() { + if !circuit_path.exists() { + panic!("Missing `circuit.mlir` file in the export. Did you save your server with the `via_mlir` option ?"); + } + let mlir = read_to_string(circuit_path).expect("Failed to read mlir sources to string"); + + if !simulated_path.exists() { + panic!("Missing `is_simulated` file in the export. Did you save your server with the `via_mlir` option ?"); + } + let is_simulated = read_to_string(simulated_path) + .expect("Failed to read is_simulated") + .parse::() + .expect("is_simulated can not be parsed as an u8 ..."); + + if !config_path.exists() { + panic!("Missing `configuration.json` file in the export. Did you save your server with the `via_mlir` option ?"); + } + let configuration_string = read_to_string(config_path).expect("Failed to read configuration to string"); + let conf: Configuration = serde_json::from_str(configuration_string.as_str()).expect("Failed to deserialize configuration"); + + if !composition_rules_path.exists() { + panic!("Missing `composition_rules.json` file in the export. Did you save your server with the `via_mlir` option ?"); + } + let composition_rules_string = read_to_string(composition_rules_path).expect("Failed to read composition rules to string"); + let composition_rules: Vec = + serde_json::from_str(composition_rules_string.as_str()).expect("Failed to deserialize composition rules"); + + let mut opts = compiler::CompilationOptions::new(); + opts.pin_mut() + .set_display_optimizer_choice(conf.show_optimizer.unwrap_or(false)); + opts.pin_mut().set_loop_parallelize(conf.loop_parallelize); + opts.pin_mut() + .set_dataflow_parallelize(conf.dataflow_parallelize); + opts.pin_mut().set_auto_parallelize(conf.auto_parallelize); + opts.pin_mut() + .set_compress_evaluation_keys(conf.compress_evaluation_keys); + opts.pin_mut() + .set_compress_input_ciphertexts(conf.compress_input_ciphertexts); + + let global_p_error_is_set = conf.global_p_error.is_some(); + let p_error_is_set = conf.p_error.is_some(); + if global_p_error_is_set && p_error_is_set { + opts.pin_mut() + .set_global_p_error(conf.global_p_error.unwrap()); + opts.pin_mut().set_p_error(conf.p_error.unwrap()); + } else if global_p_error_is_set { + opts.pin_mut() + .set_global_p_error(conf.global_p_error.unwrap()); + opts.pin_mut().set_p_error(1.0) + } else if p_error_is_set { + opts.pin_mut().set_global_p_error(1.0); + opts.pin_mut().set_p_error(conf.p_error.unwrap()); + } else { + opts.pin_mut() + .set_global_p_error(DEFAULT_GLOBAL_P_ERROR.unwrap_or(1.0)); + opts.pin_mut().set_p_error(DEFAULT_P_ERROR.unwrap_or(1.0)); + } + + match conf.parameter_selection_strategy { + configuration::ParameterSelectionStrategy::V0 => { + opts.pin_mut().set_optimizer_strategy(0) + } + configuration::ParameterSelectionStrategy::Mono => { + opts.pin_mut().set_optimizer_strategy(1) + } + configuration::ParameterSelectionStrategy::Multi => { + opts.pin_mut().set_optimizer_strategy(2) + } + } + + match conf.multi_parameter_strategy { + configuration::MultiParameterStrategy::Precision => { + opts.pin_mut().set_optimizer_multi_parameter_strategy(0) + } + configuration::MultiParameterStrategy::PrecisionAndNorm2 => { + opts.pin_mut().set_optimizer_multi_parameter_strategy(1) + } + } + + opts.pin_mut().set_enable_tlu_fusing(conf.enable_tlu_fusing); + opts.pin_mut().set_simulate(is_simulated != 0); + opts.pin_mut() + .set_enable_overflow_detection_in_simulation(conf.detect_overflow_in_simulation); + opts.pin_mut().set_composable(conf.composable); + opts.pin_mut() + .set_range_restriction(&conf.range_restriction.map(|a| a.0).unwrap_or("".into())); + opts.pin_mut() + .set_keyset_restriction(&conf.keyset_restriction.map(|a| a.0).unwrap_or("".into())); + match conf.security_level { + configuration::SecurityLevel::Security128Bits => opts.pin_mut().set_security_level(128), + configuration::SecurityLevel::Security132Bits => opts.pin_mut().set_security_level(132), + } + + for rule in composition_rules { + let serde_json::Value::Array(arr) = rule else { + panic!() + }; + let Some(serde_json::Value::Array(from)) = arr.get(0) else { + panic!() + }; + let Some(serde_json::Value::String(from_func)) = from.get(0) else { + panic!() + }; + let Some(serde_json::Value::Number(from_pos)) = from.get(1) else { + panic!() + }; + + let Some(serde_json::Value::Array(to)) = arr.get(0) else { + panic!() + }; + let Some(serde_json::Value::String(to_func)) = to.get(0) else { + panic!() + }; + let Some(serde_json::Value::Number(to_pos)) = to.get(1) else { + panic!() + }; + opts.pin_mut().add_composition_rule( + from_func, + from_pos.as_u64().unwrap() as usize, + to_func, + to_pos.as_u64().unwrap() as usize, + ); + } + + compiler::compile( + mlir.as_str(), + &opts, + concrete_hash_dir.as_os_str().to_str().unwrap(), + ) + .expect("Failed to compile sources"); + } + + let output_path = concrete_build_dir.join(format!("libconcrete-artifact-{hash_val}.a")); + if !output_path.exists() { + std::fs::copy(concrete_hash_dir.join(PATH_STATIC_LIB), output_path) + .unwrap(); + } + + let concrete_program_info_path = concrete_hash_dir.join(PATH_PROGRAM_INFO); + if !concrete_program_info_path.exists() { + panic!("Missing `program_info.concrete.params.json` file after compilation. Something is wrong. Delete target folder and re-compile."); + } + let program_info: ProgramInfo = serde_json::from_reader( + std::fs::File::open(concrete_program_info_path).unwrap(), + ) + .unwrap(); + + lock_file.unlock().unwrap(); + + let lib_name = format!("concrete-artifact-{hash_val}"); + let unsafe_binding = generation::generate_unsafe_binding(&program_info); + let infos = generation::generate_infos(&program_info); + let keyset = generation::generate_keyset(); + let client = generation::generate_client(&program_info); + let server = generation::generate_server(&program_info); + + quote! { + #infos + #keyset + #client + #server + + #[doc(hidden)] + pub mod _binding { + #[link(name = "ConcretelangRuntime", kind="dylib")] + #[link(name = #lib_name, kind="static")] + #unsafe_binding + } + } + .into() +} diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-macro/src/unzip.rs b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-macro/src/unzip.rs new file mode 100644 index 000000000..6f690de32 --- /dev/null +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-macro/src/unzip.rs @@ -0,0 +1,36 @@ +use std::path::Path; +use zip; + +pub fn unzip(zip_path: &Path, to: &Path) { + let file = std::fs::File::open(zip_path).unwrap(); + let mut archive = zip::ZipArchive::new(file).unwrap(); + for i in 0..archive.len() { + let mut file = archive.by_index(i).unwrap(); + let outpath = match file.enclosed_name() { + Some(path) => to.join(path), + None => continue, + }; + if file.is_dir() { + std::fs::create_dir_all(&outpath).unwrap(); + } else { + if let Some(p) = outpath.parent() { + if !p.exists() { + std::fs::create_dir_all(p).unwrap(); + } + } + let mut outfile = std::fs::File::create(&outpath).unwrap(); + std::io::copy(&mut file, &mut outfile).unwrap(); + } + + // Get and Set permissions + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + if let Some(mode) = file.unix_mode() { + std::fs::set_permissions(&outpath, std::fs::Permissions::from_mode(mode)) + .unwrap(); + } + } + } +} diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/Cargo.toml b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/Cargo.toml deleted file mode 100644 index 9a0bf1f28..000000000 --- a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "concrete-sys" -version = "0.1.0" -edition = "2021" - -[dependencies] -cxx = "1.0" - -[build-dependencies] -cxx-build = "1.0" - -[dev-dependencies] -ar = "0.9" diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/build.rs b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/build.rs deleted file mode 100644 index 5df633e2c..000000000 --- a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/build.rs +++ /dev/null @@ -1,33 +0,0 @@ -use std::path::PathBuf; - -#[cfg(target_os = "macos")] -const ARTIFACT_NAME: &str = "libConcreteSys.dylib"; -#[cfg(target_os = "linux")] -const ARTIFACT_NAME: &str = "libConcreteSys.so"; - -fn main() { - let mut from = PathBuf::from( - std::env::var("COMPILER_BUILD_DIRECTORY") - .expect("Environment variable COMPILER_BUILD_DIRECTORY must be set"), - ); - from.push("lib"); - from.push(ARTIFACT_NAME); - let from = from.canonicalize().unwrap(); - let mut to = PathBuf::from(std::env::var("OUT_DIR").unwrap()); - to.push(ARTIFACT_NAME); - std::fs::copy(&from, &to).unwrap(); - println!( - "cargo::rustc-link-search={}", - std::env::var("OUT_DIR").unwrap() - ); - println!("cargo::rustc-link-lib=dylib=ConcreteSys"); - println!("cargo::rustc-link-lib=z"); - #[cfg(target_os = "macos")] - { - println!("cargo::rustc-link-search=/opt/homebrew/lib"); - println!("cargo::rustc-link-lib=curses"); - println!("cargo::rustc-link-lib=zstd"); - } - println!("cargo::rerun-if-changed=src"); - println!("cargo::rerun-if-changed={}", from.display()); -} diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/src/.gitignore b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/src/.gitignore deleted file mode 100644 index 24b269554..000000000 --- a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/src/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -lib.rs.cc -lib.rs.h diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/src/lib.cpp b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/src/lib.cpp deleted file mode 100644 index 3b0ffad40..000000000 --- a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/src/lib.cpp +++ /dev/null @@ -1,57 +0,0 @@ -// Part of the Concrete Compiler Project, under the BSD3 License with Zama -// Exceptions. See -// https://github.com/zama-ai/concrete/blob/main/LICENSE.txt -// for license information. - -#include "lib.h" -#include "concretelang/Support/CompilerEngine.h" -#include "cxx.h" -#include - -using mlir::concretelang::CompilationContext; -using mlir::concretelang::CompilerEngine; - -namespace concrete_sys { - -// CompilationOptions -std::unique_ptr New() { - return std::make_unique(CompilationOptions{mlir::concretelang::CompilationOptions()}); -} - -// Library -std::unique_ptr Library::getProgramInfo() { - auto maybeProgramInfo = this->inner.getProgramInfo(); - if (maybeProgramInfo.has_error()) { - throw std::runtime_error(std::move(maybeProgramInfo.error().mesg)); - } - return std::make_unique(ProgramInfo{maybeProgramInfo.value()}); -} - -rust::String Library::getStaticLibraryPath(){ - return this->inner.getStaticLibraryPath(); -} - -std::unique_ptr compile(rust::Str sources, - const CompilationOptions &options, - rust::Str outputDirPath) { - auto cxxSources = (std::string)sources; - auto cxxOutputDirPath = (std::string)outputDirPath; - auto context = CompilationContext::createShared(); - auto engine = CompilerEngine(context); - engine.setCompilationOptions(options.inner); - std::unique_ptr mb = - llvm::MemoryBuffer::getMemBuffer(cxxSources); - llvm::SourceMgr sm; - sm.AddNewSourceBuffer(std::move(mb), llvm::SMLoc()); - auto maybeLibrary = - engine.compile(sm, cxxOutputDirPath, "", false, true, true, true); - if (auto err = maybeLibrary.takeError()) { - throw std::runtime_error(llvm::toString(std::move(err))); - } - mlir::concretelang::Library output = *maybeLibrary; - return std::make_unique(Library{output}); -} - - - -} // namespace concrete_sys diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/src/lib.h b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/src/lib.h deleted file mode 100644 index ec5516379..000000000 --- a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/src/lib.h +++ /dev/null @@ -1,40 +0,0 @@ -// Part of the Concrete Compiler Project, under the BSD3 License with Zama -// Exceptions. See -// https://github.com/zama-ai/concrete/blob/main/LICENSE.txt -// for license information. - -#ifndef CONCRETE_SYS_LIB_H -#define CONCRETE_SYS_LIB_H - -#include "cxx.h" -#include "concretelang/Support/CompilerEngine.h" -#include - -// Contains helpers not available from the original compiler api. -namespace concrete_sys{ - -struct CompilationOptions{ - mlir::concretelang::CompilationOptions inner; -}; - -std::unique_ptr New(); - -struct ProgramInfo { - concretelang::protocol::Message inner; -}; - -struct Library { - mlir::concretelang::Library inner; - std::unique_ptr getProgramInfo(); - rust::String getStaticLibraryPath(); -}; - -std::unique_ptr compile( - rust::Str sm, - const CompilationOptions &options, - rust::Str outputDirPath -); - -} // namespace concrete_sys - -#endif diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/src/lib.rs b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/src/lib.rs deleted file mode 100644 index ef4c3353c..000000000 --- a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/src/lib.rs +++ /dev/null @@ -1,64 +0,0 @@ -#![allow(unused_imports)] - -use cxx::UniquePtr; - -#[cxx::bridge(namespace = "concrete_sys")] -mod ffi { - - unsafe extern "C++" { - include!("lib.h"); - - // ------------------------------------------------------------------------------- CompilationOptions - type CompilationOptions; - fn New() -> UniquePtr; - - // -------------------------------------------------------------------------------------- ProgramInfo - type ProgramInfo; - - // ------------------------------------------------------------------------------------------ Library - type Library; - fn getStaticLibraryPath(self: Pin<&mut Library>) -> String; - fn getProgramInfo(self: Pin<&mut Library>) -> UniquePtr; - - // ------------------------------------------------------------------------------------------ compile - fn compile( - sources: &str, - options: &CompilationOptions, - output_dir_path: &str, - ) -> Result>; - } -} -pub use ffi::*; - -#[cfg(test)] -mod test { - use super::*; - - const TEST_FOLDER: &str = "/tmp/test_concrete_sys"; - - #[test] - fn test_compile() { - let sources = " - func.func @dec(%arg0: !FHE.eint<3>) -> !FHE.eint<3> { - %cst_1 = arith.constant 1 : i4 - %1 = \"FHE.sub_eint_int\"(%arg0, %cst_1) : (!FHE.eint<3>, i4) -> !FHE.eint<3> - return %1: !FHE.eint<3> - } - "; - let options = New(); - let _ = std::fs::remove_dir(TEST_FOLDER); - let _ = std::fs::create_dir_all(TEST_FOLDER); - let mut output = compile(sources, &options, TEST_FOLDER).unwrap(); - let static_library_path = output.as_mut().unwrap().getStaticLibraryPath(); - let mut archive = ar::Archive::new(std::fs::File::open(static_library_path).unwrap()); - let symbols = archive - .symbols() - .expect("failed to parse symbols") - .map(|a| String::from_utf8_lossy(a).to_string()) - .collect::>(); - #[cfg(target_os = "macos")] - assert!(symbols.contains(&"_concrete_dec".to_string())); - #[cfg(target_os = "linux")] - assert!(symbols.contains(&"concrete_dec".to_string())); - } -} diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/.gitignore b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/.gitignore similarity index 100% rename from compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/.gitignore rename to compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/.gitignore diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/Cargo.toml b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/Cargo.toml new file mode 100644 index 000000000..c89f59599 --- /dev/null +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "concrete" +version = "2.10.1" +edition = "2021" +links = "concrete" +readme = "../../../../../../../README.md" +keywords = ["fully", "homomorphic", "encryption", "fhe", "cryptography"] +homepage = "https://zama.ai/" +documentation = "https://docs.zama.ai/concrete" +repository = "https://github.com/zama-ai/concrete" +license = "BSD-3-Clause-Clear" +description = "Concrete is an open-source FHE Compiler that simplifies the use of fully homomorphic encryption (FHE)." +build = "build.rs" + +[dependencies] +cxx = "1.0" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +quote = {version = "1.0", optional =true } +proc-macro2 = {version = "1.0", optional = true} + +[build-dependencies] +zip = "2.6" +curl = "0.4.47" +cxx-build = "1.0" + +[dev-dependencies] +ar = "0.9" + +[features] +compiler = ["dep:quote", "dep:proc-macro2"] diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/build.rs b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/build.rs new file mode 100644 index 000000000..64da7a21a --- /dev/null +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/build.rs @@ -0,0 +1,158 @@ +#![allow(stable_features)] +#![feature(file_lock)] +use curl::easy::{Handler, WriteError}; +use curl::{self}; +use std::io::Cursor; +use std::path::{Path, PathBuf}; + +const ARTIFACTS: &[&str] = if cfg!(target_os = "macos") { + &[ + "libConcreteRust.dylib", + "libConcretelangRuntime.dylib", + "libomp.dylib" + ] +} else if cfg!(target_os = "linux") { + &[ + "libConcreteRust.so", + "libConcretelangRuntime.so", + "libomp.so", + "libhpx.so", + "libhpx_core.so", + "libhpx_iostreams.so", + ] +} else { + panic!("Unsupported platform"); +}; + +const ARCHIVE: &str = if cfg!(all(target_os = "macos", target_arch = "x86_64")) { + "cpu-binaries-macosx_x86_64.zip" +} else if cfg!(all(target_os = "macos", target_arch = "aarch64")) { + "cpu-binaries-macosx_arm64.zip" +} else if cfg!(all(target_os = "linux", target_arch = "x86_64")) { + "cpu-binaries-linux-x86_64.zip" +} else { + panic!("Unsupported platform"); +}; + +const INSTALL_LOCK: &str = "install"; +const INSTALLED_FILE: &str = "installed"; +const URL: &str = "https://github.com/zama-ai/concrete/releases/download/v2.10.1-rc1"; + +fn do_with_lock(file: &Path, f: F) { + let mut lock_file_name = file.to_path_buf(); + lock_file_name.set_extension("lock"); + let Ok(lock_file) = std::fs::OpenOptions::new() + .write(true) + .create(true) + .open(&lock_file_name) + else { + panic!("Failed to open lock file {}", lock_file_name.display()) + }; + assert!( + lock_file.lock().is_ok(), + "Failed to acquire lock on file {}", + lock_file_name.display() + ); + let mut f = f; + f(); + assert!( + lock_file.unlock().is_ok(), + "Failed to release lock on file {}", + lock_file_name.display() + ); +} + +fn install_artifacts(out_dir: &Path) { + if !out_dir.join(INSTALLED_FILE).exists() { + match std::env::var("COMPILER_BUILD_DIRECTORY") { + Ok(v) => copy_local_artifacts(out_dir, PathBuf::from(v)), + Err(_) => fetch_artifacts(out_dir), + } + } +} + +fn copy_local_artifacts(out_dir: &Path, mut lib_dir: PathBuf) { + println!("cargo:warning=\"COMPILER_BUILD_DIRECTORY detected: building from local artifacts\""); + lib_dir.push("lib"); + lib_dir = lib_dir.canonicalize().unwrap(); + do_with_lock(&out_dir.join(INSTALL_LOCK), || { + for art in ARTIFACTS { + std::fs::copy(&lib_dir.join(art), &out_dir.join(art)).unwrap(); + } + }); +} + +fn fetch_artifacts(out_dir: &Path) { + + struct Archive(Vec); + impl Handler for Archive { + fn write(&mut self, data: &[u8]) -> Result { + self.0.extend_from_slice(data); + Ok(data.len()) + } + } + + do_with_lock(&out_dir.join(INSTALL_LOCK), || { + if out_dir.join(INSTALLED_FILE).exists() { + return; + } + let url = format!("{URL}/{ARCHIVE}"); + let mut handle = curl::easy::Easy2::new(Archive(Vec::new())); + handle.get(true).unwrap(); + handle.follow_location(true).unwrap(); + handle.url(&url).unwrap(); + handle.perform().unwrap(); + assert_eq!( + handle.response_code().unwrap(), + 200, + "Failed to fetch the remote artifacts from {url}" + ); + let mut zip = zip::ZipArchive::new(Cursor::new(handle.get_ref().0.as_slice())).unwrap(); + for i in 0..zip.len() { + let mut content = zip.by_index(i).unwrap(); + let path = PathBuf::from(content.name()); + let mut file = std::fs::OpenOptions::new() + .write(true) + .create(true) + .open(out_dir.join(path.file_name().unwrap())) + .unwrap(); + std::io::copy(&mut content, &mut file).unwrap(); + } + std::fs::File::create(out_dir.join(INSTALLED_FILE)).unwrap(); + }); +} + +fn symlink_outdir(concrete_dir: &Path, out_dir: &Path) { + if !concrete_dir.exists() { + let _ = std::fs::create_dir(&concrete_dir); + } + if !out_dir.is_symlink() { + assert!( + std::fs::remove_dir(&out_dir).is_ok(), + "Failed to delete original out_dir {}", + out_dir.display() + ); + std::os::unix::fs::symlink(concrete_dir, &out_dir).unwrap(); + } +} + +fn main() { + let out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap()); + let target_dir = out_dir.join("../../../..").canonicalize().unwrap(); + let concrete_dir = target_dir.join("concrete"); + + symlink_outdir(&concrete_dir, &out_dir); + install_artifacts(&out_dir); + + println!("cargo::rustc-link-search={}", out_dir.display()); + println!("cargo::metadata=build_dir={}", out_dir.display()); + println!("cargo::rustc-link-lib=dylib=ConcreteRust"); + println!("cargo::rustc-link-lib=z"); + #[cfg(target_os = "macos")] + { + println!("cargo::rustc-link-search=/opt/homebrew/lib"); + println!("cargo::rustc-link-lib=curses"); + println!("cargo::rustc-link-lib=zstd"); + } + println!("cargo::rerun-if-changed=src"); +} diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/src/.gitignore b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/src/.gitignore new file mode 100644 index 000000000..225fd3d98 --- /dev/null +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/src/.gitignore @@ -0,0 +1 @@ +cxx.h diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/src/cxx.h b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/src/cxx.h similarity index 99% rename from compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/src/cxx.h rename to compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/src/cxx.h index 6a006fbb1..3414e4c8a 100644 --- a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete-sys/src/cxx.h +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/src/cxx.h @@ -1,5 +1,3 @@ -//From https://github.com/dtolnay/cxx/blob/master/include/cxx.h - #pragma once #include #include diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/src/ffi.h b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/src/ffi.h new file mode 100644 index 000000000..c11045bfa --- /dev/null +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/src/ffi.h @@ -0,0 +1,581 @@ +// Part of the Concrete Compiler Project, under the BSD3 License with Zama +// Exceptions. See +// https://github.com/zama-ai/concrete/blob/main/LICENSE.txt +// for license information. + +#ifndef CONCRETE_RUST_COMPILER_H +#define CONCRETE_RUST_COMPILER_H + +#include "concrete-optimizer.hpp" +#include "concrete-protocol.capnp.h" +#include "concretelang/ClientLib/ClientLib.h" +#include "concretelang/Common/Csprng.h" +#include "concretelang/Common/Keys.h" +#include "concretelang/Common/Keysets.h" +#include "concretelang/Common/Values.h" +#include "concretelang/Runtime/context.h" +#include "concretelang/ServerLib/ServerLib.h" +#include "concretelang/Support/CompilerEngine.h" +#include "concretelang/Support/V0Parameters.h" +#include "cxx.h" +#include +#include +#include +#include +#include + +using mlir::concretelang::CompilationContext; +using mlir::concretelang::CompilerEngine; + +inline constexpr capnp::ReaderOptions DESER_OPTIONS = {7000000000, 64}; + +class VecOStream : public std::streambuf { +public: + VecOStream(rust::Vec& vec) : vec_(vec) {} +protected: + int overflow(int c) override { + if (c != EOF) { + vec_.push_back(static_cast(c)); + } + return c; + } +private: + rust::Vec& vec_; +}; + +class SliceIStream : public std::streambuf { +public: + SliceIStream(const rust::Slice& slice) : slice_(slice), pos_(0) { + setg(reinterpret_cast(const_cast(slice_.data())), + reinterpret_cast(const_cast(slice_.data())), + reinterpret_cast(const_cast(slice_.data() + slice_.size()))); + } +protected: + int underflow() override { + if (pos_ >= slice_.size()) { + return EOF; + } + return slice_[pos_++]; + } +private: + const rust::Slice& slice_; + size_t pos_; +}; + +namespace concrete_rust { + +struct CompilationOptions : mlir::concretelang::CompilationOptions { + + void set_display_optimizer_choice(bool val) { + this->optimizerConfig.display = val; + } + + void set_loop_parallelize(bool val) { this->loopParallelize = val; } + + void set_dataflow_parallelize(bool val) { this->dataflowParallelize = val; } + + void set_auto_parallelize(bool val) { this->autoParallelize = val; } + + void set_compress_evaluation_keys(bool val) { + this->compressEvaluationKeys = val; + } + + void set_compress_input_ciphertexts(bool val) { + this->compressInputCiphertexts = val; + } + + void set_p_error(double val) { this->optimizerConfig.p_error = val; } + + void set_global_p_error(double val) { + this->optimizerConfig.global_p_error = val; + } + + void set_optimizer_strategy(uint8_t val) { + this->optimizerConfig.strategy = + static_cast(val); + } + + void set_optimizer_multi_parameter_strategy(uint8_t val) { + this->optimizerConfig.multi_param_strategy = + static_cast(val); + } + void set_enable_tlu_fusing(bool val) { this->enableTluFusing = val; } + + void set_print_tlu_fusing(bool val) { this->printTluFusing = val; } + + void set_enable_overflow_detection_in_simulation(bool val) { + this->enableOverflowDetectionInSimulation = val; + } + void set_simulate(bool val) { this->simulate = val; } + + void set_composable(bool val) { this->optimizerConfig.composable = val; } + + void set_range_restriction(rust::Str json) { + if (!json.empty()) { + this->optimizerConfig.range_restriction = + std::make_shared( + concrete_optimizer::restriction::range_restriction_from_json( + json)); + } + } + + void set_keyset_restriction(rust::Str json) { + if (!json.empty()) { + this->optimizerConfig.keyset_restriction = + std::make_shared( + concrete_optimizer::restriction::keyset_restriction_from_json( + json)); + } + } + + void set_security_level(uint64_t val) { + this->optimizerConfig.security = val; + } + + void add_composition_rule(rust::Str from_func, rust::usize from_pos, + rust::Str to_func, rust::usize to_pos) { + this->optimizerConfig.composition_rules.push_back( + mlir::concretelang::optimizer::CompositionRule{ + std::string(from_func), from_pos, std::string(to_func), to_pos}); + } +}; + +std::unique_ptr _compilation_options_new() { + auto output = std::make_unique(); + return std::unique_ptr( + reinterpret_cast(output.release())); +} + +struct Library : mlir::concretelang::Library { + rust::String _get_program_info_json() const { + return this->programInfo.value().writeJsonToString().value(); + } + rust::String get_static_library_path() const { + return this->getStaticLibraryPath(); + } +}; + +std::unique_ptr compile(rust::Str sources, + const CompilationOptions &options, + rust::Str outputDirPath) { + auto cxxSources = (std::string)sources; + auto cxxOutputDirPath = (std::string)outputDirPath; + auto context = CompilationContext::createShared(); + auto engine = CompilerEngine(context); + engine.setCompilationOptions(options); + std::unique_ptr mb = + llvm::MemoryBuffer::getMemBuffer(cxxSources); + llvm::SourceMgr sm; + sm.AddNewSourceBuffer(std::move(mb), llvm::SMLoc()); + auto maybeLibrary = + engine.compile(sm, cxxOutputDirPath, "", false, true, true, true); + if (auto err = maybeLibrary.takeError()) { + throw std::runtime_error(llvm::toString(std::move(err))); + } + return std::make_unique(Library{*maybeLibrary}); +} + +template struct Key : T { + rust::Slice get_buffer() { + auto buffer = this->getBuffer(); + return {buffer.data(), buffer.size()}; + } + rust::String _get_info_json() const { + return this->getInfo().writeJsonToString().value(); + } +}; + +typedef Key LweSecretKey; +typedef Key LweBootstrapKey; +typedef Key LweKeyswitchKey; +typedef Key PackingKeyswitchKey; + +typedef concretelang::csprng::SecretCSPRNG SecretCsprng; + +std::unique_ptr _secret_csprng_new(uint64_t high, uint64_t low) { + return std::make_unique((static_cast<__uint128_t>(high) << 64) | + low); +} + +typedef concretelang::csprng::EncryptionCSPRNG EncryptionCsprng; + +std::unique_ptr _encryption_csprng_new(uint64_t high, + uint64_t low) { + return std::make_unique( + (static_cast<__uint128_t>(high) << 64) | low); +} + +struct ServerKeyset : concretelang::keysets::ServerKeyset { + size_t _lwe_bootstrap_keys_len() const { + return this->lweBootstrapKeys.size(); + } + + const LweBootstrapKey &_lwe_bootstrap_keys_nth(size_t nth) const { + return static_cast(this->lweBootstrapKeys.at(nth)); + } + + size_t _lwe_keyswitch_keys_len() const { + return this->lweKeyswitchKeys.size(); + } + + const LweKeyswitchKey &_lwe_keyswitch_keys_nth(size_t nth) const { + return static_cast(this->lweKeyswitchKeys.at(nth)); + } + + size_t _packing_keyswitch_keys_len() const { + return this->packingKeyswitchKeys.size(); + } + + const PackingKeyswitchKey &_packing_keyswitch_keys_nth(size_t nth) const { + return static_cast( + this->packingKeyswitchKeys.at(nth)); + } + + rust::Vec serialize() const { + auto proto = toProto(); + auto output = rust::Vec(); + auto vec_ostream = VecOStream(output); + auto ostream = std::ostream(&vec_ostream); + proto.writeBinaryToOstream( + ostream + ).value(); + ostream.flush(); + return output; + } +}; + +std::unique_ptr _deserialize_server_keyset(rust::Slice slice) { + auto proto = Message(); + auto slice_istream = SliceIStream(slice); + auto istream = std::istream(&slice_istream); + proto.readBinaryFromIstream(istream, DESER_OPTIONS).value(); + auto output = std::make_unique(ServerKeyset::fromProto(proto)); + return std::unique_ptr(reinterpret_cast(output.release())); +} + +struct ClientKeyset : concretelang::keysets::ClientKeyset { + size_t _lwe_secret_keys_len() const { return this->lweSecretKeys.size(); } + + const LweSecretKey &_lwe_secret_keys_nth(size_t nth) const { + return static_cast(this->lweSecretKeys.at(nth)); + } + + rust::Vec serialize() const { + auto proto = toProto(); + auto output = rust::Vec(); + auto vec_ostream = VecOStream(output); + auto ostream = std::ostream(&vec_ostream); + proto.writeBinaryToOstream( + ostream + ).value(); + ostream.flush(); + return output; + } +}; + +std::unique_ptr _deserialize_client_keyset(rust::Slice slice) { + auto proto = Message(); + auto slice_istream = SliceIStream(slice); + auto istream = std::istream(&slice_istream); + proto.readBinaryFromIstream(istream, DESER_OPTIONS).value(); + auto output = std::make_unique(ClientKeyset::fromProto(proto)); + return std::unique_ptr(reinterpret_cast(output.release())); +} + +struct Keyset : concretelang::keysets::Keyset { + + std::unique_ptr get_server() const { + auto copy = + std::make_unique(this->server); + return std::unique_ptr( + reinterpret_cast(copy.release())); + } + + std::unique_ptr get_client() const { + auto copy = + std::make_unique(this->client); + return std::unique_ptr( + reinterpret_cast(copy.release())); + } +}; + +std::unique_ptr _keyset_new(rust::Str keyset_info, + SecretCsprng &secret_csprng, + EncryptionCsprng &encryption_csprng) { + auto info = Message(); + info.readJsonFromString(std::string(keyset_info)).value(); + auto output = std::make_unique( + info, secret_csprng, encryption_csprng); + return std::unique_ptr(reinterpret_cast(output.release())); +} + +template struct Tensor : ::concretelang::values::Tensor { + rust::Slice _get_dimensions() const { + return {this->dimensions.data(), this->dimensions.size()}; + } + rust::Slice _get_values() const { + return {this->values.data(), this->values.size()}; + } +}; + +typedef Tensor TensorU8; +typedef Tensor TensorI8; +typedef Tensor TensorU16; +typedef Tensor TensorI16; +typedef Tensor TensorU32; +typedef Tensor TensorI32; +typedef Tensor TensorU64; +typedef Tensor TensorI64; + +template +std::unique_ptr _tensor_new(rust::Slice values, + rust::Slice dimensions) { + std::vector values_vec = {values.begin(), values.end()}; + std::vector dimensions_vec = {dimensions.begin(), + dimensions.end()}; + auto output = std::make_unique<::concretelang::values::Tensor>(values_vec, dimensions_vec); + return std::unique_ptr(reinterpret_cast(output.release())); +} +const auto _tensor_u8_new = _tensor_new; +const auto _tensor_i8_new = _tensor_new; +const auto _tensor_u16_new = _tensor_new; +const auto _tensor_i16_new = _tensor_new; +const auto _tensor_u32_new = _tensor_new; +const auto _tensor_i32_new = _tensor_new; +const auto _tensor_u64_new = _tensor_new; +const auto _tensor_i64_new = _tensor_new; + +struct Value : concretelang::values::Value { + bool _has_element_type_u8() const { return hasElementType(); } + bool _has_element_type_i8() const { return hasElementType(); } + bool _has_element_type_u16() const { return hasElementType(); } + bool _has_element_type_i16() const { return hasElementType(); } + bool _has_element_type_u32() const { return hasElementType(); } + bool _has_element_type_i32() const { return hasElementType(); } + bool _has_element_type_u64() const { return hasElementType(); } + bool _has_element_type_i64() const { return hasElementType(); } + std::unique_ptr _get_tensor_i8() const { + auto output = + std::make_unique<::concretelang::values::Tensor>(std::move(getTensor().value())); + return std::unique_ptr(reinterpret_cast(output.release())); + } + + std::unique_ptr _get_tensor_u8() const { + auto output = + std::make_unique<::concretelang::values::Tensor>(std::move(getTensor().value())); + return std::unique_ptr(reinterpret_cast(output.release())); + } + + std::unique_ptr _get_tensor_i16() const { + auto output = + std::make_unique<::concretelang::values::Tensor>(std::move(getTensor().value())); + return std::unique_ptr(reinterpret_cast(output.release())); + } + + std::unique_ptr _get_tensor_u16() const { + auto output = + std::make_unique<::concretelang::values::Tensor>(std::move(getTensor().value())); + return std::unique_ptr(reinterpret_cast(output.release())); + } + + std::unique_ptr _get_tensor_i32() const { + auto output = + std::make_unique<::concretelang::values::Tensor>(std::move(getTensor().value())); + return std::unique_ptr(reinterpret_cast(output.release())); + } + + std::unique_ptr _get_tensor_u32() const { + auto output = + std::make_unique<::concretelang::values::Tensor>(std::move(getTensor().value())); + return std::unique_ptr(reinterpret_cast(output.release())); + } + + std::unique_ptr _get_tensor_i64() const { + auto output = + std::make_unique<::concretelang::values::Tensor>(std::move(getTensor().value())); + return std::unique_ptr(reinterpret_cast(output.release())); + } + + std::unique_ptr _get_tensor_u64() const { + auto output = + std::make_unique<::concretelang::values::Tensor>(std::move(getTensor().value())); + return std::unique_ptr(reinterpret_cast(output.release())); + } + + rust::Slice get_dimensions() const { + auto vecref = getDimensions(); + return {vecref.data(), vecref.size()}; + } +}; + +template +std::unique_ptr _value_from_tensor(std::unique_ptr tensor) { + T otensor = *tensor; + auto val = std::make_unique(otensor); + return std::unique_ptr(reinterpret_cast(val.release())); +} +const auto _value_from_tensor_u8 = _value_from_tensor; +const auto _value_from_tensor_i8 = _value_from_tensor; +const auto _value_from_tensor_u16 = _value_from_tensor; +const auto _value_from_tensor_i16 = _value_from_tensor; +const auto _value_from_tensor_u32 = _value_from_tensor; +const auto _value_from_tensor_i32 = _value_from_tensor; +const auto _value_from_tensor_u64 = _value_from_tensor; +const auto _value_from_tensor_i64 = _value_from_tensor; + +struct TransportValue : concretelang::values::TransportValue { + std::unique_ptr to_owned() const { + return std::make_unique(*this); + } + + rust::Vec serialize() const { + auto output = rust::Vec(); + auto vec_ostream = VecOStream(output); + auto ostream = std::ostream(&vec_ostream); + this->writeBinaryToOstream( + ostream + ).value(); + ostream.flush(); + return output; + } +}; + +std::unique_ptr _deserialize_transport_value(rust::Slice slice) { + auto output = TransportValue(); + auto slice_istream = SliceIStream(slice); + auto istream = std::istream(&slice_istream); + output.readBinaryFromIstream(istream).value(); + return std::make_unique(output); +} + +struct ClientFunction : concretelang::clientlib::ClientCircuit { + std::unique_ptr prepare_input(std::unique_ptr arg, + size_t pos) { + auto oarg = *arg.release(); + auto output = std::make_unique<::concretelang::values::TransportValue>(prepareInput(oarg, pos).value()); + return std::unique_ptr( + reinterpret_cast(output.release())); + } + std::unique_ptr simulate_prepare_input(const Value &arg, + size_t pos) { + Value oarg = {arg}; + auto output = std::make_unique<::concretelang::values::TransportValue>(simulatePrepareInput(oarg, pos).value()); + return std::unique_ptr( + reinterpret_cast(output.release())); + } + std::unique_ptr process_output(std::unique_ptr result, + size_t pos) { + auto oresult = *result.release(); + auto output = std::make_unique<::concretelang::values::Value>(processOutput(oresult, pos).value()); + return std::unique_ptr(reinterpret_cast(output.release())); + } + std::unique_ptr simulate_process_output(const TransportValue &result, + size_t pos) { + TransportValue oresult = {result}; + auto output = std::make_unique<::concretelang::values::Value>(simulateProcessOutput(oresult, pos).value()); + return std::unique_ptr(reinterpret_cast(output.release())); + } +}; + +std::unique_ptr +_client_function_new_encrypted(rust::Str circuit_info_json, + const ClientKeyset &client_keyset, + std::unique_ptr csprng) { + auto info = Message(); + info.readJsonFromString(std::string(circuit_info_json)).value(); + auto inner = std::make_unique<::concretelang::clientlib::ClientCircuit>(::concretelang::clientlib::ClientCircuit::createEncrypted( + info, client_keyset, std::move(csprng)) + .value()); + return std::unique_ptr( + reinterpret_cast(inner.release())); +} + +std::unique_ptr +_client_function_new_simulated(rust::Str circuit_info_json, + std::unique_ptr csprng) { + auto info = Message(); + info.readJsonFromString(std::string(circuit_info_json)).value(); + auto inner = std::make_unique<::concretelang::clientlib::ClientCircuit>(::concretelang::clientlib::ClientCircuit::createSimulated( + info, std::move(csprng)) + .value()); + return std::unique_ptr( + reinterpret_cast(inner.release())); +} + +struct ClientModule : concretelang::clientlib::ClientProgram { + std::unique_ptr _get_client_function(rust::Str name) const { + auto output = std::make_unique<::concretelang::clientlib::ClientCircuit>(this->getClientCircuit(std::string(name)).value()); + return std::unique_ptr( + reinterpret_cast(output.release())); + } +}; + +std::unique_ptr +_client_module_new_encrypted(rust::Str program_info_json, + const ClientKeyset &client_keyset, + std::unique_ptr csprng) { + auto info = Message(); + info.readJsonFromString(std::string(program_info_json)).value(); + auto output = std::make_unique<::concretelang::clientlib::ClientProgram>(::concretelang::clientlib::ClientProgram::createEncrypted( + info, client_keyset, std::move(csprng)) + .value()); + return std::unique_ptr( + reinterpret_cast(output.release())); +} + +std::unique_ptr +_client_module_new_simulated(rust::Str program_info_json, + std::unique_ptr csprng) { + auto info = Message(); + info.readJsonFromString(std::string(program_info_json)).value(); + auto output = std::make_unique<::concretelang::clientlib::ClientProgram>(::concretelang::clientlib::ClientProgram::createSimulated( + info, std::move(csprng)) + .value()); + return std::unique_ptr( + reinterpret_cast(output.release())); +} + +struct ServerFunction : concretelang::serverlib::ServerCircuit { + std::unique_ptr> + _call(const ServerKeyset &keys, + rust::Slice> args) { + std::vector oargs{}; + for (size_t i = 0; i < args.length(); i++) { + oargs.push_back(*args[i].release()); + } + auto res = std::make_unique>(call(keys, oargs).value()); + return std::unique_ptr>( + reinterpret_cast *>(res.release())); + } + + std::unique_ptr> + _simulate(rust::Slice> args) { + std::vector oargs{}; + for (size_t i = 0; i < args.length(); i++) { + oargs.push_back(*args[i].release()); + } + auto res = std::make_unique>(simulate(oargs).value()); + return std::unique_ptr>( + reinterpret_cast *>(res.release())); + } +}; + +typedef void (*FnPtr)(void *, ...); +using c_void = void; + +std::unique_ptr _server_function_new(rust::Str circuit_info_json, + void *func, + bool use_simulation) { + auto info = Message(); + info.readJsonFromString(std::string(circuit_info_json)).value(); + FnPtr fn_ptr = reinterpret_cast(func); + auto output = concretelang::serverlib::ServerCircuit::fromFnPtr( + info, fn_ptr, use_simulation) + .value(); + return std::make_unique( + *reinterpret_cast(&output)); +} + +} // namespace concrete_rust + +#endif diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/src/ffi.rs b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/src/ffi.rs new file mode 100644 index 000000000..23d6f3bc7 --- /dev/null +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/src/ffi.rs @@ -0,0 +1,940 @@ +#![allow(unused_imports,unused)] +use std::{any::Any, marker::PhantomData, pin::Pin}; + +use crate::protocol::{ + KeysetInfo, LweBootstrapKeyInfo, LweKeyswitchKeyInfo, LweSecretKeyInfo, + PackingKeyswitchKeyInfo, ProgramInfo, CircuitInfo +}; +use cxx::{SharedPtr, UniquePtr, CxxVector}; + +#[cxx::bridge(namespace = "concrete_rust")] +mod ffi { + + + unsafe extern "C++" { + include!("ffi.h"); + type c_void; + + // ------------------------------------------------------------------------------------------- Compiler + + /// Holds different flags and options of the compilation process. + type CompilationOptions; + + #[doc(hidden)] + fn _compilation_options_new() -> UniquePtr; + + /// Set display flag of optimizer choices. + fn set_display_optimizer_choice(self: Pin<&mut CompilationOptions>, val: bool); + + /// Set option for loop parallelization. + fn set_loop_parallelize(self: Pin<&mut CompilationOptions>, val: bool); + + /// Set option for dataflow parallelization. + fn set_dataflow_parallelize(self: Pin<&mut CompilationOptions>, val: bool); + + /// Set option for auto parallelization. + fn set_auto_parallelize(self: Pin<&mut CompilationOptions>, val: bool); + + /// Set option for compression of evaluation keys. + fn set_compress_evaluation_keys(self: Pin<&mut CompilationOptions>, val: bool); + + /// Set option for compression of input ciphertexts. + fn set_compress_input_ciphertexts(self: Pin<&mut CompilationOptions>, val: bool); + + /// Set error probability for shared by each pbs. + fn set_p_error(self: Pin<&mut CompilationOptions>, val: f64); + + /// Set global error probability for the full circuit. + fn set_global_p_error(self: Pin<&mut CompilationOptions>, val: f64); + + /// Set the strategy of the optimizer. + fn set_optimizer_strategy(self: Pin<&mut CompilationOptions>, val: u8); + + /// Set the strategy of the optimizer for multi-parameter. + fn set_optimizer_multi_parameter_strategy(self: Pin<&mut CompilationOptions>, val: u8); + + /// Enable or disable tlu fusing. + fn set_enable_tlu_fusing(self: Pin<&mut CompilationOptions>, val: bool); + + /// Enable or disable printing tlu fusing. + fn set_print_tlu_fusing(self: Pin<&mut CompilationOptions>, val: bool); + + /// Enable or disable simulation. + fn set_simulate(self: Pin<&mut CompilationOptions>, val: bool); + + /// Enable or disable overflow detection during simulation. + fn set_enable_overflow_detection_in_simulation( + self: Pin<&mut CompilationOptions>, + val: bool, + ); + + /// Set composable flag. + fn set_composable(self: Pin<&mut CompilationOptions>, val: bool); + + /// Add a range restriction. + fn set_range_restriction(self: Pin<&mut CompilationOptions>, json: &str); + + /// Add a keyset restriction. + fn set_keyset_restriction(self: Pin<&mut CompilationOptions>, json: &str); + + /// Set security level. + fn set_security_level(self: Pin<&mut CompilationOptions>, val: u64); + + /// Add a composition rule. + fn add_composition_rule( + self: Pin<&mut CompilationOptions>, + from_func: &str, + from_pos: usize, + to_func: &str, + to_pos: usize, + ); + + /// Library object representing the output of a compilation. + type Library; + /// Return the path to the static library. + fn get_static_library_path(self: &Library) -> String; + #[doc(hidden)] + fn _get_program_info_json(self: &Library) -> String; + + /// Compile the `mlir` source string to a static library available at `output_dir_path` using the `options`. + fn compile( + sources: &str, + options: &CompilationOptions, + output_dir_path: &str, + ) -> Result>; + + // ------------------------------------------------------------------------------------------- Commons + + /// A CSPRNG used to generate keys. + type SecretCsprng; + #[doc(hidden)] + fn _secret_csprng_new(high: u64, low: u64) -> UniquePtr; + + /// A CSPRNG used to encrypt ciphertexts. + type EncryptionCsprng; + #[doc(hidden)] + fn _encryption_csprng_new(high: u64, low: u64) -> UniquePtr; + + /// Represents an LWE Bootstrap Key. + type LweBootstrapKey; + /// Get the buffer of the LWE Bootstrap Key. + fn get_buffer(self: Pin<&mut LweBootstrapKey>) -> &[u64]; + #[doc(hidden)] + fn _get_info_json(self: &LweBootstrapKey) -> String; + + /// Represents an LWE Keyswitch Key. + type LweKeyswitchKey; + /// Get the buffer of the LWE Keyswitch Key. + fn get_buffer(self: Pin<&mut LweKeyswitchKey>) -> &[u64]; + #[doc(hidden)] + fn _get_info_json(self: &LweKeyswitchKey) -> String; + + /// Represents a Packing Keyswitch Key. + type PackingKeyswitchKey; + /// Get the buffer of the Packing Keyswitch Key. + fn get_buffer(self: Pin<&mut PackingKeyswitchKey>) -> &[u64]; + #[doc(hidden)] + fn _get_info_json(self: &PackingKeyswitchKey) -> String; + + /// Represents an LWE Secret Key. + type LweSecretKey; + /// Get the buffer of the LWE Secret Key. + fn get_buffer(self: Pin<&mut LweSecretKey>) -> &[u64]; + #[doc(hidden)] + fn _get_info_json(self: &LweSecretKey) -> String; + + /// A Keyset object holding both the [`ClientKeyset`] and the [`ServerKeyset`]. + type Keyset; + #[doc(hidden)] + fn _keyset_new( + keyset_info_json: &str, + secret_csprng: Pin<&mut SecretCsprng>, + encryption_csprng: Pin<&mut EncryptionCsprng>, + ) -> UniquePtr; + /// Return the associated server keyset. + fn get_server(self: &Keyset) -> UniquePtr; + /// Return the associated client keyset. + fn get_client(self: &Keyset) -> UniquePtr; + + /// A server keyset holding the keys necessary to __execute__ an FHE program on encrypted data. + /// + /// Note: + /// ----- + /// This is public material, and as such can be sent safely to the execution platform. + type ServerKeyset; + #[doc(hidden)] + fn _lwe_bootstrap_keys_len(self: &ServerKeyset) -> usize; + #[doc(hidden)] + fn _lwe_bootstrap_keys_nth(self: &ServerKeyset, nth: usize) -> &LweBootstrapKey; + #[doc(hidden)] + fn _lwe_keyswitch_keys_len(self: &ServerKeyset) -> usize; + #[doc(hidden)] + fn _lwe_keyswitch_keys_nth(self: &ServerKeyset, nth: usize) -> &LweKeyswitchKey; + #[doc(hidden)] + fn _packing_keyswitch_keys_len(self: &ServerKeyset) -> usize; + #[doc(hidden)] + fn _packing_keyswitch_keys_nth(self: &ServerKeyset, nth: usize) -> &PackingKeyswitchKey; + /// Serialize a server keyset to bytes. + fn serialize(self: &ServerKeyset) -> Vec; + #[doc(hidden)] + fn _deserialize_server_keyset(bytes: &[u8]) -> UniquePtr; + + + /// A client keyset holding the keys necessary to __encrypt__ input data (and decrypt outputs). + /// + /// Warning: + /// -------- + /// This is private material, and as such SHOULD NOT BE SHARED with untrusted third-party. + type ClientKeyset; + #[doc(hidden)] + fn _lwe_secret_keys_len(self: &ClientKeyset) -> usize; + #[doc(hidden)] + fn _lwe_secret_keys_nth(self: &ClientKeyset, nth: usize) -> &LweSecretKey; + /// Serialize a client keyset to bytes. + fn serialize(self: &ClientKeyset) -> Vec; + #[doc(hidden)] + fn _deserialize_client_keyset(bytes: &[u8]) -> UniquePtr; + + #[doc(hidden)] + type TensorU8; + #[doc(hidden)] + fn _tensor_u8_new(values: &[u8], dimensions: &[usize]) -> UniquePtr; + #[doc(hidden)] + fn _get_dimensions(self: &TensorU8) -> &[usize]; + #[doc(hidden)] + fn _get_values(self: &TensorU8) -> &[u8]; + + #[doc(hidden)] + type TensorU16; + #[doc(hidden)] + fn _tensor_u16_new(values: &[u16], dimensions: &[usize]) -> UniquePtr; + #[doc(hidden)] + fn _get_dimensions(self: &TensorU16) -> &[usize]; + #[doc(hidden)] + fn _get_values(self: &TensorU16) -> &[u16]; + + #[doc(hidden)] + type TensorU32; + #[doc(hidden)] + fn _tensor_u32_new(values: &[u32], dimensions: &[usize]) -> UniquePtr; + #[doc(hidden)] + fn _get_dimensions(self: &TensorU32) -> &[usize]; + #[doc(hidden)] + fn _get_values(self: &TensorU32) -> &[u32]; + + #[doc(hidden)] + type TensorU64; + #[doc(hidden)] + fn _tensor_u64_new(values: &[u64], dimensions: &[usize]) -> UniquePtr; + #[doc(hidden)] + fn _get_dimensions(self: &TensorU64) -> &[usize]; + #[doc(hidden)] + fn _get_values(self: &TensorU64) -> &[u64]; + + #[doc(hidden)] + type TensorI8; + #[doc(hidden)] + fn _tensor_i8_new(values: &[i8], dimensions: &[usize]) -> UniquePtr; + #[doc(hidden)] + fn _get_dimensions(self: &TensorI8) -> &[usize]; + #[doc(hidden)] + fn _get_values(self: &TensorI8) -> &[i8]; + + #[doc(hidden)] + type TensorI16; + #[doc(hidden)] + fn _tensor_i16_new(values: &[i16], dimensions: &[usize]) -> UniquePtr; + #[doc(hidden)] + fn _get_dimensions(self: &TensorI16) -> &[usize]; + #[doc(hidden)] + fn _get_values(self: &TensorI16) -> &[i16]; + + #[doc(hidden)] + type TensorI32; + #[doc(hidden)] + fn _tensor_i32_new(values: &[i32], dimensions: &[usize]) -> UniquePtr; + #[doc(hidden)] + fn _get_dimensions(self: &TensorI32) -> &[usize]; + #[doc(hidden)] + fn _get_values(self: &TensorI32) -> &[i32]; + + #[doc(hidden)] + type TensorI64; + #[doc(hidden)] + fn _tensor_i64_new(values: &[i64], dimensions: &[usize]) -> UniquePtr; + #[doc(hidden)] + fn _get_dimensions(self: &TensorI64) -> &[usize]; + #[doc(hidden)] + fn _get_values(self: &TensorI64) -> &[i64]; + + /// A generic value type carrying a `Tensor` of elements `(u|i)(8|16|32|64)`. + type Value; + #[doc(hidden)] + fn _value_from_tensor_u8(tensor: UniquePtr) -> UniquePtr; + #[doc(hidden)] + fn _value_from_tensor_u16(tensor: UniquePtr) -> UniquePtr; + #[doc(hidden)] + fn _value_from_tensor_u32(tensor: UniquePtr) -> UniquePtr; + #[doc(hidden)] + fn _value_from_tensor_u64(tensor: UniquePtr) -> UniquePtr; + #[doc(hidden)] + fn _value_from_tensor_i8(tensor: UniquePtr) -> UniquePtr; + #[doc(hidden)] + fn _value_from_tensor_i16(tensor: UniquePtr) -> UniquePtr; + #[doc(hidden)] + fn _value_from_tensor_i32(tensor: UniquePtr) -> UniquePtr; + #[doc(hidden)] + fn _value_from_tensor_i64(tensor: UniquePtr) -> UniquePtr; + #[doc(hidden)] + fn _has_element_type_u8(self: &Value) -> bool; + #[doc(hidden)] + fn _has_element_type_u16(self: &Value) -> bool; + #[doc(hidden)] + fn _has_element_type_u32(self: &Value) -> bool; + #[doc(hidden)] + fn _has_element_type_u64(self: &Value) -> bool; + #[doc(hidden)] + fn _has_element_type_i8(self: &Value) -> bool; + #[doc(hidden)] + fn _has_element_type_i16(self: &Value) -> bool; + #[doc(hidden)] + fn _has_element_type_i32(self: &Value) -> bool; + #[doc(hidden)] + fn _has_element_type_i64(self: &Value) -> bool; + #[doc(hidden)] + fn _get_tensor_u8(self: &Value) -> UniquePtr; + #[doc(hidden)] + fn _get_tensor_u16(self: &Value) -> UniquePtr; + #[doc(hidden)] + fn _get_tensor_u32(self: &Value) -> UniquePtr; + #[doc(hidden)] + fn _get_tensor_u64(self: &Value) -> UniquePtr; + #[doc(hidden)] + fn _get_tensor_i8(self: &Value) -> UniquePtr; + #[doc(hidden)] + fn _get_tensor_i16(self: &Value) -> UniquePtr; + #[doc(hidden)] + fn _get_tensor_i32(self: &Value) -> UniquePtr; + #[doc(hidden)] + fn _get_tensor_i64(self: &Value) -> UniquePtr; + fn get_dimensions(self: &Value) -> &[usize]; + + /// A serialized value which can be transported between the server and the client. + type TransportValue; + /// Build an owned copy from a reference. + fn to_owned(self: &TransportValue) -> UniquePtr; + /// Serialize a transport value to bytes. + fn serialize(self: &TransportValue) -> Vec; + #[doc(hidden)] + fn _deserialize_transport_value(bytes: &[u8]) -> UniquePtr; + + // ------------------------------------------------------------------------------------------- Client + + /// Client-side interface to an FHE module. + type ClientModule; + #[doc(hidden)] + fn _client_module_new_encrypted( + program_info_json: &str, + client_keyset: &ClientKeyset, + encryption_prng: UniquePtr, + ) -> UniquePtr; + #[doc(hidden)] + fn _client_module_new_simulated( + program_info_json: &str, + encryption_prng: UniquePtr, + ) -> UniquePtr; + #[doc(hidden)] + fn _get_client_function(self: &ClientModule, name: &str) -> Result>; + + /// Client-side interface to an FHE function. + /// + /// Note : + /// ------ + /// Among other things, this object is responsible for encrypting the function inputs before they can be sent to the server. + type ClientFunction; + #[doc(hidden)] + fn _client_function_new_encrypted( + circuit_info_json: &str, + client_keyset: &ClientKeyset, + encryption_prng: UniquePtr + ) -> UniquePtr; + #[doc(hidden)] + fn _client_function_new_simulated( + circuit_info_json: &str, + encryption_prng: UniquePtr + ) -> UniquePtr; + /// Prepare one function input to be sent to the server. + /// + /// Note: + /// ----- + /// This include encoding -> encryption -> conversion to serializable value. + fn prepare_input(self: Pin<&mut ClientFunction>, arg: UniquePtr, pos: usize) -> UniquePtr; + /// Process one function output retrieved from the server. + /// + /// Note: + /// ----- + /// This include conversion from deserializable value -> decryption -> decoding. + fn process_output(self: Pin<&mut ClientFunction>,result: UniquePtr, pos: usize) -> UniquePtr; + #[doc(hidden)] + fn simulate_prepare_input(self: Pin<&mut ClientFunction>,arg: &Value, pos: usize) -> UniquePtr; + #[doc(hidden)] + fn simulate_process_output(self: Pin<&mut ClientFunction>,result: &TransportValue, pos: usize) -> UniquePtr; + + // ------------------------------------------------------------------------------------------- Server + // + /// Server-side interface to an FHE function. + /// + /// Note: + /// ----- + /// This object allows to invoke the FHE function on the encrypted inputs coming from the client. + type ServerFunction; + #[doc(hidden)] + unsafe fn _server_function_new(circuit_info_json: &str, func: *mut c_void, use_simulation: bool) -> UniquePtr; + #[doc(hidden)] + fn _call(self: Pin<&mut ServerFunction>, keys: &ServerKeyset, args: &mut [UniquePtr]) -> UniquePtr>; + #[doc(hidden)] + fn _simulate(self: Pin<&mut ServerFunction>, args: &mut [UniquePtr]) -> UniquePtr>; + } +} + +pub use ffi::*; + +impl ServerKeyset{ + /// Deserialize a server keyset from bytes. + pub fn deserialize(bytes: &[u8]) -> UniquePtr { + _deserialize_server_keyset(bytes) + } +} + +impl ClientKeyset{ + /// Deserialize a client keyset from bytes. + pub fn deserialize(bytes: &[u8]) -> UniquePtr { + _deserialize_client_keyset(bytes) + } +} + +impl TransportValue{ + /// Deserialize a `TransportValue` from bytes. + pub fn deserialize(bytes: &[u8]) -> UniquePtr { + _deserialize_transport_value(bytes) + } +} + +impl ServerFunction { + + #[doc(hidden)] + pub fn new(circuit_info: &CircuitInfo, func: *mut c_void, use_simulation: bool) -> UniquePtr { + unsafe{ + _server_function_new( + &serde_json::to_string(circuit_info).unwrap(), + func, + use_simulation + ) + } + } + + /// Performs a call to the FHE function using the `keys` server keyset and the `args` arguments. + pub fn call(self: Pin<&mut ServerFunction>, keys: &ServerKeyset, mut args: Vec>) -> Vec>{ + let output = self._call(keys, args.as_mut_slice()); + output.iter().map(|v| v.to_owned()).collect() + } + + #[doc(hidden)] + pub fn simulate(self: Pin<&mut ServerFunction>, mut args: Vec>) -> Vec>{ + let output = self._simulate(args.as_mut_slice()); + output.iter().map(|v| v.to_owned()).collect() + } +} + +pub trait FromElements { + type Element; + fn from_elements(elements: Vec, dimensions: Vec) -> Self; +} + +impl FromElements for Tensor { + type Element = u8; + fn from_elements(values: Vec, dimensions: Vec) -> Tensor { + Tensor { + inner: InnerTensor::U8(_tensor_u8_new(&values, &dimensions)), + phantom: PhantomData, + } + } +} + +impl FromElements for Tensor { + type Element = u16; + fn from_elements(values: Vec, dimensions: Vec) -> Tensor { + Tensor { + inner: InnerTensor::U16(_tensor_u16_new(&values, &dimensions)), + phantom: PhantomData, + } + } +} + +impl FromElements for Tensor { + type Element = u32; + fn from_elements(values: Vec, dimensions: Vec) -> Tensor { + Tensor { + inner: InnerTensor::U32(_tensor_u32_new(&values, &dimensions)), + phantom: PhantomData, + } + } +} + +impl FromElements for Tensor { + type Element = u64; + fn from_elements(values: Vec, dimensions: Vec) -> Tensor { + Tensor { + inner: InnerTensor::U64(_tensor_u64_new(&values, &dimensions)), + phantom: PhantomData, + } + } +} + +impl FromElements for Tensor { + type Element = i8; + fn from_elements(values: Vec, dimensions: Vec) -> Tensor { + Tensor { + inner: InnerTensor::I8(_tensor_i8_new(&values, &dimensions)), + phantom: PhantomData, + } + } +} + +impl FromElements for Tensor { + type Element = i16; + fn from_elements(values: Vec, dimensions: Vec) -> Tensor { + Tensor { + inner: InnerTensor::I16(_tensor_i16_new(&values, &dimensions)), + phantom: PhantomData, + } + } +} + +impl FromElements for Tensor { + type Element = i32; + fn from_elements(values: Vec, dimensions: Vec) -> Tensor { + Tensor { + inner: InnerTensor::I32(_tensor_i32_new(&values, &dimensions)), + phantom: PhantomData, + } + } +} + +impl FromElements for Tensor { + type Element = i64; + fn from_elements(values: Vec, dimensions: Vec) -> Tensor { + Tensor { + inner: InnerTensor::I64(_tensor_i64_new(&values, &dimensions)), + phantom: PhantomData, + } + } +} + +pub trait GetValues { + type Element; + fn get_values(&self) -> &[Self::Element]; +} + +impl GetValues for Tensor { + type Element = u8; + fn get_values(&self) -> &[Self::Element] { + let InnerTensor::U8(inner) = &self.inner else {unreachable!()}; + inner._get_values() + } +} + +impl GetValues for Tensor { + type Element = u16; + fn get_values(&self) -> &[Self::Element] { + let InnerTensor::U16(inner) = &self.inner else {unreachable!()}; + inner._get_values() + } +} + +impl GetValues for Tensor { + type Element = u32; + fn get_values(&self) -> &[Self::Element] { + let InnerTensor::U32(inner) = &self.inner else {unreachable!()}; + inner._get_values() + } +} + +impl GetValues for Tensor { + type Element = u64; + fn get_values(&self) -> &[Self::Element] { + let InnerTensor::U64(inner) = &self.inner else {unreachable!()}; + inner._get_values() + } +} + +impl GetValues for Tensor { + type Element = i8; + fn get_values(&self) -> &[Self::Element] { + let InnerTensor::I8(inner) = &self.inner else {unreachable!()}; + inner._get_values() + } +} + +impl GetValues for Tensor { + type Element = i16; + fn get_values(&self) -> &[Self::Element] { + let InnerTensor::I16(inner) = &self.inner else {unreachable!()}; + inner._get_values() + } +} + +impl GetValues for Tensor { + type Element = i32; + fn get_values(&self) -> &[Self::Element] { + let InnerTensor::I32(inner) = &self.inner else {unreachable!()}; + inner._get_values() + } +} + +impl GetValues for Tensor { + type Element = i64; + fn get_values(&self) -> &[Self::Element] { + let InnerTensor::I64(inner) = &self.inner else {unreachable!()}; + inner._get_values() + } +} + +/// A generic tensor type. +pub struct Tensor{ + inner: InnerTensor, + phantom: PhantomData +} + +impl Tensor { + /// Create a new tensor from values and dimensions (shape). + pub fn new(values: Vec, dimensions: Vec) -> Tensor where Tensor:FromElements { + Self::from_elements(values, dimensions) + } + + /// Return the dimensions of the tensor. + pub fn dimensions(&self) -> &[usize] { + match self.inner{ + InnerTensor::U8(ref unique_ptr) => unique_ptr._get_dimensions(), + InnerTensor::U16(ref unique_ptr) => unique_ptr._get_dimensions(), + InnerTensor::U32(ref unique_ptr) => unique_ptr._get_dimensions(), + InnerTensor::U64(ref unique_ptr) => unique_ptr._get_dimensions(), + InnerTensor::I8(ref unique_ptr) => unique_ptr._get_dimensions(), + InnerTensor::I16(ref unique_ptr) => unique_ptr._get_dimensions(), + InnerTensor::I32(ref unique_ptr) => unique_ptr._get_dimensions(), + InnerTensor::I64(ref unique_ptr) => unique_ptr._get_dimensions(), + } + } + + /// Return the values of the tensor. + pub fn values(&self) -> &[T] where Tensor:GetValues { + self.get_values() + } +} + +enum InnerTensor { + U8(UniquePtr), + U16(UniquePtr), + U32(UniquePtr), + U64(UniquePtr), + I8(UniquePtr), + I16(UniquePtr), + I32(UniquePtr), + I64(UniquePtr), +} + +pub trait GetTensor { + fn _get_tensor(&self) -> Option>; +} + +impl GetTensor for Value { + fn _get_tensor(&self) -> Option> { + self._has_element_type_u8().then(|| { + Tensor { + inner: InnerTensor::U8(self._get_tensor_u8()), + phantom: PhantomData, + } + }) + } +} + +impl GetTensor for Value { + fn _get_tensor(&self) -> Option> { + self._has_element_type_u16().then(|| { + Tensor { + inner: InnerTensor::U16(self._get_tensor_u16()), + phantom: PhantomData, + } + }) + } +} + +impl GetTensor for Value { + fn _get_tensor(&self) -> Option> { + self._has_element_type_u32().then(|| { + Tensor { + inner: InnerTensor::U32(self._get_tensor_u32()), + phantom: PhantomData, + } + }) + } +} + +impl GetTensor for Value { + fn _get_tensor(&self) -> Option> { + self._has_element_type_u64().then(|| { + Tensor { + inner: InnerTensor::U64(self._get_tensor_u64()), + phantom: PhantomData, + } + }) + } +} + +impl GetTensor for Value { + fn _get_tensor(&self) -> Option> { + self._has_element_type_i8().then(|| { + Tensor { + inner: InnerTensor::I8(self._get_tensor_i8()), + phantom: PhantomData, + } + }) + } +} + +impl GetTensor for Value { + fn _get_tensor(&self) -> Option> { + self._has_element_type_i16().then(|| { + Tensor { + inner: InnerTensor::I16(self._get_tensor_i16()), + phantom: PhantomData, + } + }) + } +} + +impl GetTensor for Value { + fn _get_tensor(&self) -> Option> { + self._has_element_type_i32().then(|| { + Tensor { + inner: InnerTensor::I32(self._get_tensor_i32()), + phantom: PhantomData, + } + }) + } +} + +impl GetTensor for Value { + fn _get_tensor(&self) -> Option> { + self._has_element_type_i64().then(|| { + Tensor { + inner: InnerTensor::I64(self._get_tensor_i64()), + phantom: PhantomData, + } + }) + } +} + +impl Value{ + /// Create a `Value` from a `Tensor`. + pub fn from_tensor(input: Tensor) -> UniquePtr { + match input.inner{ + InnerTensor::U8(unique_ptr) => _value_from_tensor_u8(unique_ptr), + InnerTensor::U16(unique_ptr) => _value_from_tensor_u16(unique_ptr), + InnerTensor::U32(unique_ptr) => _value_from_tensor_u32(unique_ptr), + InnerTensor::U64(unique_ptr) => _value_from_tensor_u64(unique_ptr), + InnerTensor::I8(unique_ptr) => _value_from_tensor_i8(unique_ptr), + InnerTensor::I16(unique_ptr) => _value_from_tensor_i16(unique_ptr), + InnerTensor::I32(unique_ptr) => _value_from_tensor_i32(unique_ptr), + InnerTensor::I64(unique_ptr) => _value_from_tensor_i64(unique_ptr), + } + } + + /// Unwrap the value to a tensor of the given type (if it indeed holds a tensor with elements of this type). + pub fn get_tensor(&self) -> Option> where Self: GetTensor{ + self._get_tensor() + } +} + +impl ClientModule { + /// Create a new client module. + pub fn new_encrypted(program_info: &ProgramInfo, client_keyset: &ClientKeyset, csprng: UniquePtr) -> UniquePtr { + _client_module_new_encrypted( + &serde_json::to_string(program_info).unwrap(), + client_keyset, + csprng, + ) + } + + #[doc(hidden)] + pub fn new_simulated(program_info: &ProgramInfo, csprng: UniquePtr) -> UniquePtr { + _client_module_new_simulated( + &serde_json::to_string(program_info).unwrap(), + csprng, + ) + } +} + +impl ClientFunction { + /// Create a new client function. + pub fn new_encrypted(circuit_info: &CircuitInfo, client_keyset: &ClientKeyset, csprng: UniquePtr) -> UniquePtr { + _client_function_new_encrypted( + &serde_json::to_string(circuit_info).unwrap(), + client_keyset, + csprng, + ) + } + + #[doc(hidden)] + pub fn new_simulated(circuit_info: &CircuitInfo, csprng: UniquePtr) -> UniquePtr { + _client_function_new_simulated( + &serde_json::to_string(circuit_info).unwrap(), + csprng, + ) + } +} + +impl ServerKeyset { + + /// Return references to the lwe bootstrap keys of this server keyset. + pub fn lwe_bootstrap_keys(&self) -> Vec<&LweBootstrapKey> { + (0..self._lwe_bootstrap_keys_len()) + .map(|i| self._lwe_bootstrap_keys_nth(i)) + .collect() + } + + /// Return references to the lwe kewysitch keys of this server keyset. + pub fn lwe_keyswitch_keys(&self) -> Vec<&LweKeyswitchKey> { + (0..self._lwe_keyswitch_keys_len()) + .map(|i| self._lwe_keyswitch_keys_nth(i)) + .collect() + } + + /// Return references to the packing keyswitch keys of this server keyset. + pub fn packing_keyswitch_keys(&self) -> Vec<&PackingKeyswitchKey> { + (0..self._packing_keyswitch_keys_len()) + .map(|i| self._packing_keyswitch_keys_nth(i)) + .collect() + } +} + +impl ClientKeyset { + /// Return references to the lwe secret keys of this client keyset. + pub fn lwe_secret_keys(&self) -> Vec<&LweSecretKey> { + (0..self._lwe_secret_keys_len()) + .map(|i| self._lwe_secret_keys_nth(i)) + .collect() + } +} + +impl Keyset { + /// Generate a new keyset. + pub fn new( + keyset_info: &KeysetInfo, + secret_csprng: Pin<&mut SecretCsprng>, + encryption_csprng: Pin<&mut EncryptionCsprng>, + ) -> UniquePtr { + _keyset_new( + &serde_json::to_string(keyset_info).unwrap(), + secret_csprng, + encryption_csprng, + ) + } +} + +impl SecretCsprng { + /// Create a new secret CSPRNG. + pub fn new(seed: u128) -> UniquePtr { + let words: [u64; 2] = unsafe { std::mem::transmute::(seed) }; + _secret_csprng_new(words[0], words[1]) + } +} + +impl EncryptionCsprng { + /// Create a new encryption CSPRNG. + pub fn new(seed: u128) -> UniquePtr { + let words: [u64; 2] = unsafe { std::mem::transmute::(seed) }; + _encryption_csprng_new(words[0], words[1]) + } +} + +impl CompilationOptions { + /// Create a new set of compilation options. + pub fn new() -> UniquePtr { + _compilation_options_new() + } +} + +impl Library { + /// Return the associated program info. + pub fn get_program_info(&self) -> ProgramInfo { + serde_json::from_str(&self._get_program_info_json()).unwrap() + } +} + +impl LweSecretKey { + /// Return the associated info. + pub fn get_info(&self) -> LweSecretKeyInfo { + serde_json::from_str(&self._get_info_json()).unwrap() + } +} + +impl LweKeyswitchKey { + /// Return the associated info. + pub fn get_info(&self) -> LweKeyswitchKeyInfo { + serde_json::from_str(&self._get_info_json()).unwrap() + } +} + +impl PackingKeyswitchKey { + /// Return the associated info. + pub fn get_info(&self) -> PackingKeyswitchKeyInfo { + serde_json::from_str(&self._get_info_json()).unwrap() + } +} + +impl LweBootstrapKey { + /// Return the associated info. + pub fn get_info(&self) -> LweBootstrapKeyInfo { + serde_json::from_str(&self._get_info_json()).unwrap() + } +} + +impl std::fmt::Debug for TransportValue{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "TransportValue") + } +} + +#[cfg(test)] +mod test { + use super::*; + + const TEST_FOLDER: &str = "/tmp/test_concrete"; + + #[test] + fn test_compile() { + let sources = " + func.func @dec(%arg0: !FHE.eint<3>) -> !FHE.eint<3> { + %cst_1 = arith.constant 1 : i4 + %1 = \"FHE.sub_eint_int\"(%arg0, %cst_1) : (!FHE.eint<3>, i4) -> !FHE.eint<3> + return %1: !FHE.eint<3> + } + "; + let options = _compilation_options_new(); + let _ = std::fs::remove_dir(TEST_FOLDER); + let _ = std::fs::create_dir_all(TEST_FOLDER); + let mut output = compile(sources, &options, TEST_FOLDER).unwrap(); + let static_library_path = output.as_mut().unwrap().get_static_library_path(); + let mut archive = ar::Archive::new(std::fs::File::open(static_library_path).unwrap()); + let symbols = archive + .symbols() + .expect("failed to parse symbols") + .map(|a| String::from_utf8_lossy(a).to_string()) + .collect::>(); + #[cfg(target_os = "macos")] + assert!(symbols.contains(&"_concrete_dec".to_string())); + #[cfg(target_os = "linux")] + assert!(symbols.contains(&"concrete_dec".to_string())); + } +} diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/src/lib.rs b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/src/lib.rs new file mode 100644 index 000000000..8f74414de --- /dev/null +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/src/lib.rs @@ -0,0 +1,27 @@ +pub use cxx::{UniquePtr, SharedPtr}; +pub use ffi::c_void; + +mod ffi; + +#[cfg(feature = "compiler")] +#[doc(hidden)] +pub mod compiler { + pub use crate::ffi::{compile, CompilationOptions, Library}; +} + +pub mod common { + pub use crate::ffi::{ + ClientKeyset, EncryptionCsprng, Keyset, LweBootstrapKey, LweKeyswitchKey, LweSecretKey, + PackingKeyswitchKey, SecretCsprng, ServerKeyset, Tensor, TransportValue, Value, + }; +} + +pub mod client { + pub use crate::ffi::{ClientFunction, ClientModule}; +} + +pub mod server { + pub use crate::ffi::ServerFunction; +} + +pub mod protocol; diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/src/protocol.rs b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/src/protocol.rs new file mode 100644 index 000000000..145f8bffa --- /dev/null +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/src/protocol.rs @@ -0,0 +1,977 @@ +#![allow(non_camel_case_types, non_snake_case, unused)] +use serde::{Deserialize, Serialize}; + +/// A complete program can be described by the ensemble of circuit signatures, and the description +/// of the keyset that go with it. This structure regroup those informations. +#[derive(Debug, Deserialize, Serialize)] +pub struct ProgramInfo { + /// The informations on the keyset of the program. + pub keyset: KeysetInfo, + /// The informations for the different circuits of the program. + pub circuits: Vec, +} + +/// A circuit signature can be described completely by the type informations for its input and +/// outputs, as well as its name. This structure regroup those informations. +/// +/// Note: +/// The order of the input and output lists matters. The order of values should be the same when +/// executing the circuit. Also, the name is expected to be unique in the program. +#[derive(Debug, Deserialize, Serialize)] +pub struct CircuitInfo { + /// The ordered list of input types. + pub inputs: Vec, + /// The ordered list of output types. + pub outputs: Vec, + /// The name of the circuit. + pub name: String, +} + +/// A value flowing in or out of a circuit is expected to be of a given type, according to the +/// signature of this circuit. This structure represents such a type in a circuit signature. +#[derive(Debug, Deserialize, Serialize)] +pub struct GateInfo { + /// The raw information that raw data must be possible to parse with. + pub rawInfo: RawInfo, + /// The type of the value expected at the gate. + pub typeInfo: TypeInfo, +} + +/// A value exchanged at the boundary between two parties of a computation will be transmitted as a +/// binary payload containing a tensor of integers. This payload will first have to be parsed to a +/// tensor of proper shape, signedness and precision before being pre-processed and passed to the +/// computation. This structure represents the informations needed to parse this payload into the +/// expected tensor. +#[derive(Debug, Deserialize, Serialize)] +pub struct RawInfo { + /// The shape of the tensor. + pub shape: Shape, + /// The precision of the integers. + pub integerPrecision: u32, + /// The signedness of the integers. + pub isSigned: bool, +} + +/// Scalar and tensor values are represented by the same types. This structure contains a +/// description of the shape of value. +/// +/// Note: +/// If the dimensions vector is empty, the message is interpreted as a scalar. +#[derive(Debug, Deserialize, Serialize)] +pub struct Shape { + /// The dimensions of the value. + pub dimensions: Vec, +} + +/// The different possible type of values. +#[derive(Debug, Deserialize, Serialize)] +pub enum TypeInfo { + lweCiphertext(LweCiphertextTypeInfo), + plaintext(PlaintextTypeInfo), + index(IndexTypeInfo), +} + +/// A plaintext value can flow in and out of a circuit. This structure represents the informations +/// needed to verify and pre-or-post process this value. +#[derive(Debug, Deserialize, Serialize)] +pub struct PlaintextTypeInfo { + /// The shape of the value. + pub shape: Shape, + /// The precision of the integers. + pub integerPrecision: u32, + /// The signedness of the integers. + pub isSigned: bool, +} + +/// A plaintext value can flow in and out of a circuit. This structure represents the informations +/// needed to verify and pre-or-post process this value. +#[derive(Debug, Deserialize, Serialize)] +pub struct IndexTypeInfo { + /// The shape of the value. + pub shape: Shape, + /// The precision of the indexes. + pub integerPrecision: u32, + /// The signedness of the indexes. + pub isSigned: bool, +} + +/// A ciphertext value can flow in and out of a circuit. This structure represents the informations +/// needed to verify and pre-or-post process this value. +/// +/// Note: +/// Two shape information are carried in this type. The abstract shape is the shape the tensor +/// would have if the values were cleartext. That is, it does not take into account the encryption +/// process. The concrete shape is the final shape of the object accounting for the encryption, +/// that usually add one or more dimension to the object. +#[derive(Debug, Deserialize, Serialize)] +pub struct LweCiphertextTypeInfo { + /// The abstract shape of the value. + pub abstractShape: Shape, + /// The concrete shape of the value. + pub concreteShape: Shape, + /// The precision of the integers used for storage. + pub integerPrecision: u32, + /// The informations relative to the encryption. + pub encryption: LweCiphertextEncryptionInfo, + /// The compression used for this value. + pub compression: Compression, + /// The encoding of the value stored inside the ciphertext. + pub encoding: LweCiphretextTypeInfo_Encoding, +} + +/// The encryption of a cleartext value requires some parameters to operate. This structure +/// represents those parameters. +#[derive(Debug, Deserialize, Serialize)] +pub struct LweCiphertextEncryptionInfo { + /// The identifier of the secret key used to perform the encryption. + pub keyId: u32, + /// The variance of the noise injected during encryption. + pub variance: f64, + /// The lwe dimension of the ciphertext. + pub lweDimension: u32, +} + +/// Evaluation keys and ciphertexts can be compressed when transported over the wire. This +/// enumeration encodes the different compressions that can be used to compress scheme objects. +/// +/// Note: +/// Not all compressions are available for every types of evaluation keys or ciphertexts. +#[derive(Debug, Deserialize, Serialize)] +pub enum Compression { + none, + seed, + paillier, +} + +/// The encoding of the value stored inside the ciphertext. +#[derive(Debug, Deserialize, Serialize)] +pub enum LweCiphretextTypeInfo_Encoding { + integer(IntegerCiphertextEncodingInfo), + boolean(BooleanCiphertextEncodingInfo), +} + +/// A ciphertext can be used to represent an integer value. This structure represents the +/// informations needed to encode such an integer. +#[derive(Debug, Deserialize, Serialize)] +pub struct IntegerCiphertextEncodingInfo { + /// The bitwidth of the encoded integer. + pub width: u32, + /// The signedness of the encoded integer. + pub isSigned: bool, + /// The mode used to encode the integer. + pub mode: IntegerCiphertextEncodingInfo_Mode, +} + +/// The mode used to encode the integer. +#[derive(Debug, Deserialize, Serialize)] +pub enum IntegerCiphertextEncodingInfo_Mode { + native(IntegerCiphertextEncodingInfo_Mode_NativeMode), + chunked(IntegerCiphertextEncodingInfo_Mode_ChunkedMode), + crt(IntegerCiphertextEncodingInfo_Mode_CrtMode), +} + +/// An integer of width from 1 to 8 bits can be encoded in a single ciphertext natively, by +/// being shifted in the most significant bits. This structure represents this integer encoding +/// mode. +#[derive(Debug, Deserialize, Serialize)] +pub struct IntegerCiphertextEncodingInfo_Mode_NativeMode {} + +/// An integer of width from 1 to n can be encoded in a set of ciphertexts by chunking the bits +/// of the original integer. This structure represents this integer encoding mode. +#[derive(Debug, Deserialize, Serialize)] +pub struct IntegerCiphertextEncodingInfo_Mode_ChunkedMode { + /// The number of chunks to be used. + pub size: u32, + /// The number of bits encoded by each chunks. + pub width: u32, +} + +/// An integer of width 1 to 16 can be encoded in a set of ciphertexts, by decomposing a value +/// using a set of pairwise coprimes. This structure represents this integer encoding mode. +#[derive(Debug, Deserialize, Serialize)] +pub struct IntegerCiphertextEncodingInfo_Mode_CrtMode { + /// The coprimes used to decompose the original value. + pub moduli: Vec, +} + +/// A ciphertext can be used to represent a boolean value. This structure represents such an +/// encoding. +#[derive(Debug, Deserialize, Serialize)] +pub struct BooleanCiphertextEncodingInfo {} + +/// Secret Keys can be drawn from different ranges of values, using different distributions. This +/// enumeration encodes the different supported ways. +#[derive(Debug, Deserialize, Serialize)] +pub enum KeyType { + binary = 0, + ternary = 1, +} + +/// Ciphertext operations are performed using modular arithmetic. Depending on the use, different +/// modulus can be used for the operations. This structure encodes the different supported ways. +#[derive(Debug, Deserialize, Serialize)] +pub struct Modulus { + /// The modulus expected to be used. + pub modulus: Modulus_enum, +} + +/// The modulus expected to be used. +#[derive(Debug, Deserialize, Serialize)] +pub enum Modulus_enum { + native(NativeModulus), + powerOfTwo(PowerOfTwoModulus), + integer(IntegerModulus), +} + +/// Operations are performed using the modulus of the integers used to store the ciphertexts. +/// +/// Note: +/// The bitwidth of the integer storage is represented implicitly here, and must be grabbed from +/// the rest of the description. +/// +/// Example: +/// 2^64 when the ciphertext is stored using 64 bits integers. +#[derive(Debug, Deserialize, Serialize)] +pub struct NativeModulus {} + +/// Operations are performed using a modulus that is a power of two. +/// +/// Example: +/// 2^n for any n between 0 and the bitwidth of the integer used to store the ciphertext. +#[derive(Debug, Deserialize, Serialize)] +pub struct PowerOfTwoModulus { + /// The power used to raise 2. + pub power: u32, +} + +/// Operations are performed using a modulus that is an arbitrary integer. +/// +/// Example: +/// n for any n between 0 and 2^N where N is the bitwidth of the integer used to store the +/// ciphertext. +#[derive(Debug, Deserialize, Serialize)] +pub struct IntegerModulus { + /// The value used as modulus. + pub modulus: u32, +} + +/// A secret key value is uniquely described by cryptographic parameters and an identifier. This +/// structure represents this description of a secret key. +/// +/// Note: +/// Secret keys with same parameters are allowed to co-exist in a program, as long as they +/// have different ids. +#[derive(Debug, Deserialize, Serialize)] +pub struct LweSecretKeyInfo { + /// The identifier of the key. + pub id: u32, + /// The cryptographic parameters of the keys. + pub params: LweSecretKeyParams, +} + +/// A secret key is parameterized by a few quantities of cryptographic importance. This structure +/// represents those parameters. +#[derive(Debug, Deserialize, Serialize)] +pub struct LweSecretKeyParams { + /// The LWE dimension, e.g. the length of the key. + pub lweDimension: u32, + /// The bitwidth of the integers used for storage. + pub integerPrecision: u32, + /// The kind of distribution used to sample the key. + pub keyType: KeyType, +} + +/// A keyswitch key value is uniquely described by cryptographic parameters and a few application +/// related quantities. This structure represents this description of a keyswitch key. +/// +/// Note: +/// Keyswitch keys with same parameters, compression, input and output id, are allowed to co-exist +/// in a program as long as they have different ids. +#[derive(Debug, Deserialize, Serialize)] +pub struct LweKeyswitchKeyInfo { + /// The identifier of the keyswitch key. + pub id: u32, + /// The identifier of the input secret key. + pub inputId: u32, + /// The identifier of the output secret key. + pub outputId: u32, + /// The cryptographic parameters of the key. + pub params: LweKeyswitchKeyParams, + /// The compression used to store the key. + pub compression: Compression, +} + +/// A keyswitch key is parameterized by a few quantities of cryptographic importance. This structure +/// represents those parameters. +/// +/// Note: +/// For now, only keys with the same input and output key types can be represented. +#[derive(Debug, Deserialize, Serialize)] +pub struct LweKeyswitchKeyParams { + /// The number of levels of the ciphertexts. + pub levelCount: u32, + /// The logarithm of the base of ciphertexts. + pub baseLog: u32, + /// The variance used to encrypt the ciphertexts. + pub variance: f64, + /// The bitwidth of the integers used to store the ciphertexts. + pub integerPrecision: u32, + /// The dimension of the input secret key. + pub inputLweDimension: u32, + /// The dimension of the output secret key. + pub outputLweDimension: u32, + /// The modulus used to perform operations with this key. + pub modulus: Modulus, + /// The distribution of the input and output secret keys. + pub keyType: KeyType, +} + +/// A packing keyswitch key value is uniquely described by cryptographic parameters and a few +/// application related quantities. This structure represents this description of a packing +/// keyswitch key. +/// +/// Note: +/// Packing keyswitch keys with same parameters, compression, input and output id, are allowed to +/// co-exist in a program as long as they have different ids. +#[derive(Debug, Deserialize, Serialize)] +pub struct PackingKeyswitchKeyInfo { + /// The identifier of the packing keyswitch key. + pub id: u32, + /// The identifier of the input secret key. + pub inputId: u32, + /// The identifier of the output secret key. + pub outputId: u32, + /// The cryptographic parameters of the key. + pub params: PackingKeyswitchKeyParams, + /// The compression used to store the key. + pub compression: Compression, +} + +/// A packing keyswitch key is parameterized by a few quantities of cryptographic importance. This +/// structure represents those parameters. +/// +/// Note: +/// For now, only keys with the same input and output key types can be represented. +#[derive(Debug, Deserialize, Serialize)] +pub struct PackingKeyswitchKeyParams { + /// The number of levels of the ciphertexts. + pub levelCount: u32, + /// The logarithm of the base of the ciphertexts. + pub baseLog: u32, + /// The glwe dimension of the ciphertexts. + pub glweDimension: u32, + /// The polynomial size of the ciphertexts. + pub polynomialSize: u32, + /// The input lwe dimension. + pub inputLweDimension: u32, + /// The intermediate lwe dimension. + pub innerLweDimension: u32, + /// The variance used to encrypt the ciphertexts. + pub variance: f64, + /// The bitwidth of the integers used to store the ciphertexts. + pub integerPrecision: u32, + /// The modulus used to perform operations with this key. + pub modulus: Modulus, + /// The distribution of the input and output secret keys. + pub keyType: KeyType, +} + +/// A bootstrap key value is uniquely described by cryptographic parameters and a few application +/// related quantities. This structure represents this description of a bootstrap key. +/// +/// Note: +/// Bootstrap keys with same parameters, compression, input and output id, are allowed to co-exist +/// in a program as long as they have different ids. +#[derive(Debug, Deserialize, Serialize)] +pub struct LweBootstrapKeyInfo { + /// The identifier of the bootstrap key. + pub id: u32, + /// The identifier of the input secret key. + pub inputId: u32, + /// The identifier of the output secret key. + pub outputId: u32, + /// The cryptographic parameters of the key. + pub params: LweBootstrapKeyParams, + /// The compression used to store the key. + pub compression: Compression, +} + +/// A bootstrap key is parameterized by a few quantities of cryptographic importance. This structure +/// represents those parameters. +/// +/// Note: +/// For now, only keys with the same input and output key types can be represented. +#[derive(Debug, Deserialize, Serialize)] +pub struct LweBootstrapKeyParams { + /// The number of levels of the ciphertexts. + pub levelCount: u32, + /// The logarithm of the base of the ciphertext. + pub baseLog: u32, + /// The dimension of the ciphertexts. + pub glweDimension: u32, + /// The polynomial size of the ciphertexts. + pub polynomialSize: u32, + /// The dimension of the input lwe secret key. + pub inputLweDimension: u32, + /// The variance used to encrypt the ciphertexts. + pub variance: f64, + /// The bitwidth of the integers used to store the ciphertexts. + pub integerPrecision: u32, + /// The modulus used to perform operations with this key. + pub modulus: Modulus, + /// The distribution of the input and output secret keys. + pub keyType: KeyType, +} + +/// The keyset needed for an application can be described by an ensemble of descriptions of the +/// different keys used in the program. This structure represents such a description. +#[derive(Debug, Deserialize, Serialize)] +pub struct KeysetInfo { + /// The secret key descriptions. + pub lweSecretKeys: Vec, + /// The bootstrap key descriptions + pub lweBootstrapKeys: Vec, + /// The keyswitch key descriptions. + pub lweKeyswitchKeys: Vec, + /// The packing keyswitch key descriptions. + pub packingKeyswitchKeys: Vec, +} + +#[cfg(feature = "compiler")] +mod to_tokens { + //! This module contains `ToTokens` implementations for the protocol types. This allows protocol + //! values to be interpolated in the `quote!` macro as constructors of the values. + //! Useful to construct static protocol values. + + use super::*; + use proc_macro2::TokenStream; + use quote::{quote, ToTokens}; + + impl ToTokens for ProgramInfo { + fn to_tokens(&self, tokens: &mut TokenStream) { + let keyset = &self.keyset; + let circuits = &self + .circuits + .iter() + .map(|circuit| quote! { #circuit }) + .collect::>(); + tokens.extend(quote! { + ::concrete::protocol::ProgramInfo { + keyset: #keyset, + circuits: vec![#(#circuits),*], + } + }); + } + } + + impl ToTokens for CircuitInfo { + fn to_tokens(&self, tokens: &mut TokenStream) { + let inputs = self + .inputs + .iter() + .map(|input| quote! { #input }) + .collect::>(); + let outputs = self + .outputs + .iter() + .map(|output| quote! { #output }) + .collect::>(); + let name = &self.name; + tokens.extend(quote! { + ::concrete::protocol::CircuitInfo { + inputs: vec![#(#inputs),*], + outputs: vec![#(#outputs),*], + name: String::from(#name), + } + }); + } + } + impl ToTokens for GateInfo { + fn to_tokens(&self, tokens: &mut TokenStream) { + let raw_info = &self.rawInfo; + let type_info = &self.typeInfo; + tokens.extend(quote! { + ::concrete::protocol::GateInfo { + rawInfo: #raw_info, + typeInfo: #type_info, + } + }); + } + } + + impl ToTokens for RawInfo { + fn to_tokens(&self, tokens: &mut TokenStream) { + let shape = &self.shape; + let integer_precision = &self.integerPrecision; + let is_signed = &self.isSigned; + tokens.extend(quote! { + ::concrete::protocol::RawInfo { + shape: #shape, + integerPrecision: #integer_precision, + isSigned: #is_signed, + } + }); + } + } + + impl ToTokens for Shape { + fn to_tokens(&self, tokens: &mut TokenStream) { + let dimensions = self + .dimensions + .iter() + .map(|dim| quote! { #dim }) + .collect::>(); + tokens.extend(quote! { + ::concrete::protocol::Shape { + dimensions: vec![#(#dimensions),*], + } + }); + } + } + + impl ToTokens for TypeInfo { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + TypeInfo::lweCiphertext(info) => { + tokens.extend(quote! { ::concrete::protocol::TypeInfo::lweCiphertext(#info) }) + } + TypeInfo::plaintext(info) => { + tokens.extend(quote! { ::concrete::protocol::TypeInfo::plaintext(#info) }) + } + TypeInfo::index(info) => { + tokens.extend(quote! { ::concrete::procotol::TypeInfo::index(#info) }) + } + } + } + } + + impl ToTokens for PlaintextTypeInfo { + fn to_tokens(&self, tokens: &mut TokenStream) { + let shape = &self.shape; + let integer_precision = &self.integerPrecision; + let is_signed = &self.isSigned; + tokens.extend(quote! { + ::concrete::protocol::PlaintextTypeInfo { + shape: #shape, + integerPrecision: #integer_precision, + isSigned: #is_signed, + } + }); + } + } + + impl ToTokens for IndexTypeInfo { + fn to_tokens(&self, tokens: &mut TokenStream) { + let shape = &self.shape; + let integer_precision = &self.integerPrecision; + let is_signed = &self.isSigned; + tokens.extend(quote! { + ::concrete::protocol::IndexTypeInfo { + shape: #shape, + integerPrecision: #integer_precision, + isSigned: #is_signed, + } + }); + } + } + + impl ToTokens for LweCiphertextTypeInfo { + fn to_tokens(&self, tokens: &mut TokenStream) { + let abstract_shape = &self.abstractShape; + let concrete_shape = &self.concreteShape; + let integer_precision = &self.integerPrecision; + let encryption = &self.encryption; + let compression = &self.compression; + let encoding = &self.encoding; + tokens.extend(quote! { + ::concrete::protocol::LweCiphertextTypeInfo { + abstractShape: #abstract_shape, + concreteShape: #concrete_shape, + integerPrecision: #integer_precision, + encryption: #encryption, + compression: #compression, + encoding: #encoding, + } + }); + } + } + + impl ToTokens for LweCiphertextEncryptionInfo { + fn to_tokens(&self, tokens: &mut TokenStream) { + let key_id = &self.keyId; + let variance = &self.variance; + let lwe_dimension = &self.lweDimension; + tokens.extend(quote! { + ::concrete::protocol::LweCiphertextEncryptionInfo { + keyId: #key_id, + variance: #variance, + lweDimension: #lwe_dimension, + } + }); + } + } + + impl ToTokens for Compression { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + Compression::none => { + tokens.extend(quote! { ::concrete::protocol::Compression::none }) + } + Compression::seed => { + tokens.extend(quote! { ::concrete::protocol::Compression::seed }) + } + Compression::paillier => { + tokens.extend(quote! { ::concrete::protocol::Compression::paillier }) + } + } + } + } + + impl ToTokens for LweCiphretextTypeInfo_Encoding { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + LweCiphretextTypeInfo_Encoding::integer(info) => tokens.extend( + quote! { ::concrete::protocol::LweCiphretextTypeInfo_Encoding::integer(#info) }, + ), + LweCiphretextTypeInfo_Encoding::boolean(info) => tokens.extend( + quote! { ::concrete::protocol::LweCiphretextTypeInfo_Encoding::boolean(#info) }, + ), + } + } + } + + impl ToTokens for IntegerCiphertextEncodingInfo { + fn to_tokens(&self, tokens: &mut TokenStream) { + let width = &self.width; + let is_signed = &self.isSigned; + let mode = &self.mode; + tokens.extend(quote! { + ::concrete::protocol::IntegerCiphertextEncodingInfo { + width: #width, + isSigned: #is_signed, + mode: #mode, + } + }); + } + } + + impl ToTokens for IntegerCiphertextEncodingInfo_Mode { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + IntegerCiphertextEncodingInfo_Mode::native(info) => tokens.extend(quote! { ::concrete::protocol::IntegerCiphertextEncodingInfo_Mode::native(#info) }), + IntegerCiphertextEncodingInfo_Mode::chunked(info) => tokens.extend(quote! { ::concrete::protocol::IntegerCiphertextEncodingInfo_Mode::chunked(#info) }), + IntegerCiphertextEncodingInfo_Mode::crt(info) => tokens.extend(quote! { ::concrete::protocol::IntegerCiphertextEncodingInfo_Mode::crt(#info) }), + } + } + } + + impl ToTokens for IntegerCiphertextEncodingInfo_Mode_NativeMode { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.extend(quote! { + ::concrete::protocol::IntegerCiphertextEncodingInfo_Mode_NativeMode {} + }); + } + } + + impl ToTokens for IntegerCiphertextEncodingInfo_Mode_ChunkedMode { + fn to_tokens(&self, tokens: &mut TokenStream) { + let size = &self.size; + let width = &self.width; + tokens.extend(quote! { + ::concrete::protocol::IntegerCiphertextEncodingInfo_Mode_ChunkedMode { + size: #size, + width: #width, + } + }); + } + } + + impl ToTokens for IntegerCiphertextEncodingInfo_Mode_CrtMode { + fn to_tokens(&self, tokens: &mut TokenStream) { + let moduli = self + .moduli + .iter() + .map(|modulus| quote! { #modulus }) + .collect::>(); + tokens.extend(quote! { + ::concrete::protocol::IntegerCiphertextEncodingInfo_Mode_CrtMode { + moduli: vec![#(#moduli),*], + } + }); + } + } + impl ToTokens for BooleanCiphertextEncodingInfo { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.extend(quote! { + ::concrete::protocol::BooleanCiphertextEncodingInfo {} + }); + } + } + + impl ToTokens for KeyType { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + KeyType::binary => tokens.extend(quote! { ::concrete::protocol::KeyType::binary }), + KeyType::ternary => { + tokens.extend(quote! { ::concrete::protocol::KeyType::ternary }) + } + } + } + } + + impl ToTokens for Modulus { + fn to_tokens(&self, tokens: &mut TokenStream) { + let modulus = &self.modulus; + tokens.extend(quote! { + ::concrete::protocol::Modulus { + modulus: #modulus, + } + }); + } + } + + impl ToTokens for Modulus_enum { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + Modulus_enum::native(info) => { + tokens.extend(quote! { ::concrete::protocol::Modulus_enum::native(#info) }) + } + Modulus_enum::powerOfTwo(info) => { + tokens.extend(quote! { ::concrete::protocol::Modulus_enum::powerOfTwo(#info) }) + } + Modulus_enum::integer(info) => { + tokens.extend(quote! { ::concrete::protocol::Modulus_enum::integer(#info) }) + } + } + } + } + + impl ToTokens for NativeModulus { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.extend(quote! { + ::concrete::protocol::NativeModulus {} + }); + } + } + + impl ToTokens for PowerOfTwoModulus { + fn to_tokens(&self, tokens: &mut TokenStream) { + let power = &self.power; + tokens.extend(quote! { + ::concrete::protocol::PowerOfTwoModulus { + power: #power, + } + }); + } + } + + impl ToTokens for IntegerModulus { + fn to_tokens(&self, tokens: &mut TokenStream) { + let modulus = &self.modulus; + tokens.extend(quote! { + ::concrete::protocol::IntegerModulus { + modulus: #modulus, + } + }); + } + } + + impl ToTokens for LweSecretKeyInfo { + fn to_tokens(&self, tokens: &mut TokenStream) { + let id = &self.id; + let params = &self.params; + tokens.extend(quote! { + ::concrete::protocol::LweSecretKeyInfo { + id: #id, + params: #params, + } + }); + } + } + + impl ToTokens for LweSecretKeyParams { + fn to_tokens(&self, tokens: &mut TokenStream) { + let lwe_dimension = &self.lweDimension; + let integer_precision = &self.integerPrecision; + let key_type = &self.keyType; + tokens.extend(quote! { + ::concrete::protocol::LweSecretKeyParams { + lweDimension: #lwe_dimension, + integerPrecision: #integer_precision, + keyType: #key_type, + } + }); + } + } + + impl ToTokens for LweKeyswitchKeyInfo { + fn to_tokens(&self, tokens: &mut TokenStream) { + let id = &self.id; + let input_id = &self.inputId; + let output_id = &self.outputId; + let params = &self.params; + let compression = &self.compression; + tokens.extend(quote! { + ::concrete::protocol::LweKeyswitchKeyInfo { + id: #id, + inputId: #input_id, + outputId: #output_id, + params: #params, + compression: #compression, + } + }); + } + } + + impl ToTokens for LweKeyswitchKeyParams { + fn to_tokens(&self, tokens: &mut TokenStream) { + let level_count = &self.levelCount; + let base_log = &self.baseLog; + let variance = &self.variance; + let integer_precision = &self.integerPrecision; + let input_lwe_dimension = &self.inputLweDimension; + let output_lwe_dimension = &self.outputLweDimension; + let modulus = &self.modulus; + let key_type = &self.keyType; + tokens.extend(quote! { + ::concrete::protocol::LweKeyswitchKeyParams { + levelCount: #level_count, + baseLog: #base_log, + variance: #variance, + integerPrecision: #integer_precision, + inputLweDimension: #input_lwe_dimension, + outputLweDimension: #output_lwe_dimension, + modulus: #modulus, + keyType: #key_type, + } + }); + } + } + + impl ToTokens for PackingKeyswitchKeyInfo { + fn to_tokens(&self, tokens: &mut TokenStream) { + let id = &self.id; + let input_id = &self.inputId; + let output_id = &self.outputId; + let params = &self.params; + let compression = &self.compression; + tokens.extend(quote! { + ::concrete::protocol::PackingKeyswitchKeyInfo { + id: #id, + inputId: #input_id, + outputId: #output_id, + params: #params, + compression: #compression, + } + }); + } + } + + impl ToTokens for PackingKeyswitchKeyParams { + fn to_tokens(&self, tokens: &mut TokenStream) { + let level_count = &self.levelCount; + let base_log = &self.baseLog; + let glwe_dimension = &self.glweDimension; + let polynomial_size = &self.polynomialSize; + let input_lwe_dimension = &self.inputLweDimension; + let inner_lwe_dimension = &self.innerLweDimension; + let variance = &self.variance; + let integer_precision = &self.integerPrecision; + let modulus = &self.modulus; + let key_type = &self.keyType; + tokens.extend(quote! { + ::concrete::protocol::PackingKeyswitchKeyParams { + levelCount: #level_count, + baseLog: #base_log, + glweDimension: #glwe_dimension, + polynomialSize: #polynomial_size, + inputLweDimension: #input_lwe_dimension, + innerLweDimension: #inner_lwe_dimension, + variance: #variance, + integerPrecision: #integer_precision, + modulus: #modulus, + keyType: #key_type, + } + }); + } + } + + impl ToTokens for LweBootstrapKeyInfo { + fn to_tokens(&self, tokens: &mut TokenStream) { + let id = &self.id; + let input_id = &self.inputId; + let output_id = &self.outputId; + let params = &self.params; + let compression = &self.compression; + tokens.extend(quote! { + ::concrete::protocol::LweBootstrapKeyInfo { + id: #id, + inputId: #input_id, + outputId: #output_id, + params: #params, + compression: #compression, + } + }); + } + } + + impl ToTokens for LweBootstrapKeyParams { + fn to_tokens(&self, tokens: &mut TokenStream) { + let level_count = &self.levelCount; + let base_log = &self.baseLog; + let glwe_dimension = &self.glweDimension; + let polynomial_size = &self.polynomialSize; + let input_lwe_dimension = &self.inputLweDimension; + let variance = &self.variance; + let integer_precision = &self.integerPrecision; + let modulus = &self.modulus; + let key_type = &self.keyType; + tokens.extend(quote! { + ::concrete::protocol::LweBootstrapKeyParams { + levelCount: #level_count, + baseLog: #base_log, + glweDimension: #glwe_dimension, + polynomialSize: #polynomial_size, + inputLweDimension: #input_lwe_dimension, + variance: #variance, + integerPrecision: #integer_precision, + modulus: #modulus, + keyType: #key_type, + } + }); + } + } + + impl ToTokens for KeysetInfo { + fn to_tokens(&self, tokens: &mut TokenStream) { + let lwe_secret_keys = self + .lweSecretKeys + .iter() + .map(|key| quote! { #key }) + .collect::>(); + let lwe_bootstrap_keys = self + .lweBootstrapKeys + .iter() + .map(|key| quote! { #key }) + .collect::>(); + let lwe_keyswitch_keys = self + .lweKeyswitchKeys + .iter() + .map(|key| quote! { #key }) + .collect::>(); + let packing_keyswitch_keys = self + .packingKeyswitchKeys + .iter() + .map(|key| quote! { #key }) + .collect::>(); + tokens.extend(quote! { + ::concrete::protocol::KeysetInfo { + lweSecretKeys: vec![#(#lwe_secret_keys),*], + lweBootstrapKeys: vec![#(#lwe_bootstrap_keys),*], + lweKeyswitchKeys: vec![#(#lwe_keyswitch_keys),*], + packingKeyswitchKeys: vec![#(#packing_keyswitch_keys),*], + } + }); + } + } +} diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/rust-toolchain.toml b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/rust-toolchain.toml new file mode 100644 index 000000000..5d56faf9a --- /dev/null +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/rust-toolchain.toml @@ -0,0 +1,2 @@ +[toolchain] +channel = "nightly" diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/test/Cargo.toml b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/test/Cargo.toml new file mode 100644 index 000000000..5a15b23e4 --- /dev/null +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/test/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "concrete-test" +version = "0.0.0" +edition = "2021" + +[dependencies] +concrete-macro = { version = "2.10.1-rc1", path = "../concrete-macro" } +concrete = { version = "2.10.1-rc1", path = "../concrete" } diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/test/src/lib.rs b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/test/src/lib.rs new file mode 100644 index 000000000..d29e64b93 --- /dev/null +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/test/src/lib.rs @@ -0,0 +1,36 @@ +mod precompile{ + use concrete_macro::from_concrete_python_export_zip; + from_concrete_python_export_zip!("src/test.zip"); +} + +#[cfg(test)] +mod test { + use concrete::common::{ClientKeyset, ServerKeyset, Tensor, TransportValue}; + use crate::precompile; + + #[test] + fn test() { + let mut secret_csprng = concrete::common::SecretCsprng::new(0u128); + let mut encryption_csprng = concrete::common::EncryptionCsprng::new(0u128); + let keyset = precompile::new_keyset(secret_csprng.pin_mut(), encryption_csprng.pin_mut()); + let client_keyset = keyset.get_client(); + let serialized_client_keyset = client_keyset.serialize(); + let deserialized_client_keyset = ClientKeyset::deserialize(serialized_client_keyset.as_slice()); + let server_keyset = keyset.get_server(); + let serialized_server_keyset = server_keyset.serialize(); + let deserialized_server_keyset = ServerKeyset::deserialize(serialized_server_keyset.as_slice()); + let mut dec_client = + precompile::client::dec::ClientFunction::new(&deserialized_client_keyset, encryption_csprng); + let mut dec_server = precompile::server::dec::ServerFunction::new(); + let input = Tensor::new(vec![5], vec![]); + let prepared_input = dec_client.prepare_inputs(input); + let serialized_input = prepared_input.serialize(); + let deserialized_input = TransportValue::deserialize(serialized_input.as_slice()); + let output = dec_server.invoke(&deserialized_server_keyset, deserialized_input); + let serialized_output = output.serialize(); + let deserialized_output = TransportValue::deserialize(serialized_output.as_slice()); + let processed_output = dec_client.process_outputs(deserialized_output); + assert_eq!(processed_output.values(), [4]); + assert_eq!(processed_output.dimensions().len(), 0); + } +} diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/test/src/main.rs b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/test/src/main.rs new file mode 100644 index 000000000..16eafa253 --- /dev/null +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/test/src/main.rs @@ -0,0 +1,22 @@ +use concrete::common::Tensor; + +mod precompile { + use concrete_macro::from_concrete_python_export_zip; + from_concrete_python_export_zip!("src/test.zip"); +} + +fn main() { + let mut secret_csprng = concrete::common::SecretCsprng::new(0u128); + let mut encryption_csprng = concrete::common::EncryptionCsprng::new(0u128); + let keyset = precompile::new_keyset(secret_csprng.pin_mut(), encryption_csprng.pin_mut()); + let client_keyset = keyset.get_client(); + let server_keyset = keyset.get_server(); + let mut dec_client = + precompile::client::dec::ClientFunction::new(&client_keyset, encryption_csprng); + let mut dec_server = precompile::server::dec::ServerFunction::new(); + let input = Tensor::new(vec![5], vec![]); + let prepared_input = dec_client.prepare_inputs(input); + let output = dec_server.invoke(&server_keyset, prepared_input); + let processed_output = dec_client.process_outputs(output); + println!("{:?}", processed_output.values()); +} diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/test/src/test.zip b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/test/src/test.zip new file mode 100644 index 0000000000000000000000000000000000000000..393f0be539af0341e094804089cc34601619e154 GIT binary patch literal 1696 zcmWIWW@Zs#U|`^2@biv|QoMir3p0?%2*f-!E3Y8oFzvq0jZ|-THfP{NVla%=R^MCHEUUuV$ z%kFslEtk}O{YYH?N1$h^hKcD?;pXe*TR$Jpu6^}4uq3ce&MACNY)bFRM;RsWT0iMM z)#yF_GE{ozq?dObwwq0E`*K&I+4Q5=6KAvLpVQYp)jXXOWR@lDFkO*Da&zX=Q=S#- zU5kSirV6f~dTg3U&b;J@F(*$R6Uv#Cw?s2Yh8_|-#WTm=jQ%JkBpeIg}&(Iu4I&S-j$vdqY-riuR0jiJ!A<#=4E?vQnNqH0?L= zI9944JzF|L-u%|VI0Wo>0B6|05PkY_| z8^5{UmHKGs8n3frR{YiHz2|9>SZ7XA?e5#L9;w?6*(@gV1jSgDtzIA;RXy$e-&36@ zGA=E%i@7jOX=i7PdPE|(46oXA-pN;Hi!oQ3M2H?R`*GT0>X&qvL%+A*;W^gnIPacO z;t9KtbK=jIil1^5Iw^Kf`~csis)DQ4s}nz*-Wv4vsO*6#+2iN;Fh|)asB_HA+ZvR8 z_~b_q*Sz}m0qvP4S_jrP9{+GB>PqI}FAJ0+vh!Abw>G&ipI*NGZDMqw7)gnYZ|>ebfJKziMX0{HV3}jtoe7j!?Q<2e%vmdu;@x%KGe^)-$MwwyZjc-D32rp6uC7ph8%47jR%mq+cJ zcg3+nhVA~k8`F22M6VD3ll$9JrHn1ho=v9Xkwe1Yy|P9xtx5zEV|r&X#+CB_XS{KJ zKjWcUYHyET@)OJdnf>egruu|%erf-nrVR!^{9e7G1+6M`p)rRxO&j$e5mOK zwwHy~As?SFuex$k-RIldc~=*D9=c?_(ALrUUf0eznWBlZ@1o?^@x<@g{i7mypHg~t zm9W+VZYi!idFQqM|2L?d!?gE&QpKFwf042GblUyx>z^L|@JSx6xKQho-~kpaPk@;l zRJJ5%7A2Qvmgwc?WEQ=h=;(deL7?^hzpnI!7Y<1o-8ReWIN1HDULbPY$yMEj!jpJ4 zF8zO(BYkj-ZH1t|Tp9o8eKWWJezsNb_o)YuSt{Pd3bt}zU#giFeAn)%reFU1Jf-Bh z=fi%=r2ky8;VttrR!!|f*1J*vWkk2W-cu&mA` ClientProgram::createSimulated( return output; } -Result ClientProgram::getClientCircuit(std::string circuitName) { +Result +ClientProgram::getClientCircuit(std::string circuitName) const { for (auto circuit : circuits) { if (circuit.getName() == circuitName) { return circuit; diff --git a/compilers/concrete-compiler/compiler/lib/Common/CMakeLists.txt b/compilers/concrete-compiler/compiler/lib/Common/CMakeLists.txt index 55fd6cf0b..e32a613a4 100644 --- a/compilers/concrete-compiler/compiler/lib/Common/CMakeLists.txt +++ b/compilers/concrete-compiler/compiler/lib/Common/CMakeLists.txt @@ -12,8 +12,12 @@ add_mlir_library( Values.cpp LINK_LIBS PUBLIC + rust_deps_bundle concrete-protocol capnp kj) -target_include_directories(ConcretelangCommon PUBLIC ${CONCRETE_CPU_INCLUDE_DIR}) +target_include_directories( + ConcretelangCommon + PUBLIC ${CONCRETE_CPU_INCLUDE_DIR} + PUBLIC ${CONCRETE_CPU_NOISE_MODEL_INCLUDE_DIR}) diff --git a/compilers/concrete-compiler/compiler/lib/Common/Transformers.cpp b/compilers/concrete-compiler/compiler/lib/Common/Transformers.cpp index c6e581cab..9a70ba7d9 100644 --- a/compilers/concrete-compiler/compiler/lib/Common/Transformers.cpp +++ b/compilers/concrete-compiler/compiler/lib/Common/Transformers.cpp @@ -11,8 +11,9 @@ #include "concretelang/Common/Csprng.h" #include "concretelang/Common/Error.h" #include "concretelang/Common/Keysets.h" +#include "concretelang/Common/Security.h" #include "concretelang/Common/Values.h" -#include "concretelang/Runtime/simulation.h" + #include #include #include @@ -24,6 +25,31 @@ using concretelang::values::Tensor; using concretelang::values::TransportValue; using concretelang::values::Value; +namespace { +thread_local auto default_csprng = concretelang::csprng::SoftCSPRNG(0); + +inline concretelang::security::SecurityCurve *security_curve() { + return concretelang::security::getSecurityCurve( + 128, concretelang::security::BINARY); +} + +uint64_t gaussian_noise(double variance, Csprng *csprng = default_csprng.ptr) { + uint64_t random_gaussian_buff[2]; + + double std_dev = std::sqrt(variance); + concrete_cpu_fill_with_random_gaussian(random_gaussian_buff, 2, std_dev, + csprng); + return random_gaussian_buff[0]; +} +} // namespace + +uint64_t sim_encrypt_lwe_u64(uint64_t message, uint32_t lwe_dim, + Csprng *csprng) { + double variance = security_curve()->getVariance(1, lwe_dim, 64); + uint64_t encryption_noise = gaussian_noise(variance, (Csprng *)csprng); + return message + encryption_noise; +} + namespace concretelang { namespace transformers { diff --git a/compilers/concrete-compiler/compiler/lib/Common/Values.cpp b/compilers/concrete-compiler/compiler/lib/Common/Values.cpp index 0f2efeb53..9dbd5e2db 100644 --- a/compilers/concrete-compiler/compiler/lib/Common/Values.cpp +++ b/compilers/concrete-compiler/compiler/lib/Common/Values.cpp @@ -127,26 +127,32 @@ Message Value::intoProtoShape() const { return dimensionsToProtoShape(getDimensions()); } -std::vector Value::getDimensions() const { - if (auto tensor = getTensor(); tensor) { - return tensor.value().dimensions; - } else if (auto tensor = getTensor(); tensor) { - return tensor.value().dimensions; - } else if (auto tensor = getTensor(); tensor) { - return tensor.value().dimensions; - } else if (auto tensor = getTensor(); tensor) { - return tensor.value().dimensions; - } else if (auto tensor = getTensor(); tensor) { - return tensor.value().dimensions; - } else if (auto tensor = getTensor(); tensor) { - return tensor.value().dimensions; - } else if (auto tensor = getTensor(); tensor) { - return tensor.value().dimensions; - } else if (auto tensor = getTensor(); tensor) { - return tensor.value().dimensions; - } else { - assert(false); +const std::vector &Value::getDimensions() const { + if (hasElementType()) { + return std::get>(inner).dimensions; } + if (hasElementType()) { + return std::get>(inner).dimensions; + } + if (hasElementType()) { + return std::get>(inner).dimensions; + } + if (hasElementType()) { + return std::get>(inner).dimensions; + } + if (hasElementType()) { + return std::get>(inner).dimensions; + } + if (hasElementType()) { + return std::get>(inner).dimensions; + } + if (hasElementType()) { + return std::get>(inner).dimensions; + } + if (hasElementType()) { + return std::get>(inner).dimensions; + } + assert(false); } size_t Value::getLength() const { diff --git a/compilers/concrete-compiler/compiler/lib/Runtime/CMakeLists.txt b/compilers/concrete-compiler/compiler/lib/Runtime/CMakeLists.txt index 9f1b768ca..2ea777750 100644 --- a/compilers/concrete-compiler/compiler/lib/Runtime/CMakeLists.txt +++ b/compilers/concrete-compiler/compiler/lib/Runtime/CMakeLists.txt @@ -3,24 +3,22 @@ add_compile_options(-fsized-deallocation) if(CONCRETELANG_CUDA_SUPPORT) add_library( ConcretelangRuntime SHARED - utils.cpp context.cpp - simulation.cpp - wrappers.cpp DFRuntime.cpp key_manager.cpp GPUDFG.cpp + simulation.cpp + wrappers.cpp time_util.cpp) else() add_library( ConcretelangRuntime SHARED - utils.cpp context.cpp - simulation.cpp - wrappers.cpp DFRuntime.cpp key_manager.cpp GPUDFG.cpp + simulation.cpp + wrappers.cpp time_util.cpp) endif() @@ -30,6 +28,7 @@ if(CONCRETELANG_DATAFLOW_EXECUTION_ENABLED) target_link_libraries(ConcretelangRuntime PUBLIC HPX::hpx HPX::iostreams_component HPX::component) target_link_libraries(ConcretelangRuntime PRIVATE hwloc) set_source_files_properties(DFRuntime.cpp PROPERTIES COMPILE_FLAGS "-fopenmp") + add_dependencies(ConcretelangRuntime HPXLibs) endif() if(CONCRETELANG_CUDA_SUPPORT) @@ -59,15 +58,13 @@ target_include_directories( target_link_libraries( ConcretelangRuntime - PRIVATE rust_deps_bundle - concrete-protocol - pthread + PRIVATE pthread m dl $ $ $ - ConcretelangCommon) + $) if(CONCRETELANG_CUDA_SUPPORT) install(TARGETS ConcretelangRuntime omp tfhe_cuda_backend EXPORT ConcretelangRuntime) @@ -81,11 +78,11 @@ install(EXPORT ConcretelangRuntime DESTINATION "./") add_library( ConcretelangRuntimeStatic STATIC context.cpp - simulation.cpp - wrappers.cpp DFRuntime.cpp key_manager.cpp GPUDFG.cpp + simulation.cpp + wrappers.cpp time_util.cpp) if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") diff --git a/compilers/concrete-compiler/compiler/lib/Runtime/GPUDFG.cpp b/compilers/concrete-compiler/compiler/lib/Runtime/GPUDFG.cpp index 646311734..6fbb77252 100644 --- a/compilers/concrete-compiler/compiler/lib/Runtime/GPUDFG.cpp +++ b/compilers/concrete-compiler/compiler/lib/Runtime/GPUDFG.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include diff --git a/compilers/concrete-compiler/compiler/lib/Runtime/simulation.cpp b/compilers/concrete-compiler/compiler/lib/Runtime/simulation.cpp index 825a06bd4..78bc9d9ea 100644 --- a/compilers/concrete-compiler/compiler/lib/Runtime/simulation.cpp +++ b/compilers/concrete-compiler/compiler/lib/Runtime/simulation.cpp @@ -38,13 +38,6 @@ uint64_t gaussian_noise(double variance, Csprng *csprng = default_csprng.ptr) { return random_gaussian_buff[0]; } -uint64_t sim_encrypt_lwe_u64(uint64_t message, uint32_t lwe_dim, - Csprng *csprng) { - double variance = security_curve()->getVariance(1, lwe_dim, 64); - uint64_t encryption_noise = gaussian_noise(variance, (Csprng *)csprng); - return message + encryption_noise; -} - uint64_t sim_keyswitch_lwe_u64(uint64_t plaintext, uint32_t level, uint32_t base_log, uint32_t input_lwe_dim, uint32_t output_lwe_dim) { diff --git a/compilers/concrete-compiler/compiler/lib/Runtime/utils.cpp b/compilers/concrete-compiler/compiler/lib/Runtime/utils.cpp deleted file mode 100644 index 2067b339b..000000000 --- a/compilers/concrete-compiler/compiler/lib/Runtime/utils.cpp +++ /dev/null @@ -1,15 +0,0 @@ -// Part of the Concrete Compiler Project, under the BSD3 License with Zama -// Exceptions. See -// https://github.com/zama-ai/concrete/blob/main/LICENSE.txt -// for license information. - -#include "concretelang/Runtime/utils.h" - -namespace mlir { -namespace concretelang { -void LLVMInitializeNativeTarget() { - llvm::InitializeNativeTarget(); - llvm::InitializeNativeTargetAsmPrinter(); -} -} // namespace concretelang -} // namespace mlir diff --git a/compilers/concrete-compiler/compiler/lib/Runtime/wrappers.cpp b/compilers/concrete-compiler/compiler/lib/Runtime/wrappers.cpp index 9527463b7..cd7b84f4b 100644 --- a/compilers/concrete-compiler/compiler/lib/Runtime/wrappers.cpp +++ b/compilers/concrete-compiler/compiler/lib/Runtime/wrappers.cpp @@ -18,9 +18,11 @@ #include #include "concretelang/Common/CRT.h" -#include "concretelang/Runtime/wrappers.h" #ifdef CONCRETELANG_CUDA_SUPPORT +#include "device.h" +#include "keyswitch.h" +#include "programmable_bootstrap.h" // CUDA memory utils function ///////////////////////////////////////////////// diff --git a/compilers/concrete-compiler/compiler/lib/ServerLib/ServerLib.cpp b/compilers/concrete-compiler/compiler/lib/ServerLib/ServerLib.cpp index 9c50d504f..ee9b843d7 100644 --- a/compilers/concrete-compiler/compiler/lib/ServerLib/ServerLib.cpp +++ b/compilers/concrete-compiler/compiler/lib/ServerLib/ServerLib.cpp @@ -431,9 +431,11 @@ bool getGateIsSigned(const Message &gateInfo) { Result> ServerCircuit::call(const ServerKeyset &serverKeyset, - std::vector &args) { + const std::vector &args) { std::vector returns(returnsBuffer.size()); - mlir::concretelang::dfr::_dfr_register_lib(dynamicModule->libraryHandle); + if (dynamicModule) { + mlir::concretelang::dfr::_dfr_register_lib(dynamicModule->libraryHandle); + } if (!mlir::concretelang::dfr::_dfr_is_root_node()) { mlir::concretelang::dfr::_dfr_run_remote_scheduler(); return returns; @@ -461,7 +463,7 @@ ServerCircuit::call(const ServerKeyset &serverKeyset, } Result> -ServerCircuit::simulate(std::vector &args) { +ServerCircuit::simulate(const std::vector &args) { ServerKeyset emptyKeyset; return call(emptyKeyset, args); } @@ -470,23 +472,13 @@ std::string ServerCircuit::getName() { return circuitInfo.asReader().getName(); } -Result ServerCircuit::fromDynamicModule( +Result ServerCircuit::fromFnPtr( const Message &circuitInfo, - std::shared_ptr dynamicModule, bool useSimulation = false) { - + void (*func)(void *...), bool useSimulation = false) { ServerCircuit output; output.circuitInfo = circuitInfo; output.useSimulation = useSimulation; - output.dynamicModule = dynamicModule; - output.func = (void (*)(void *, ...))dlsym( - dynamicModule->libraryHandle, - (std::string("_mlir_concrete_") + - std::string(circuitInfo.asReader().getName().cStr())) - .c_str()); - if (auto err = dlerror()) { - return StringError("Circuit symbol not found in dynamic module: ") - << std::string(err); - } + output.func = func; // We prepare the args transformers used to transform transport values into // arg values. @@ -558,6 +550,28 @@ Result ServerCircuit::fromDynamicModule( return output; } +Result ServerCircuit::fromDynamicModule( + const Message &circuitInfo, + std::shared_ptr dynamicModule, bool useSimulation = false) { + + auto func = (void (*)(void *, ...))dlsym( + dynamicModule->libraryHandle, + (std::string("_mlir_concrete_") + + std::string(circuitInfo.asReader().getName().cStr())) + .c_str()); + if (auto err = dlerror()) { + return StringError("Circuit symbol not found in dynamic module: ") + << std::string(err); + } + + auto output = ServerCircuit::fromFnPtr(circuitInfo, func, useSimulation); + if (output.has_value()) { + output.value().dynamicModule = dynamicModule; + } + + return output; +} + void ServerCircuit::invoke(const ServerKeyset &serverKeyset) { // We create a runtime context from the keyset, and place a pointer to it in diff --git a/compilers/concrete-compiler/compiler/lib/Support/Pipeline.cpp b/compilers/concrete-compiler/compiler/lib/Support/Pipeline.cpp index d88808f63..b3efe4345 100644 --- a/compilers/concrete-compiler/compiler/lib/Support/Pipeline.cpp +++ b/compilers/concrete-compiler/compiler/lib/Support/Pipeline.cpp @@ -51,7 +51,6 @@ #include "concretelang/Dialect/SDFG/Transforms/Passes.h" #include "concretelang/Dialect/TFHE/Analysis/ExtractStatistics.h" #include "concretelang/Dialect/TFHE/Transforms/Transforms.h" -#include "concretelang/Runtime/utils.h" #include "concretelang/Support/CompilerEngine.h" #include "concretelang/Support/Error.h" #include "concretelang/Support/Pipeline.h" @@ -632,7 +631,8 @@ std::unique_ptr lowerLLVMDialectToLLVMIR(mlir::MLIRContext &context, llvm::LLVMContext &llvmContext, mlir::ModuleOp &module) { - mlir::concretelang::LLVMInitializeNativeTarget(); + llvm::InitializeNativeTarget(); + llvm::InitializeNativeTargetAsmPrinter(); mlir::registerLLVMDialectTranslation(*module->getContext()); mlir::registerOpenMPDialectTranslation(*module->getContext()); diff --git a/compilers/concrete-optimizer/Cargo.lock b/compilers/concrete-optimizer/Cargo.lock index 9a27da08d..10b3d5f68 100644 --- a/compilers/concrete-optimizer/Cargo.lock +++ b/compilers/concrete-optimizer/Cargo.lock @@ -371,6 +371,8 @@ dependencies = [ "concrete-optimizer", "cxx", "cxx-build", + "serde", + "serde_json", ] [[package]] diff --git a/compilers/concrete-optimizer/concrete-optimizer-cpp/Cargo.toml b/compilers/concrete-optimizer/concrete-optimizer-cpp/Cargo.toml index 5666fec85..16930afab 100644 --- a/compilers/concrete-optimizer/concrete-optimizer-cpp/Cargo.toml +++ b/compilers/concrete-optimizer/concrete-optimizer-cpp/Cargo.toml @@ -8,6 +8,8 @@ edition = "2021" [dependencies] cxx = "1" concrete-optimizer = { path = "../concrete-optimizer" } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" [build-dependencies] cxx-build = "1" diff --git a/compilers/concrete-optimizer/concrete-optimizer-cpp/src/concrete-optimizer.rs b/compilers/concrete-optimizer/concrete-optimizer-cpp/src/concrete-optimizer.rs index 7c1e784b4..ef24def3c 100644 --- a/compilers/concrete-optimizer/concrete-optimizer-cpp/src/concrete-optimizer.rs +++ b/compilers/concrete-optimizer/concrete-optimizer-cpp/src/concrete-optimizer.rs @@ -975,6 +975,38 @@ impl Into for ffi::Encoding { } } +impl ffi::RangeRestriction { + fn range_restriction_to_json(&self) -> String { + unsafe { + serde_json::to_string(std::mem::transmute::<&Self, &RangeRestriction>(self)).unwrap() + } + } +} + +fn range_restriction_from_json(input: &str) -> ffi::RangeRestriction { + unsafe { + std::mem::transmute::( + serde_json::from_str(input).unwrap(), + ) + } +} + +impl ffi::KeysetRestriction { + fn keyset_restriction_to_json(&self) -> String { + unsafe { + serde_json::to_string(std::mem::transmute::<&Self, &KeysetRestriction>(self)).unwrap() + } + } +} + +fn keyset_restriction_from_json(input: &str) -> ffi::KeysetRestriction { + unsafe { + std::mem::transmute::( + serde_json::from_str(input).unwrap(), + ) + } +} + #[allow( unused_must_use, clippy::needless_lifetimes, @@ -1156,6 +1188,18 @@ mod ffi { fn get_output_indices(self: &Dag) -> Vec; fn NO_KEY_ID() -> u64; + + #[namespace = "concrete_optimizer::restriction"] + fn range_restriction_to_json(self: &RangeRestriction) -> String; + + #[namespace = "concrete_optimizer::restriction"] + fn range_restriction_from_json(input: &str) -> RangeRestriction; + + #[namespace = "concrete_optimizer::restriction"] + fn keyset_restriction_to_json(self: &KeysetRestriction) -> String; + + #[namespace = "concrete_optimizer::restriction"] + fn keyset_restriction_from_json(input: &str) -> KeysetRestriction; } #[derive(Debug, Clone, Copy)] diff --git a/compilers/concrete-optimizer/concrete-optimizer-cpp/src/cpp/concrete-optimizer.cpp b/compilers/concrete-optimizer/concrete-optimizer-cpp/src/cpp/concrete-optimizer.cpp index 833a42c87..7f2a46eb1 100644 --- a/compilers/concrete-optimizer/concrete-optimizer-cpp/src/cpp/concrete-optimizer.cpp +++ b/compilers/concrete-optimizer/concrete-optimizer-cpp/src/cpp/concrete-optimizer.cpp @@ -1171,6 +1171,7 @@ struct RangeRestriction final { ::rust::Vec<::std::uint64_t> ks_level_count; ::rust::Vec<::std::uint64_t> ks_base_log; + ::rust::String range_restriction_to_json() const noexcept; using IsRelocatable = ::std::true_type; }; #endif // CXXBRIDGE1_STRUCT_concrete_optimizer$restriction$RangeRestriction @@ -1396,6 +1397,7 @@ struct KeysetInfo final { struct KeysetRestriction final { ::concrete_optimizer::restriction::KeysetInfo info; + ::rust::String keyset_restriction_to_json() const noexcept; using IsRelocatable = ::std::true_type; }; #endif // CXXBRIDGE1_STRUCT_concrete_optimizer$restriction$KeysetRestriction @@ -1527,6 +1529,18 @@ void concrete_optimizer$cxxbridge1$Dag$get_output_indices(::concrete_optimizer:: ::std::uint64_t concrete_optimizer$cxxbridge1$NO_KEY_ID() noexcept; } // extern "C" +namespace restriction { +extern "C" { +void concrete_optimizer$restriction$cxxbridge1$RangeRestriction$range_restriction_to_json(::concrete_optimizer::restriction::RangeRestriction const &self, ::rust::String *return$) noexcept; + +void concrete_optimizer$restriction$cxxbridge1$range_restriction_from_json(::rust::Str input, ::concrete_optimizer::restriction::RangeRestriction *return$) noexcept; + +void concrete_optimizer$restriction$cxxbridge1$KeysetRestriction$keyset_restriction_to_json(::concrete_optimizer::restriction::KeysetRestriction const &self, ::rust::String *return$) noexcept; + +void concrete_optimizer$restriction$cxxbridge1$keyset_restriction_from_json(::rust::Str input, ::concrete_optimizer::restriction::KeysetRestriction *return$) noexcept; +} // extern "C" +} // namespace restriction + namespace v0 { ::concrete_optimizer::v0::Solution optimize_bootstrap(::std::uint64_t precision, double noise_factor, ::concrete_optimizer::Options const &options) noexcept { return concrete_optimizer$v0$cxxbridge1$optimize_bootstrap(precision, noise_factor, options); @@ -1741,6 +1755,32 @@ namespace weights { ::std::uint64_t NO_KEY_ID() noexcept { return concrete_optimizer$cxxbridge1$NO_KEY_ID(); } + +namespace restriction { +::rust::String RangeRestriction::range_restriction_to_json() const noexcept { + ::rust::MaybeUninit<::rust::String> return$; + concrete_optimizer$restriction$cxxbridge1$RangeRestriction$range_restriction_to_json(*this, &return$.value); + return ::std::move(return$.value); +} + +::concrete_optimizer::restriction::RangeRestriction range_restriction_from_json(::rust::Str input) noexcept { + ::rust::MaybeUninit<::concrete_optimizer::restriction::RangeRestriction> return$; + concrete_optimizer$restriction$cxxbridge1$range_restriction_from_json(input, &return$.value); + return ::std::move(return$.value); +} + +::rust::String KeysetRestriction::keyset_restriction_to_json() const noexcept { + ::rust::MaybeUninit<::rust::String> return$; + concrete_optimizer$restriction$cxxbridge1$KeysetRestriction$keyset_restriction_to_json(*this, &return$.value); + return ::std::move(return$.value); +} + +::concrete_optimizer::restriction::KeysetRestriction keyset_restriction_from_json(::rust::Str input) noexcept { + ::rust::MaybeUninit<::concrete_optimizer::restriction::KeysetRestriction> return$; + concrete_optimizer$restriction$cxxbridge1$keyset_restriction_from_json(input, &return$.value); + return ::std::move(return$.value); +} +} // namespace restriction } // namespace concrete_optimizer extern "C" { diff --git a/compilers/concrete-optimizer/concrete-optimizer-cpp/src/cpp/concrete-optimizer.hpp b/compilers/concrete-optimizer/concrete-optimizer-cpp/src/cpp/concrete-optimizer.hpp index e14171d8a..4ec5b9230 100644 --- a/compilers/concrete-optimizer/concrete-optimizer-cpp/src/cpp/concrete-optimizer.hpp +++ b/compilers/concrete-optimizer/concrete-optimizer-cpp/src/cpp/concrete-optimizer.hpp @@ -1145,6 +1145,7 @@ struct RangeRestriction final { ::rust::Vec<::std::uint64_t> ks_level_count; ::rust::Vec<::std::uint64_t> ks_base_log; + ::rust::String range_restriction_to_json() const noexcept; using IsRelocatable = ::std::true_type; }; #endif // CXXBRIDGE1_STRUCT_concrete_optimizer$restriction$RangeRestriction @@ -1370,6 +1371,7 @@ struct KeysetInfo final { struct KeysetRestriction final { ::concrete_optimizer::restriction::KeysetInfo info; + ::rust::String keyset_restriction_to_json() const noexcept; using IsRelocatable = ::std::true_type; }; #endif // CXXBRIDGE1_STRUCT_concrete_optimizer$restriction$KeysetRestriction @@ -1418,4 +1420,10 @@ namespace weights { } // namespace weights ::std::uint64_t NO_KEY_ID() noexcept; + +namespace restriction { +::concrete_optimizer::restriction::RangeRestriction range_restriction_from_json(::rust::Str input) noexcept; + +::concrete_optimizer::restriction::KeysetRestriction keyset_restriction_from_json(::rust::Str input) noexcept; +} // namespace restriction } // namespace concrete_optimizer diff --git a/compilers/concrete-optimizer/concrete-optimizer/src/optimization/dag/multi_parameters/optimize/restriction.rs b/compilers/concrete-optimizer/concrete-optimizer/src/optimization/dag/multi_parameters/optimize/restriction.rs index aea75f68b..f00bcb248 100644 --- a/compilers/concrete-optimizer/concrete-optimizer/src/optimization/dag/multi_parameters/optimize/restriction.rs +++ b/compilers/concrete-optimizer/concrete-optimizer/src/optimization/dag/multi_parameters/optimize/restriction.rs @@ -4,6 +4,7 @@ use crate::{ }, parameters::{BrDecompositionParameters, GlweParameters, KsDecompositionParameters}, }; +use serde::{Deserialize, Serialize}; use super::MacroParameters; @@ -224,6 +225,7 @@ impl SearchSpaceRestriction for NoSearchSpaceRestriction { } /// An object restricting the search space based on smaller ranges. +#[derive(Serialize, Deserialize)] pub struct RangeRestriction { pub glwe_log_polynomial_sizes: Vec, pub glwe_dimensions: Vec, @@ -315,10 +317,12 @@ impl SearchSpaceRestriction for RangeRestriction { } #[allow(unused)] +#[derive(Serialize, Deserialize)] pub struct LweSecretKeyInfo { lwe_dimension: u64, } +#[derive(Serialize, Deserialize)] pub struct LweBootstrapKeyInfo { level_count: u64, base_log: u64, @@ -327,6 +331,7 @@ pub struct LweBootstrapKeyInfo { input_lwe_dimension: u64, } +#[derive(Serialize, Deserialize)] pub struct LweKeyswitchKeyInfo { level_count: u64, base_log: u64, @@ -335,6 +340,7 @@ pub struct LweKeyswitchKeyInfo { } #[allow(unused)] +#[derive(Serialize, Deserialize)] pub struct KeysetInfo { lwe_secret_keys: Vec, lwe_bootstrap_keys: Vec, @@ -342,6 +348,7 @@ pub struct KeysetInfo { } /// An object restricting the search space based on a keyset. +#[derive(Serialize, Deserialize)] pub struct KeysetRestriction { info: KeysetInfo, } diff --git a/docker/Dockerfile.concrete-compiler-env b/docker/Dockerfile.concrete-compiler-env index 5abb16e6f..8fea3da8d 100644 --- a/docker/Dockerfile.concrete-compiler-env +++ b/docker/Dockerfile.concrete-compiler-env @@ -7,6 +7,8 @@ RUN dnf update -y && dnf install -y ninja-build hwloc-devel ccache ncurses-devel RUN mkdir -p ~/.ssh/ && ssh-keyscan -t ecdsa github.com >> ~/.ssh/known_hosts # Setup gcc-11 (required for cuda11.8) RUN dnf install -y gcc-toolset-11 && dnf clean all +# Setup ssl +RUN dnf install -y libatomic openssl openssl-devel && dnf clean all ENV CC_COMPILER=/opt/rh/gcc-toolset-11/root/usr/bin/gcc ENV CXX_COMPILER=/opt/rh/gcc-toolset-11/root/usr/bin/c++ ENV PATH "/opt/rh/gcc-toolset-11/root/usr/bin:$PATH" @@ -18,6 +20,8 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ENV PATH=/root/.cargo/bin:$PATH RUN rustup install nightly-2024-09-30 +RUN rustup install nightly +RUN cargo install cxxbridge-cmd # Install Cap'n Proto ENV CAPNP_VERSION=1.1.0 # hadolint ignore=DL3003 From d1ec15d0a52fa65f57de40503618f5a9f229d6fe Mon Sep 17 00:00:00 2001 From: Quentin Bourgerie Date: Tue, 1 Apr 2025 15:25:08 +0200 Subject: [PATCH 07/17] chore(ci): Update benchmaks backend by updating the ami --- ci/slab.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/slab.toml b/ci/slab.toml index 41985791a..636f8b476 100644 --- a/ci/slab.toml +++ b/ci/slab.toml @@ -5,8 +5,8 @@ instance_type = "m7i.16xlarge" security_group = ["sg-0e55cc31dfda0d8a7", ] [backend.aws.cpu-bench] -region = "eu-west-1" -image_id = "ami-002bdcd64b8472cf9" +region = "eu-west-3" +image_id = "ami-0d87d688748d653f7" # Based on Ubuntu 24.04 instance_type = "hpc7a.96xlarge" [backend.aws.gpu-test] From 84f7fe1804c90d589ccc4e8813f1899c7f92917a Mon Sep 17 00:00:00 2001 From: Quentin Bourgerie Date: Tue, 1 Apr 2025 16:15:12 +0200 Subject: [PATCH 08/17] chore(ci): Add pip install requirements in the macos workflow --- .../concrete_compiler_test_cpu_distributed.yml | 16 ++++++++-------- .../concrete_compiler_test_macos_cpu.yml | 15 +++++++++------ 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/workflows/concrete_compiler_test_cpu_distributed.yml b/.github/workflows/concrete_compiler_test_cpu_distributed.yml index f4b80000c..487d822a4 100644 --- a/.github/workflows/concrete_compiler_test_cpu_distributed.yml +++ b/.github/workflows/concrete_compiler_test_cpu_distributed.yml @@ -2,14 +2,14 @@ name: concrete-compiler test linux-cpu-distributed on: workflow_dispatch: - pull_request: - paths: - - .github/workflows/concrete_compiler_test_cpu_distributed.yml - - compilers/concrete-compiler/** - push: - branches: - - 'main' - - 'release/*' + # pull_request: + # paths: + # - .github/workflows/concrete_compiler_test_cpu_distributed.yml + # - compilers/concrete-compiler/** + # push: + # branches: + # - 'main' + # - 'release/*' env: ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/concrete_compiler_test_macos_cpu.yml b/.github/workflows/concrete_compiler_test_macos_cpu.yml index 0b9e18efb..8a415471d 100644 --- a/.github/workflows/concrete_compiler_test_macos_cpu.yml +++ b/.github/workflows/concrete_compiler_test_macos_cpu.yml @@ -36,6 +36,10 @@ jobs: outputs: slack_message: ${{ steps.prepare_slack_notif.outputs.slack_message }} slack_color: ${{ steps.prepare_slack_notif.outputs.slack_color }} + env: + pip: pip${{ matrix.python-version }} + python: python${{ matrix.python-version }} + concrete-compiler-dir: ${{ github.workspace }}/compilers/concrete-compiler/compiler steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -46,18 +50,17 @@ jobs: - name: Set python variables run: | { - echo "PIP=${{ format('pip{0}', matrix.python-version) }}" - echo "PYTHON=${{ format('python{0}', matrix.python-version) }}" - echo "PYTHON_EXEC=$(which ${{ format('python{0}', matrix.python-version) }})" + echo "PYTHON_EXEC=$(which ${{ env.python }})" } >> "${GITHUB_ENV}" - name: Install dependencies run: | brew install ninja ccache zstd - ${PIP} install pytest + ${{ env.pip }} install -r ${{ env.concrete-compiler-dir }}/lib/Bindings/Python/requirements_dev.txt + ${{ env.pip }} install -r ${{ env.concrete-compiler-dir }}/../llvm-project/mlir/python/requirements.txt - name: Build compiler run: | set -e - cd compilers/concrete-compiler/compiler + cd ${{ env.concrete-compiler-dir }} echo "Debug: ccache statistics (prior to the build):" ccache -s cargo install cxxbridge-cmd @@ -80,7 +83,7 @@ jobs: - name: Test run: | set -e - cd compilers/concrete-compiler/compiler + cd ${{ env.concrete-compiler-dir }} export CONCRETE_COMPILER_DATAFLOW_EXECUTION_ENABLED=OFF make MINIMAL_TESTS=${{ env.MINIMAL_TESTS }} Python3_EXECUTABLE="${PYTHON_EXEC}" run-tests - name: Cleanup host From d3bd9ae3d09a33e3f60ba99436b1a31296f56547 Mon Sep 17 00:00:00 2001 From: Quentin Bourgerie Date: Tue, 1 Apr 2025 17:38:04 +0200 Subject: [PATCH 09/17] chore(ci): Update cpu backend by updating the ami --- ci/slab.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/slab.toml b/ci/slab.toml index 636f8b476..69a062114 100644 --- a/ci/slab.toml +++ b/ci/slab.toml @@ -1,6 +1,6 @@ [backend.aws.cpu-test] -region = "eu-west-1" -image_id = "ami-002bdcd64b8472cf9" # Based on Ubuntu 22.4 +region = "eu-west-3" +image_id = "ami-0d87d688748d653f7" # Based on Ubuntu 24.04 instance_type = "m7i.16xlarge" security_group = ["sg-0e55cc31dfda0d8a7", ] @@ -25,6 +25,6 @@ security_group= ["sg-02dd8470fa845f31b", ] runner_name = "distributed-ci" [backend.aws.release] -region = "eu-west-1" -image_id = "ami-002bdcd64b8472cf9" +region = "eu-west-3" +image_id = "ami-0d87d688748d653f7" instance_type = "hpc7a.96xlarge" From c2c65d028cea2d19bf9472ab0e5f0be7b0034aa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexandre=20P=C3=A9r=C3=A9?= Date: Thu, 3 Apr 2025 17:05:31 +0200 Subject: [PATCH 10/17] chore(optimizer): fix clippy lints --- .../concrete-optimizer/src/dag/unparametrized.rs | 2 +- .../src/optimization/dag/multi_parameters/analyze.rs | 4 ++-- .../src/optimization/dag/solo_key/analyze.rs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/compilers/concrete-optimizer/concrete-optimizer/src/dag/unparametrized.rs b/compilers/concrete-optimizer/concrete-optimizer/src/dag/unparametrized.rs index c96fc2a92..e06225f6b 100644 --- a/compilers/concrete-optimizer/concrete-optimizer/src/dag/unparametrized.rs +++ b/compilers/concrete-optimizer/concrete-optimizer/src/dag/unparametrized.rs @@ -481,7 +481,7 @@ impl DagBuilder<'_> { .. } => shape.clone(), Operator::Dot { - kind: DotKind::Unsupported { .. }, + kind: DotKind::Unsupported, weights, inputs, } => { diff --git a/compilers/concrete-optimizer/concrete-optimizer/src/optimization/dag/multi_parameters/analyze.rs b/compilers/concrete-optimizer/concrete-optimizer/src/optimization/dag/multi_parameters/analyze.rs index 9cdb33844..f3a194e9f 100644 --- a/compilers/concrete-optimizer/concrete-optimizer/src/optimization/dag/multi_parameters/analyze.rs +++ b/compilers/concrete-optimizer/concrete-optimizer/src/optimization/dag/multi_parameters/analyze.rs @@ -342,11 +342,11 @@ impl VariancedDag { Operator::Input { .. } | Operator::ZeroNoise { .. } => unreachable!(), Operator::Dot { - kind: DotKind::CompatibleTensor { .. }, + kind: DotKind::CompatibleTensor, .. } => todo!("TODO"), Operator::Dot { - kind: DotKind::Unsupported { .. }, + kind: DotKind::Unsupported, .. } => panic!("Unsupported"), Operator::Round { .. } => { diff --git a/compilers/concrete-optimizer/concrete-optimizer/src/optimization/dag/solo_key/analyze.rs b/compilers/concrete-optimizer/concrete-optimizer/src/optimization/dag/solo_key/analyze.rs index 96b04008a..7a2498a40 100644 --- a/compilers/concrete-optimizer/concrete-optimizer/src/optimization/dag/solo_key/analyze.rs +++ b/compilers/concrete-optimizer/concrete-optimizer/src/optimization/dag/solo_key/analyze.rs @@ -169,11 +169,11 @@ fn out_variance( }) } Operator::Dot { - kind: DotKind::CompatibleTensor { .. }, + kind: DotKind::CompatibleTensor, .. } => todo!("TODO"), Operator::Dot { - kind: DotKind::Unsupported { .. }, + kind: DotKind::Unsupported, .. } => panic!("Unsupported"), Operator::Dot { From d605821c5651da1f7e6acaab4ff6a49062c09008 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexandre=20P=C3=A9r=C3=A9?= Date: Thu, 17 Apr 2025 17:11:30 +0200 Subject: [PATCH 11/17] chore(ci): fix broken wheel repair --- .../compiler/lib/Bindings/Python/CMakeLists.txt | 1 - frontends/concrete-python/Makefile | 11 ++++++++++- frontends/concrete-python/requirements.dev.txt | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Python/CMakeLists.txt b/compilers/concrete-compiler/compiler/lib/Bindings/Python/CMakeLists.txt index 61a392293..3174ec532 100644 --- a/compilers/concrete-compiler/compiler/lib/Bindings/Python/CMakeLists.txt +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Python/CMakeLists.txt @@ -112,7 +112,6 @@ target_include_directories(ConcretelangBindingsPythonCAPI PUBLIC ${CONCRETE_CPU_ target_link_libraries( ConcretelangBindingsPythonCAPI - PUBLIC ConcretelangRuntime PRIVATE $ $ $) diff --git a/frontends/concrete-python/Makefile b/frontends/concrete-python/Makefile index 02eaabe5c..28f9f874b 100644 --- a/frontends/concrete-python/Makefile +++ b/frontends/concrete-python/Makefile @@ -235,7 +235,16 @@ build-whl: patch-whl-linux: GLIBC_VER=$(shell ldd --version | head -n 1 | grep -o '[^ ]*$$'|head|tr '.' '_'); \ for PLATFORM in manylinux_$${GLIBC_VER}_x86_64 linux_x86_64; do \ - if $(PYTHON) -m auditwheel repair -w dist --plat $$PLATFORM dist/*.whl; then \ + if $(PYTHON) -m auditwheel repair \ + -w dist \ + --exclude "libstdc++.so.*" \ + --exclude "libm.so.*" \ + --exclude "libc.so.*" \ + --exclude "libgcc_s.so.*" \ + --exclude "libcap.so*" \ + --exclude "libudev.so*" \ + --plat $$PLATFORM dist/*.whl;\ + then \ echo Success for $$PLATFORM; \ break; \ else \ diff --git a/frontends/concrete-python/requirements.dev.txt b/frontends/concrete-python/requirements.dev.txt index b2426be9c..1198f3c1c 100644 --- a/frontends/concrete-python/requirements.dev.txt +++ b/frontends/concrete-python/requirements.dev.txt @@ -13,7 +13,7 @@ pydocstyle==6.3.0 pylint==2.17.1 ruff==0.6.3 -auditwheel==5.3.0; sys_platform == 'linux' +auditwheel==6.3.0; sys_platform == 'linux' delocate==0.10.4; sys_platform == 'darwin' wheel==0.40.0 From 1d69046bc033b914d04a8b013019d542740165ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexandre=20P=C3=A9r=C3=A9?= Date: Fri, 18 Apr 2025 14:21:38 +0200 Subject: [PATCH 12/17] chore(frontends): move rust front to the frontends directory --- compilers/concrete-compiler/compiler/Makefile | 8 +- .../compiler/lib/Bindings/Rust/.gitignore | 4 +- .../compiler/lib/Bindings/Rust/CMakeLists.txt | 19 +- .../compiler/lib/Bindings/Rust/Cargo.toml | 3 - .../compiler/lib/Bindings/Rust/NOTES.md | 41 - .../lib/Bindings/Rust/concrete/src/cxx.h | 1115 ----------------- .../compiler/lib/Bindings/Rust/ffi.h | 1 + .../compiler/lib/Bindings/Rust/ffi.rs | 1 + frontends/concrete-rust/Makefile | 13 +- .../concrete-rust/concrete-keygen/.gitignore | 1 + .../{ => concrete-keygen}/Cargo.lock | 0 .../{ => concrete-keygen}/Cargo.toml | 0 .../concrete-rust/concrete-keygen/Makefile | 7 + .../{ => concrete-keygen}/build.rs | 0 .../capnp/concrete-protocol.capnp | 0 .../{ => concrete-keygen}/src/keygen.rs | 0 .../{ => concrete-keygen}/src/lib.rs | 0 .../tests/keygen_wasm.js | 0 .../concrete-rust}/concrete-macro/Cargo.toml | 0 .../concrete-rust}/concrete-macro/build.rs | 0 .../concrete-macro/src/configuration.rs | 0 .../concrete-macro/src/fast_path_hasher.rs | 0 .../concrete-macro/src/generation.rs | 0 .../concrete-rust}/concrete-macro/src/lib.rs | 0 .../concrete-macro/src/unzip.rs | 0 .../concrete-rust}/concrete/.gitignore | 0 .../concrete-rust}/concrete/Cargo.toml | 0 .../concrete-rust}/concrete/build.rs | 0 .../concrete-rust}/concrete/src/.gitignore | 0 .../concrete-rust}/concrete/src/ffi.h | 0 .../concrete-rust}/concrete/src/ffi.rs | 0 .../concrete-rust}/concrete/src/lib.rs | 0 .../concrete-rust}/concrete/src/protocol.rs | 0 .../concrete-rust}/rust-toolchain.toml | 0 .../concrete-rust}/test/Cargo.toml | 0 .../concrete-rust}/test/src/lib.rs | 0 .../concrete-rust}/test/src/main.rs | 0 .../concrete-rust}/test/src/test.zip | Bin 38 files changed, 25 insertions(+), 1188 deletions(-) delete mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/Cargo.toml delete mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/NOTES.md delete mode 100644 compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/src/cxx.h create mode 120000 compilers/concrete-compiler/compiler/lib/Bindings/Rust/ffi.h create mode 120000 compilers/concrete-compiler/compiler/lib/Bindings/Rust/ffi.rs create mode 100644 frontends/concrete-rust/concrete-keygen/.gitignore rename frontends/concrete-rust/{ => concrete-keygen}/Cargo.lock (100%) rename frontends/concrete-rust/{ => concrete-keygen}/Cargo.toml (100%) create mode 100644 frontends/concrete-rust/concrete-keygen/Makefile rename frontends/concrete-rust/{ => concrete-keygen}/build.rs (100%) rename frontends/concrete-rust/{ => concrete-keygen}/capnp/concrete-protocol.capnp (100%) rename frontends/concrete-rust/{ => concrete-keygen}/src/keygen.rs (100%) rename frontends/concrete-rust/{ => concrete-keygen}/src/lib.rs (100%) rename frontends/concrete-rust/{ => concrete-keygen}/tests/keygen_wasm.js (100%) rename {compilers/concrete-compiler/compiler/lib/Bindings/Rust => frontends/concrete-rust}/concrete-macro/Cargo.toml (100%) rename {compilers/concrete-compiler/compiler/lib/Bindings/Rust => frontends/concrete-rust}/concrete-macro/build.rs (100%) rename {compilers/concrete-compiler/compiler/lib/Bindings/Rust => frontends/concrete-rust}/concrete-macro/src/configuration.rs (100%) rename {compilers/concrete-compiler/compiler/lib/Bindings/Rust => frontends/concrete-rust}/concrete-macro/src/fast_path_hasher.rs (100%) rename {compilers/concrete-compiler/compiler/lib/Bindings/Rust => frontends/concrete-rust}/concrete-macro/src/generation.rs (100%) rename {compilers/concrete-compiler/compiler/lib/Bindings/Rust => frontends/concrete-rust}/concrete-macro/src/lib.rs (100%) rename {compilers/concrete-compiler/compiler/lib/Bindings/Rust => frontends/concrete-rust}/concrete-macro/src/unzip.rs (100%) rename {compilers/concrete-compiler/compiler/lib/Bindings/Rust => frontends/concrete-rust}/concrete/.gitignore (100%) rename {compilers/concrete-compiler/compiler/lib/Bindings/Rust => frontends/concrete-rust}/concrete/Cargo.toml (100%) rename {compilers/concrete-compiler/compiler/lib/Bindings/Rust => frontends/concrete-rust}/concrete/build.rs (100%) rename {compilers/concrete-compiler/compiler/lib/Bindings/Rust => frontends/concrete-rust}/concrete/src/.gitignore (100%) rename {compilers/concrete-compiler/compiler/lib/Bindings/Rust => frontends/concrete-rust}/concrete/src/ffi.h (100%) rename {compilers/concrete-compiler/compiler/lib/Bindings/Rust => frontends/concrete-rust}/concrete/src/ffi.rs (100%) rename {compilers/concrete-compiler/compiler/lib/Bindings/Rust => frontends/concrete-rust}/concrete/src/lib.rs (100%) rename {compilers/concrete-compiler/compiler/lib/Bindings/Rust => frontends/concrete-rust}/concrete/src/protocol.rs (100%) rename {compilers/concrete-compiler/compiler/lib/Bindings/Rust => frontends/concrete-rust}/rust-toolchain.toml (100%) rename {compilers/concrete-compiler/compiler/lib/Bindings/Rust => frontends/concrete-rust}/test/Cargo.toml (100%) rename {compilers/concrete-compiler/compiler/lib/Bindings/Rust => frontends/concrete-rust}/test/src/lib.rs (100%) rename {compilers/concrete-compiler/compiler/lib/Bindings/Rust => frontends/concrete-rust}/test/src/main.rs (100%) rename {compilers/concrete-compiler/compiler/lib/Bindings/Rust => frontends/concrete-rust}/test/src/test.zip (100%) diff --git a/compilers/concrete-compiler/compiler/Makefile b/compilers/concrete-compiler/compiler/Makefile index 417508fdd..9ce1a3d38 100644 --- a/compilers/concrete-compiler/compiler/Makefile +++ b/compilers/concrete-compiler/compiler/Makefile @@ -212,7 +212,7 @@ endif build-tests: build-unit-tests build-end-to-end-tests -run-tests: run-check-tests run-unit-tests run-end-to-end-tests run-random-end-to-end-tests-for-each-options run-python-tests run-rust-tests +run-tests: run-check-tests run-unit-tests run-end-to-end-tests run-random-end-to-end-tests-for-each-options run-python-tests ## check-tests @@ -232,12 +232,6 @@ run-unit-tests: build-unit-tests run-python-tests: python-bindings concretecompiler runtime PYTHONPATH=${PYTHONPATH}:$(BUILD_DIR)/tools/concretelang/python_packages/concretelang_core LD_PRELOAD=$(BUILD_DIR)/lib/libConcretelangRuntime.so pytest -vs -m $(PYTHON_TESTS_MARKER) tests/python -## rust-tests - -run-rust-tests: concrete-rust - $(eval _abs_build_dir = $(realpath $(BUILD_DIR))) - cd lib/Bindings/Rust && COMPILER_BUILD_DIRECTORY=$(_abs_build_dir) cargo +nightly test - test-compiler-file-output: concretecompiler pytest -vs tests/test_compiler_file_output diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/.gitignore b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/.gitignore index c34b8d3a6..225fd3d98 100644 --- a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/.gitignore +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/.gitignore @@ -1,3 +1 @@ -target -Cargo.lock -.cargo +cxx.h diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/CMakeLists.txt b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/CMakeLists.txt index b40fa0ce2..896b8f034 100644 --- a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/CMakeLists.txt +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/CMakeLists.txt @@ -1,14 +1,12 @@ add_compile_options(-fexceptions) add_custom_command( - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/concrete/src/ffi.rs ${CMAKE_CURRENT_SOURCE_DIR}/concrete/src/ffi.h - OUTPUT concrete/src/ffi.rs.cc ${CMAKE_CURRENT_SOURCE_DIR}/concrete/src/cxx.h - COMMAND cxxbridge ${CMAKE_CURRENT_SOURCE_DIR}/concrete/src/ffi.rs > concrete/src/ffi.rs.cc - COMMAND cxxbridge --header > ${CMAKE_CURRENT_SOURCE_DIR}/concrete/src/cxx.h) + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/ffi.rs ${CMAKE_CURRENT_SOURCE_DIR}/ffi.h + OUTPUT ffi.rs.cc ${CMAKE_CURRENT_SOURCE_DIR}/cxx.h + COMMAND cxxbridge ${CMAKE_CURRENT_SOURCE_DIR}/ffi.rs > ffi.rs.cc + COMMAND cxxbridge --header > ${CMAKE_CURRENT_SOURCE_DIR}/cxx.h) -add_library(ConcreteRust SHARED concrete/src/ffi.h concrete/src/ffi.rs.cc) - -target_include_directories(ConcreteRust PRIVATE concrete/src) +add_library(ConcreteRust SHARED ffi.rs.cc) if(LINUX) set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-Bsymbolic") @@ -17,14 +15,9 @@ endif() target_link_libraries( ConcreteRust PRIVATE ConcretelangSupport - ConcretelangCommon ConcretelangClientLib ConcretelangServerLib - ConcretelangRuntimeStatic - LLVMSupport - capnp - capnp-json - kj) + ConcretelangRuntimeStatic) if(APPLE) find_library(SECURITY_FRAMEWORK Security) diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/Cargo.toml b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/Cargo.toml deleted file mode 100644 index 20142f1e7..000000000 --- a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/Cargo.toml +++ /dev/null @@ -1,3 +0,0 @@ -[workspace] -resolver = "2" -members = ["concrete", "concrete-macro", "test"] diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/NOTES.md b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/NOTES.md deleted file mode 100644 index e863f8ec3..000000000 --- a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/NOTES.md +++ /dev/null @@ -1,41 +0,0 @@ -# Developer notes - -## `Cargo` vs `CMake` - -Cargo/Rust and Cmake/C++ projects follows an almost opposite philosophy: -+ Cargo: dependency management is handled internally and the linking is mostly static (actually, it is more than that, it is bundled). -+ Cmake: dependency management is left to the environment and linking is mostly dynamic (no portable solution for bundled static libs). - -Cargo promotes a static (bundled) linking model to manage the dependencies of a Rust project. Specifically, the default behavior for rust libraries, is to statically bundle all the dependencies inside of the resulting `.rlib` file. At the cost of increasing the weights of the intermediate artifacts (a rather small price to pay given the cost of storage nowadays), cargo solves a lots of link-time and run-time burdens with this approach. All the dependencies of a project are pulled and built automatically, and it is rarely needed to tweak `LD_LIBRARY_PATH` to run an application. Also, resulting binaries are portable. - -Cmake promotes a dynamic linking model, in which shared object pre-built binaries are distributed via distribution package managers, and dependencies are resolved at run-time thanks to a dynamic linker. Some recent features of cmake partially help, but there is no real dependency management, no support for building bundled static libraries (static libraries containing all dependencies), and so on. - -As of now, it seems essentially impossible to reliably build an artifact for all the versions of the concrete compiler that can be statically linked into a rust library. For this reason, the concrete compiler artifacts coming from the Cmake world can only be shared objects dynamically linked to rust libraries (with all the book-keeping it brings at run-time to ensure all dependencies are available). - -## `ConcreteSys` and `concrete-sys` - -`concrete-sys` is a thin binding providing the _compilation_ features of concrete. Ideally, following the cargo philosophy, it should be up to cargo to pull the concrete sources and build the necessary artifacts to be linked with `concrete-sys`. In practice, this is not doable given how long it takes to build `llvm`. - -For this reason we pre-build, with CMake, a target specifically for the `concrete-sys` crate, which will then be pulled from the `github` repository at build-time. This Cmake library target is referred to as `ConcreteSys` (CamelCase) and is defined in the workspace level `CMakeLists.txt` file. - -In order for `concrete-sys` (the rust crate) to bind against `ConcreteSys` (the cmake library target), we need to define an interface which can be compiled on the CMake/C++ side, and used on the cargo/rust side. This interface is defined in three file: -+ `concrete-sys/lib.h` and `concrete-sys/lib.rs` which kind of mirror each others for the cpp and the rust side of the interface. -+ `concrete-sys/lib.cpp` which contains the sources of the interface, and are compiled by cmake to the `ConcreteSys` target. - -In practice, other files automatically generated by `CXX` (the rust-cpp binding tool we use), need to be included when compiling the interface. Those are automatically generated with the `cxx-bridge` command by CMake. - -Assuming the `libConcreteSys.so` artifact is built, we rely on the `concrete-sys/build.rs` to bring it to the `target` folder (either from a local build directory, or from the distributed binaries). - -## Non-transitive link-args for `concrete-sys` users. - -`concrete-sys` is built as an `rlib`, and contains a dynamic link dependency on `libConcreteSys.so` file. - -On macos, this means that for the crates depending on `concrete-sys`, they will contain a dependency to `@rpath/libConcreteSys.dylib`. The `@rpath` is a variable that can be set by the linker to help the dynamic linker locate the proper dynamic library. The dynamic linker will try the usual system-wide paths, and the different `@rpath` specified in the binary (`@rpath` can be a list of directory). Those can be set by adding link args such as `-Wl,-rpath,path_to_dylib`. - -In our case, since the `libConcreteSys.so` is not distributed by system package-managers but by cargo which will pull it to the `target` dir, it will not be possible for the dynamic linker to find it in the usual system locations (`/usr/lib/`, `/usr/local/lib` etc). We have to set the `@rpath` properly when linking the final users of the `concrete-sys` crate, for the dynamic linker to find the shared object. - -Unfortunately, `rustc-link-args` are not transitively propagated to dependent crates (See https://github.com/rust-lang/cargo/issues/9554). This means that emitting `-Wl,-rpath,path_to_dylib` from the `concrete-sys/build.rs` script won't set the `@rpath` in the user crates. The link arg must be provided when linking the final user of the `concrete-sys` crate. - -In our case, `concrete-macro` is the main user of the `concrete-sys` crate. Since it contains procedural macros (which are under the hood compiled to `dylib`), we _must_ provide the proper link args when building it. This is achieved in two steps: -+ In the `concrete-sys/build.rs` script, the `libConcreteSys.dylib` artifact is pulled to the root of the target dir (either `target/debug` or `target/release`). -+ In the `concrete-macro/build.rs` script, link args setting the `@rpath` to the target root are issued. diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/src/cxx.h b/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/src/cxx.h deleted file mode 100644 index 3414e4c8a..000000000 --- a/compilers/concrete-compiler/compiler/lib/Bindings/Rust/concrete/src/cxx.h +++ /dev/null @@ -1,1115 +0,0 @@ -#pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if defined(_WIN32) -#include -#else -#include -#endif - -namespace rust { -inline namespace cxxbridge1 { - -struct unsafe_bitcopy_t; - -namespace { -template -class impl; -} - -#ifndef CXXBRIDGE1_RUST_STRING -#define CXXBRIDGE1_RUST_STRING -// https://cxx.rs/binding/string.html -class String final { -public: - String() noexcept; - String(const String &) noexcept; - String(String &&) noexcept; - ~String() noexcept; - - String(const std::string &); - String(const char *); - String(const char *, std::size_t); - String(const char16_t *); - String(const char16_t *, std::size_t); - - // Replace invalid Unicode data with the replacement character (U+FFFD). - static String lossy(const std::string &) noexcept; - static String lossy(const char *) noexcept; - static String lossy(const char *, std::size_t) noexcept; - static String lossy(const char16_t *) noexcept; - static String lossy(const char16_t *, std::size_t) noexcept; - - String &operator=(const String &) &noexcept; - String &operator=(String &&) &noexcept; - - explicit operator std::string() const; - - // Note: no null terminator. - const char *data() const noexcept; - std::size_t size() const noexcept; - std::size_t length() const noexcept; - bool empty() const noexcept; - - const char *c_str() noexcept; - - std::size_t capacity() const noexcept; - void reserve(size_t new_cap) noexcept; - - using iterator = char *; - iterator begin() noexcept; - iterator end() noexcept; - - using const_iterator = const char *; - const_iterator begin() const noexcept; - const_iterator end() const noexcept; - const_iterator cbegin() const noexcept; - const_iterator cend() const noexcept; - - bool operator==(const String &) const noexcept; - bool operator!=(const String &) const noexcept; - bool operator<(const String &) const noexcept; - bool operator<=(const String &) const noexcept; - bool operator>(const String &) const noexcept; - bool operator>=(const String &) const noexcept; - - void swap(String &) noexcept; - - // Internal API only intended for the cxxbridge code generator. - String(unsafe_bitcopy_t, const String &) noexcept; - -private: - struct lossy_t; - String(lossy_t, const char *, std::size_t) noexcept; - String(lossy_t, const char16_t *, std::size_t) noexcept; - friend void swap(String &lhs, String &rhs) noexcept { lhs.swap(rhs); } - - // Size and alignment statically verified by rust_string.rs. - std::array repr; -}; -#endif // CXXBRIDGE1_RUST_STRING - -#ifndef CXXBRIDGE1_RUST_STR -#define CXXBRIDGE1_RUST_STR -// https://cxx.rs/binding/str.html -class Str final { -public: - Str() noexcept; - Str(const String &) noexcept; - Str(const std::string &); - Str(const char *); - Str(const char *, std::size_t); - - Str &operator=(const Str &) &noexcept = default; - - explicit operator std::string() const; - - // Note: no null terminator. - const char *data() const noexcept; - std::size_t size() const noexcept; - std::size_t length() const noexcept; - bool empty() const noexcept; - - // Important in order for System V ABI to pass in registers. - Str(const Str &) noexcept = default; - ~Str() noexcept = default; - - using iterator = const char *; - using const_iterator = const char *; - const_iterator begin() const noexcept; - const_iterator end() const noexcept; - const_iterator cbegin() const noexcept; - const_iterator cend() const noexcept; - - bool operator==(const Str &) const noexcept; - bool operator!=(const Str &) const noexcept; - bool operator<(const Str &) const noexcept; - bool operator<=(const Str &) const noexcept; - bool operator>(const Str &) const noexcept; - bool operator>=(const Str &) const noexcept; - - void swap(Str &) noexcept; - -private: - class uninit; - Str(uninit) noexcept; - friend impl; - - std::array repr; -}; -#endif // CXXBRIDGE1_RUST_STR - -#ifndef CXXBRIDGE1_RUST_SLICE -namespace detail { -template -struct copy_assignable_if {}; - -template <> -struct copy_assignable_if { - copy_assignable_if() noexcept = default; - copy_assignable_if(const copy_assignable_if &) noexcept = default; - copy_assignable_if &operator=(const copy_assignable_if &) &noexcept = delete; - copy_assignable_if &operator=(copy_assignable_if &&) &noexcept = default; -}; -} // namespace detail - -// https://cxx.rs/binding/slice.html -template -class Slice final - : private detail::copy_assignable_if::value> { -public: - using value_type = T; - - Slice() noexcept; - Slice(T *, std::size_t count) noexcept; - - template - explicit Slice(C& c) : Slice(c.data(), c.size()) {} - - Slice &operator=(const Slice &) &noexcept = default; - Slice &operator=(Slice &&) &noexcept = default; - - T *data() const noexcept; - std::size_t size() const noexcept; - std::size_t length() const noexcept; - bool empty() const noexcept; - - T &operator[](std::size_t n) const noexcept; - T &at(std::size_t n) const; - T &front() const noexcept; - T &back() const noexcept; - - // Important in order for System V ABI to pass in registers. - Slice(const Slice &) noexcept = default; - ~Slice() noexcept = default; - - class iterator; - iterator begin() const noexcept; - iterator end() const noexcept; - - void swap(Slice &) noexcept; - -private: - class uninit; - Slice(uninit) noexcept; - friend impl; - friend void sliceInit(void *, const void *, std::size_t) noexcept; - friend void *slicePtr(const void *) noexcept; - friend std::size_t sliceLen(const void *) noexcept; - - std::array repr; -}; - -template -class Slice::iterator final { -public: - using iterator_category = std::random_access_iterator_tag; - using value_type = T; - using difference_type = std::ptrdiff_t; - using pointer = typename std::add_pointer::type; - using reference = typename std::add_lvalue_reference::type; - - reference operator*() const noexcept; - pointer operator->() const noexcept; - reference operator[](difference_type) const noexcept; - - iterator &operator++() noexcept; - iterator operator++(int) noexcept; - iterator &operator--() noexcept; - iterator operator--(int) noexcept; - - iterator &operator+=(difference_type) noexcept; - iterator &operator-=(difference_type) noexcept; - iterator operator+(difference_type) const noexcept; - iterator operator-(difference_type) const noexcept; - difference_type operator-(const iterator &) const noexcept; - - bool operator==(const iterator &) const noexcept; - bool operator!=(const iterator &) const noexcept; - bool operator<(const iterator &) const noexcept; - bool operator<=(const iterator &) const noexcept; - bool operator>(const iterator &) const noexcept; - bool operator>=(const iterator &) const noexcept; - -private: - friend class Slice; - void *pos; - std::size_t stride; -}; -#endif // CXXBRIDGE1_RUST_SLICE - -#ifndef CXXBRIDGE1_RUST_BOX -// https://cxx.rs/binding/box.html -template -class Box final { -public: - using element_type = T; - using const_pointer = - typename std::add_pointer::type>::type; - using pointer = typename std::add_pointer::type; - - Box() = delete; - Box(Box &&) noexcept; - ~Box() noexcept; - - explicit Box(const T &); - explicit Box(T &&); - - Box &operator=(Box &&) &noexcept; - - const T *operator->() const noexcept; - const T &operator*() const noexcept; - T *operator->() noexcept; - T &operator*() noexcept; - - template - static Box in_place(Fields &&...); - - void swap(Box &) noexcept; - - // Important: requires that `raw` came from an into_raw call. Do not pass a - // pointer from `new` or any other source. - static Box from_raw(T *) noexcept; - - T *into_raw() noexcept; - - /* Deprecated */ using value_type = element_type; - -private: - class uninit; - class allocation; - Box(uninit) noexcept; - void drop() noexcept; - - friend void swap(Box &lhs, Box &rhs) noexcept { lhs.swap(rhs); } - - T *ptr; -}; -#endif // CXXBRIDGE1_RUST_BOX - -#ifndef CXXBRIDGE1_RUST_VEC -// https://cxx.rs/binding/vec.html -template -class Vec final { -public: - using value_type = T; - - Vec() noexcept; - Vec(std::initializer_list); - Vec(const Vec &); - Vec(Vec &&) noexcept; - ~Vec() noexcept; - - Vec &operator=(Vec &&) &noexcept; - Vec &operator=(const Vec &) &; - - std::size_t size() const noexcept; - bool empty() const noexcept; - const T *data() const noexcept; - T *data() noexcept; - std::size_t capacity() const noexcept; - - const T &operator[](std::size_t n) const noexcept; - const T &at(std::size_t n) const; - const T &front() const noexcept; - const T &back() const noexcept; - - T &operator[](std::size_t n) noexcept; - T &at(std::size_t n); - T &front() noexcept; - T &back() noexcept; - - void reserve(std::size_t new_cap); - void push_back(const T &value); - void push_back(T &&value); - template - void emplace_back(Args &&...args); - void truncate(std::size_t len); - void clear(); - - using iterator = typename Slice::iterator; - iterator begin() noexcept; - iterator end() noexcept; - - using const_iterator = typename Slice::iterator; - const_iterator begin() const noexcept; - const_iterator end() const noexcept; - const_iterator cbegin() const noexcept; - const_iterator cend() const noexcept; - - void swap(Vec &) noexcept; - - // Internal API only intended for the cxxbridge code generator. - Vec(unsafe_bitcopy_t, const Vec &) noexcept; - -private: - void reserve_total(std::size_t new_cap) noexcept; - void set_len(std::size_t len) noexcept; - void drop() noexcept; - - friend void swap(Vec &lhs, Vec &rhs) noexcept { lhs.swap(rhs); } - - // Size and alignment statically verified by rust_vec.rs. - std::array repr; -}; -#endif // CXXBRIDGE1_RUST_VEC - -#ifndef CXXBRIDGE1_RUST_FN -// https://cxx.rs/binding/fn.html -template -class Fn; - -template -class Fn final { -public: - Ret operator()(Args... args) const noexcept; - Fn operator*() const noexcept; - -private: - Ret (*trampoline)(Args..., void *fn) noexcept; - void *fn; -}; -#endif // CXXBRIDGE1_RUST_FN - -#ifndef CXXBRIDGE1_RUST_ERROR -#define CXXBRIDGE1_RUST_ERROR -// https://cxx.rs/binding/result.html -class Error final : public std::exception { -public: - Error(const Error &); - Error(Error &&) noexcept; - ~Error() noexcept override; - - Error &operator=(const Error &) &; - Error &operator=(Error &&) &noexcept; - - const char *what() const noexcept override; - -private: - Error() noexcept = default; - friend impl; - const char *msg; - std::size_t len; -}; -#endif // CXXBRIDGE1_RUST_ERROR - -#ifndef CXXBRIDGE1_RUST_ISIZE -#define CXXBRIDGE1_RUST_ISIZE -#if defined(_WIN32) -using isize = SSIZE_T; -#else -using isize = ssize_t; -#endif -#endif // CXXBRIDGE1_RUST_ISIZE - -std::ostream &operator<<(std::ostream &, const String &); -std::ostream &operator<<(std::ostream &, const Str &); - -#ifndef CXXBRIDGE1_RUST_OPAQUE -#define CXXBRIDGE1_RUST_OPAQUE -// Base class of generated opaque Rust types. -class Opaque { -public: - Opaque() = delete; - Opaque(const Opaque &) = delete; - ~Opaque() = delete; -}; -#endif // CXXBRIDGE1_RUST_OPAQUE - -template -std::size_t size_of(); -template -std::size_t align_of(); - -// IsRelocatable is used in assertions that a C++ type passed by value -// between Rust and C++ is soundly relocatable by Rust. -// -// There may be legitimate reasons to opt out of the check for support of types -// that the programmer knows are soundly Rust-movable despite not being -// recognized as such by the C++ type system due to a move constructor or -// destructor. To opt out of the relocatability check, do either of the -// following things in any header used by `include!` in the bridge. -// -// --- if you define the type: -// struct MyType { -// ... -// + using IsRelocatable = std::true_type; -// }; -// -// --- otherwise: -// + template <> -// + struct rust::IsRelocatable : std::true_type {}; -template -struct IsRelocatable; - -using u8 = std::uint8_t; -using u16 = std::uint16_t; -using u32 = std::uint32_t; -using u64 = std::uint64_t; -using usize = std::size_t; // see static asserts in cxx.cc -using i8 = std::int8_t; -using i16 = std::int16_t; -using i32 = std::int32_t; -using i64 = std::int64_t; -using f32 = float; -using f64 = double; - -// Snake case aliases for use in code that uses this style for type names. -using string = String; -using str = Str; -template -using slice = Slice; -template -using box = Box; -template -using vec = Vec; -using error = Error; -template -using fn = Fn; -template -using is_relocatable = IsRelocatable; - - - -//////////////////////////////////////////////////////////////////////////////// -/// end public API, begin implementation details - -#ifndef CXXBRIDGE1_PANIC -#define CXXBRIDGE1_PANIC -template -void panic [[noreturn]] (const char *msg); -#endif // CXXBRIDGE1_PANIC - -#ifndef CXXBRIDGE1_RUST_FN -#define CXXBRIDGE1_RUST_FN -template -Ret Fn::operator()(Args... args) const noexcept { - return (*this->trampoline)(std::forward(args)..., this->fn); -} - -template -Fn Fn::operator*() const noexcept { - return *this; -} -#endif // CXXBRIDGE1_RUST_FN - -#ifndef CXXBRIDGE1_RUST_BITCOPY_T -#define CXXBRIDGE1_RUST_BITCOPY_T -struct unsafe_bitcopy_t final { - explicit unsafe_bitcopy_t() = default; -}; -#endif // CXXBRIDGE1_RUST_BITCOPY_T - -#ifndef CXXBRIDGE1_RUST_BITCOPY -#define CXXBRIDGE1_RUST_BITCOPY -constexpr unsafe_bitcopy_t unsafe_bitcopy{}; -#endif // CXXBRIDGE1_RUST_BITCOPY - -#ifndef CXXBRIDGE1_RUST_SLICE -#define CXXBRIDGE1_RUST_SLICE -template -Slice::Slice() noexcept { - sliceInit(this, reinterpret_cast(align_of()), 0); -} - -template -Slice::Slice(T *s, std::size_t count) noexcept { - assert(s != nullptr || count == 0); - sliceInit(this, - s == nullptr && count == 0 - ? reinterpret_cast(align_of()) - : const_cast::type *>(s), - count); -} - -template -T *Slice::data() const noexcept { - return reinterpret_cast(slicePtr(this)); -} - -template -std::size_t Slice::size() const noexcept { - return sliceLen(this); -} - -template -std::size_t Slice::length() const noexcept { - return this->size(); -} - -template -bool Slice::empty() const noexcept { - return this->size() == 0; -} - -template -T &Slice::operator[](std::size_t n) const noexcept { - assert(n < this->size()); - auto ptr = static_cast(slicePtr(this)) + size_of() * n; - return *reinterpret_cast(ptr); -} - -template -T &Slice::at(std::size_t n) const { - if (n >= this->size()) { - panic("rust::Slice index out of range"); - } - return (*this)[n]; -} - -template -T &Slice::front() const noexcept { - assert(!this->empty()); - return (*this)[0]; -} - -template -T &Slice::back() const noexcept { - assert(!this->empty()); - return (*this)[this->size() - 1]; -} - -template -typename Slice::iterator::reference -Slice::iterator::operator*() const noexcept { - return *static_cast(this->pos); -} - -template -typename Slice::iterator::pointer -Slice::iterator::operator->() const noexcept { - return static_cast(this->pos); -} - -template -typename Slice::iterator::reference Slice::iterator::operator[]( - typename Slice::iterator::difference_type n) const noexcept { - auto ptr = static_cast(this->pos) + this->stride * n; - return *reinterpret_cast(ptr); -} - -template -typename Slice::iterator &Slice::iterator::operator++() noexcept { - this->pos = static_cast(this->pos) + this->stride; - return *this; -} - -template -typename Slice::iterator Slice::iterator::operator++(int) noexcept { - auto ret = iterator(*this); - this->pos = static_cast(this->pos) + this->stride; - return ret; -} - -template -typename Slice::iterator &Slice::iterator::operator--() noexcept { - this->pos = static_cast(this->pos) - this->stride; - return *this; -} - -template -typename Slice::iterator Slice::iterator::operator--(int) noexcept { - auto ret = iterator(*this); - this->pos = static_cast(this->pos) - this->stride; - return ret; -} - -template -typename Slice::iterator &Slice::iterator::operator+=( - typename Slice::iterator::difference_type n) noexcept { - this->pos = static_cast(this->pos) + this->stride * n; - return *this; -} - -template -typename Slice::iterator &Slice::iterator::operator-=( - typename Slice::iterator::difference_type n) noexcept { - this->pos = static_cast(this->pos) - this->stride * n; - return *this; -} - -template -typename Slice::iterator Slice::iterator::operator+( - typename Slice::iterator::difference_type n) const noexcept { - auto ret = iterator(*this); - ret.pos = static_cast(this->pos) + this->stride * n; - return ret; -} - -template -typename Slice::iterator Slice::iterator::operator-( - typename Slice::iterator::difference_type n) const noexcept { - auto ret = iterator(*this); - ret.pos = static_cast(this->pos) - this->stride * n; - return ret; -} - -template -typename Slice::iterator::difference_type -Slice::iterator::operator-(const iterator &other) const noexcept { - auto diff = std::distance(static_cast(other.pos), - static_cast(this->pos)); - return diff / static_cast::iterator::difference_type>( - this->stride); -} - -template -bool Slice::iterator::operator==(const iterator &other) const noexcept { - return this->pos == other.pos; -} - -template -bool Slice::iterator::operator!=(const iterator &other) const noexcept { - return this->pos != other.pos; -} - -template -bool Slice::iterator::operator<(const iterator &other) const noexcept { - return this->pos < other.pos; -} - -template -bool Slice::iterator::operator<=(const iterator &other) const noexcept { - return this->pos <= other.pos; -} - -template -bool Slice::iterator::operator>(const iterator &other) const noexcept { - return this->pos > other.pos; -} - -template -bool Slice::iterator::operator>=(const iterator &other) const noexcept { - return this->pos >= other.pos; -} - -template -typename Slice::iterator Slice::begin() const noexcept { - iterator it; - it.pos = slicePtr(this); - it.stride = size_of(); - return it; -} - -template -typename Slice::iterator Slice::end() const noexcept { - iterator it = this->begin(); - it.pos = static_cast(it.pos) + it.stride * this->size(); - return it; -} - -template -void Slice::swap(Slice &rhs) noexcept { - std::swap(*this, rhs); -} -#endif // CXXBRIDGE1_RUST_SLICE - -#ifndef CXXBRIDGE1_RUST_BOX -#define CXXBRIDGE1_RUST_BOX -template -class Box::uninit {}; - -template -class Box::allocation { - static T *alloc() noexcept; - static void dealloc(T *) noexcept; - -public: - allocation() noexcept : ptr(alloc()) {} - ~allocation() noexcept { - if (this->ptr) { - dealloc(this->ptr); - } - } - T *ptr; -}; - -template -Box::Box(Box &&other) noexcept : ptr(other.ptr) { - other.ptr = nullptr; -} - -template -Box::Box(const T &val) { - allocation alloc; - ::new (alloc.ptr) T(val); - this->ptr = alloc.ptr; - alloc.ptr = nullptr; -} - -template -Box::Box(T &&val) { - allocation alloc; - ::new (alloc.ptr) T(std::move(val)); - this->ptr = alloc.ptr; - alloc.ptr = nullptr; -} - -template -Box::~Box() noexcept { - if (this->ptr) { - this->drop(); - } -} - -template -Box &Box::operator=(Box &&other) &noexcept { - if (this->ptr) { - this->drop(); - } - this->ptr = other.ptr; - other.ptr = nullptr; - return *this; -} - -template -const T *Box::operator->() const noexcept { - return this->ptr; -} - -template -const T &Box::operator*() const noexcept { - return *this->ptr; -} - -template -T *Box::operator->() noexcept { - return this->ptr; -} - -template -T &Box::operator*() noexcept { - return *this->ptr; -} - -template -template -Box Box::in_place(Fields &&...fields) { - allocation alloc; - auto ptr = alloc.ptr; - ::new (ptr) T{std::forward(fields)...}; - alloc.ptr = nullptr; - return from_raw(ptr); -} - -template -void Box::swap(Box &rhs) noexcept { - using std::swap; - swap(this->ptr, rhs.ptr); -} - -template -Box Box::from_raw(T *raw) noexcept { - Box box = uninit{}; - box.ptr = raw; - return box; -} - -template -T *Box::into_raw() noexcept { - T *raw = this->ptr; - this->ptr = nullptr; - return raw; -} - -template -Box::Box(uninit) noexcept {} -#endif // CXXBRIDGE1_RUST_BOX - -#ifndef CXXBRIDGE1_RUST_VEC -#define CXXBRIDGE1_RUST_VEC -template -Vec::Vec(std::initializer_list init) : Vec{} { - this->reserve_total(init.size()); - std::move(init.begin(), init.end(), std::back_inserter(*this)); -} - -template -Vec::Vec(const Vec &other) : Vec() { - this->reserve_total(other.size()); - std::copy(other.begin(), other.end(), std::back_inserter(*this)); -} - -template -Vec::Vec(Vec &&other) noexcept : repr(other.repr) { - new (&other) Vec(); -} - -template -Vec::~Vec() noexcept { - this->drop(); -} - -template -Vec &Vec::operator=(Vec &&other) &noexcept { - this->drop(); - this->repr = other.repr; - new (&other) Vec(); - return *this; -} - -template -Vec &Vec::operator=(const Vec &other) & { - if (this != &other) { - this->drop(); - new (this) Vec(other); - } - return *this; -} - -template -bool Vec::empty() const noexcept { - return this->size() == 0; -} - -template -T *Vec::data() noexcept { - return const_cast(const_cast *>(this)->data()); -} - -template -const T &Vec::operator[](std::size_t n) const noexcept { - assert(n < this->size()); - auto data = reinterpret_cast(this->data()); - return *reinterpret_cast(data + n * size_of()); -} - -template -const T &Vec::at(std::size_t n) const { - if (n >= this->size()) { - panic("rust::Vec index out of range"); - } - return (*this)[n]; -} - -template -const T &Vec::front() const noexcept { - assert(!this->empty()); - return (*this)[0]; -} - -template -const T &Vec::back() const noexcept { - assert(!this->empty()); - return (*this)[this->size() - 1]; -} - -template -T &Vec::operator[](std::size_t n) noexcept { - assert(n < this->size()); - auto data = reinterpret_cast(this->data()); - return *reinterpret_cast(data + n * size_of()); -} - -template -T &Vec::at(std::size_t n) { - if (n >= this->size()) { - panic("rust::Vec index out of range"); - } - return (*this)[n]; -} - -template -T &Vec::front() noexcept { - assert(!this->empty()); - return (*this)[0]; -} - -template -T &Vec::back() noexcept { - assert(!this->empty()); - return (*this)[this->size() - 1]; -} - -template -void Vec::reserve(std::size_t new_cap) { - this->reserve_total(new_cap); -} - -template -void Vec::push_back(const T &value) { - this->emplace_back(value); -} - -template -void Vec::push_back(T &&value) { - this->emplace_back(std::move(value)); -} - -template -template -void Vec::emplace_back(Args &&...args) { - auto size = this->size(); - this->reserve_total(size + 1); - ::new (reinterpret_cast(reinterpret_cast(this->data()) + - size * size_of())) - T(std::forward(args)...); - this->set_len(size + 1); -} - -template -void Vec::clear() { - this->truncate(0); -} - -template -typename Vec::iterator Vec::begin() noexcept { - return Slice(this->data(), this->size()).begin(); -} - -template -typename Vec::iterator Vec::end() noexcept { - return Slice(this->data(), this->size()).end(); -} - -template -typename Vec::const_iterator Vec::begin() const noexcept { - return this->cbegin(); -} - -template -typename Vec::const_iterator Vec::end() const noexcept { - return this->cend(); -} - -template -typename Vec::const_iterator Vec::cbegin() const noexcept { - return Slice(this->data(), this->size()).begin(); -} - -template -typename Vec::const_iterator Vec::cend() const noexcept { - return Slice(this->data(), this->size()).end(); -} - -template -void Vec::swap(Vec &rhs) noexcept { - using std::swap; - swap(this->repr, rhs.repr); -} - -// Internal API only intended for the cxxbridge code generator. -template -Vec::Vec(unsafe_bitcopy_t, const Vec &bits) noexcept : repr(bits.repr) {} -#endif // CXXBRIDGE1_RUST_VEC - -#ifndef CXXBRIDGE1_IS_COMPLETE -#define CXXBRIDGE1_IS_COMPLETE -namespace detail { -namespace { -template -struct is_complete : std::false_type {}; -template -struct is_complete : std::true_type {}; -} // namespace -} // namespace detail -#endif // CXXBRIDGE1_IS_COMPLETE - -#ifndef CXXBRIDGE1_LAYOUT -#define CXXBRIDGE1_LAYOUT -class layout { - template - friend std::size_t size_of(); - template - friend std::size_t align_of(); - template - static typename std::enable_if::value, - std::size_t>::type - do_size_of() { - return T::layout::size(); - } - template - static typename std::enable_if::value, - std::size_t>::type - do_size_of() { - return sizeof(T); - } - template - static - typename std::enable_if::value, std::size_t>::type - size_of() { - return do_size_of(); - } - template - static typename std::enable_if::value, - std::size_t>::type - do_align_of() { - return T::layout::align(); - } - template - static typename std::enable_if::value, - std::size_t>::type - do_align_of() { - return alignof(T); - } - template - static - typename std::enable_if::value, std::size_t>::type - align_of() { - return do_align_of(); - } -}; - -template -std::size_t size_of() { - return layout::size_of(); -} - -template -std::size_t align_of() { - return layout::align_of(); -} -#endif // CXXBRIDGE1_LAYOUT - -#ifndef CXXBRIDGE1_RELOCATABLE -#define CXXBRIDGE1_RELOCATABLE -namespace detail { -template -struct make_void { - using type = void; -}; - -template -using void_t = typename make_void::type; - -template class, typename...> -struct detect : std::false_type {}; -template