From 923a1b58e18a5c3a556bf324a0e281dac32726b3 Mon Sep 17 00:00:00 2001 From: rudy Date: Wed, 13 Apr 2022 12:00:00 +0200 Subject: [PATCH] feat(optimizer): plug online parameter estimation --- .github/workflows/continuous-integration.yml | 76 ++++++- .../Dockerfile.release_manylinux_2_24_x86_64 | 3 +- .../Dockerfile.release_tarball_linux_x86_64 | 3 +- compiler/CMakeLists.txt | 11 ++ compiler/Makefile | 33 +++- compiler/include/boost/outcome.h | 1 + .../concretelang/Support/V0Parameters.h | 5 +- compiler/lib/Support/CMakeLists.txt | 4 + compiler/lib/Support/CompilerEngine.cpp | 5 +- compiler/lib/Support/V0Parameters.cpp | 186 +++--------------- compiler/src/main.cpp | 5 +- compiler/tests/TestLib/CMakeLists.txt | 7 + compiler/tests/stress_tests/test_stress.py | 13 +- compiler/tests/stress_tests/v0_parameters.py | 12 +- compiler/tests/unittest/CMakeLists.txt | 7 + 15 files changed, 173 insertions(+), 198 deletions(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 0c6c49215..f9f393da1 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -16,7 +16,7 @@ env: jobs: ######################## - # Tests and formatiing # + # Tests and formating # ######################## FormattingAndLinting: @@ -66,10 +66,25 @@ jobs: done df -h + # A SSH private key is required as some dependencies are from private repos + - uses: webfactory/ssh-agent@v0.5.2 + with: + ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }} + - uses: actions/checkout@v2 with: submodules: recursive + - name: Install rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + + - name: Concrete-Optimizer + run: | + cd compiler + make concrete-optimizer-lib + - name: Download KeySetCache if: ${{ matrix.compiler == 'gcc7' }} continue-on-error: true @@ -181,6 +196,11 @@ jobs: BuildAndTestMacOS: runs-on: macos-10.15 steps: + # A SSH private key is required as some dependencies are from private repos + - uses: webfactory/ssh-agent@v0.5.2 + with: + ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }} + - uses: actions/checkout@v2 with: submodules: recursive @@ -202,9 +222,14 @@ jobs: path: ${{ github.workspace }}/concrete/target key: ${{ runner.os }}-concrete-lib-${{ steps.concretelib-hash.outputs.COMMIT_SHA }} + - name: Install rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + - name: Install Deps run: | - curl https://sh.rustup.rs -sSf | sh -s -- -y + # curl https://sh.rustup.rs -sSf | sh -s -- -y # TODO check actions-rs/toolchain@v1 brew install ninja ccache pip3 install numpy pybind11==2.6.2 wheel delocate pip3 install pytest @@ -245,7 +270,7 @@ jobs: make python-bindings build-tests test-check echo "Debug: ccache statistics (after the build):" ccache -s - + - name: Download KeySetCache continue-on-error: true if: github.event_name == 'push' @@ -270,10 +295,25 @@ jobs: BuildAndTestDF: runs-on: ubuntu-latest steps: + # A SSH private key is required as some dependencies are from private repos + - uses: webfactory/ssh-agent@v0.5.2 + with: + ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }} + - uses: actions/checkout@v2 with: submodules: recursive + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + + - name: Concrete-Optimizer + run: | + cd compiler + make concrete-optimizer-lib + - name: Download KeySetCache continue-on-error: true run: | @@ -387,6 +427,11 @@ jobs: python-package-name-linux-py310: ${{ steps.build-wheel-linux.outputs.ASSET_NAME_PY310 }} needs: CreateRelease steps: + # A SSH private key is required as some dependencies are from private repos + - uses: webfactory/ssh-agent@v0.5.2 + with: + ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }} + - uses: actions/checkout@v2 with: submodules: recursive @@ -417,6 +462,11 @@ jobs: runs-on: ubuntu-latest needs: CreateRelease steps: + # A SSH private key is required as some dependencies are from private repos + - uses: webfactory/ssh-agent@v0.5.2 + with: + ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }} + - uses: actions/checkout@v2 with: submodules: recursive @@ -453,6 +503,11 @@ jobs: python-package-name-macos-py39: ${{ steps.build-wheel-macos.outputs.ASSET_NAME_PY39 }} python-package-name-macos-py310: ${{ steps.build-wheel-macos.outputs.ASSET_NAME_PY310 }} steps: + # A SSH private key is required as some dependencies are from private repos + - uses: webfactory/ssh-agent@v0.5.2 + with: + ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }} + - uses: actions/checkout@v2 with: submodules: recursive @@ -687,6 +742,11 @@ jobs: IMAGE: ghcr.io/zama-ai/gcc7 steps: + # A SSH private key is required as some dependencies are from private repos + - uses: webfactory/ssh-agent@v0.5.2 + with: + ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }} + - uses: actions/checkout@v2 with: fetch-depth: 0 @@ -695,6 +755,16 @@ jobs: id: changed-files uses: tj-actions/changed-files@v2.0.0 + - name: Install rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + + - name: Concrete-Optimizer + run: | + cd compiler + make concrete-optimizer-lib + - name: Login if: contains(steps.changed-files.outputs.modified_files, 'builders/Dockerfile.gcc7-env') run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io diff --git a/builders/Dockerfile.release_manylinux_2_24_x86_64 b/builders/Dockerfile.release_manylinux_2_24_x86_64 index be2a92c93..8401cc9b8 100644 --- a/builders/Dockerfile.release_manylinux_2_24_x86_64 +++ b/builders/Dockerfile.release_manylinux_2_24_x86_64 @@ -1,7 +1,8 @@ FROM quay.io/pypa/manylinux_2_24_x86_64 RUN apt-get update -RUN DEBIAN_FRONTEND="noninteractive" apt-get install -y build-essential ninja-build +RUN DEBIAN_FRONTEND="noninteractive" apt-get install -y build-essential ninja-build +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y # Set the python path. Options: [cp38-cp38, cp39-cp39, cp310-cp310] ARG python_tag=cp38-cp38 # Install python deps diff --git a/builders/Dockerfile.release_tarball_linux_x86_64 b/builders/Dockerfile.release_tarball_linux_x86_64 index 8d3f40304..335f90ca2 100644 --- a/builders/Dockerfile.release_tarball_linux_x86_64 +++ b/builders/Dockerfile.release_tarball_linux_x86_64 @@ -1,7 +1,8 @@ FROM quay.io/pypa/manylinux_2_24_x86_64 RUN apt-get update -RUN DEBIAN_FRONTEND="noninteractive" apt-get install -y build-essential ninja-build +RUN DEBIAN_FRONTEND="noninteractive" apt-get install -y build-essential ninja-build +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y # Setup gcc7 COPY --from=ghcr.io/zama-ai/gcc7:latest /gcc7 /gcc7 ENV PATH=/gcc7/bin:$PATH diff --git a/compiler/CMakeLists.txt b/compiler/CMakeLists.txt index 15a88a297..3b54015fb 100644 --- a/compiler/CMakeLists.txt +++ b/compiler/CMakeLists.txt @@ -117,6 +117,17 @@ endif() option(CONCRETELANG_UNIT_TESTS "Enables the build of unittests" ON) +#------------------------------------------------------------------------------- +# Adding rust dependencies, already compiled +#------------------------------------------------------------------------------- + +link_directories(${CONCRETE_OPTIMIZER_DIR}/target) +include_directories(${CONCRETE_OPTIMIZER_DIR}/target/include) + + +#------------------------------------------------------------------------------- +# Handling sub dirs +#------------------------------------------------------------------------------- add_subdirectory(include) add_subdirectory(lib) diff --git a/compiler/Makefile b/compiler/Makefile index b4eda2b5e..6c74bda92 100644 --- a/compiler/Makefile +++ b/compiler/Makefile @@ -5,6 +5,9 @@ PARALLEL_EXECUTION_ENABLED=OFF CC_COMPILER= CXX_COMPILER= +CONCRETE_OPTIMIZER_DIR ?= .dependencies/concrete-optimizer +CONCRETE_OPTIMIZER_BRANCH ?= master + KEYSETCACHEDEV=/tmp/KeySetCache KEYSETCACHECI=../KeySetCache @@ -58,10 +61,27 @@ $(BUILD_DIR)/configured.stamp: -DHPX_DIR=${HPX_INSTALL_DIR}/lib/cmake/HPX \ -DLLVM_EXTERNAL_PROJECTS=concretelang \ -DLLVM_EXTERNAL_CONCRETELANG_SOURCE_DIR=. \ - -DPython3_EXECUTABLE=${Python3_EXECUTABLE} + -DPython3_EXECUTABLE=${Python3_EXECUTABLE} \ + -DCONCRETE_OPTIMIZER_DIR=${CONCRETE_OPTIMIZER_DIR} touch $@ -build-initialized: $(EXTERNAL_HEADERS) $(BUILD_DIR)/configured.stamp +build-initialized: concrete-optimizer-lib $(BUILD_DIR)/configured.stamp + +# Fetch concrete-optimizer +$(CONCRETE_OPTIMIZER_DIR): + export GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=accept-new" # for CI + # change to master on merge + mkdir -p $(shell dirname ${CONCRETE_OPTIMIZER_DIR}) + git clone --branch $(CONCRETE_OPTIMIZER_BRANCH) git@github.com:zama-ai/concrete-optimizer.git $@ + +CONCRETE_OPTIMIZER_SRC := $(shell find $(CONCRETE_OPTIMIZER_DIR) -name '*.rs') + +# Build concrete-optimizer +$(CONCRETE_OPTIMIZER_DIR)/target/libconcrete_optimizer.a: $(CONCRETE_OPTIMIZER_DIR) $(CONCRETE_OPTIMIZER_SRC) + cd $(CONCRETE_OPTIMIZER_DIR)/concrete-optimizer && cargo build --release + cp $(CONCRETE_OPTIMIZER_DIR)/target/release/$(shell basename $@) $@ + +concrete-optimizer-lib: $(CONCRETE_OPTIMIZER_DIR)/target/libconcrete_optimizer.a doc: build-initialized cmake --build $(BUILD_DIR) --target mlir-doc @@ -219,16 +239,17 @@ define build_image_and_copy_wheels docker container run --rm -v ${PWD}/../wheels:/wheels_volume concrete-compiler-manylinux:$(1) cp -r /wheels/. /wheels_volume/. endef -package_py38: update_python_version +# we add CONCRETE_OPTIMIZER_DIR as sub target to upload concrete-optimizer source in compiler directory +package_py38: $(CONCRETE_OPTIMIZER_DIR) update_python_version $(call build_image_and_copy_wheels,cp38-cp38) -package_py39: update_python_version +package_py39: $(CONCRETE_OPTIMIZER_DIR) update_python_version $(call build_image_and_copy_wheels,cp39-cp39) -package_py310: update_python_version +package_py310: $(CONCRETE_OPTIMIZER_DIR) update_python_version $(call build_image_and_copy_wheels,cp310-cp310) -release_tarballs: +release_tarballs: $(CONCRETE_OPTIMIZER_DIR) docker image build -t concrete-compiler-manylinux:linux_x86_64_tarball -f ../builders/Dockerfile.release_tarball_linux_x86_64 .. docker container run --rm -v ${PWD}/../tarballs:/tarballs_volume concrete-compiler-manylinux:linux_x86_64_tarball cp -r /tarballs/. /tarballs_volume/. diff --git a/compiler/include/boost/outcome.h b/compiler/include/boost/outcome.h index 1ab25cd94..83614db76 100644 --- a/compiler/include/boost/outcome.h +++ b/compiler/include/boost/outcome.h @@ -6,6 +6,7 @@ #ifndef CONCRETELANG_BOOST_OUTCOME_H #define CONCRETELANG_BOOST_OUTCOME_H +// https://github.com/ned14/outcome/raw/master/single-header/outcome.hpp #include "boost-single-header/outcome.hpp" namespace outcome = outcome_v2_e261cebd; diff --git a/compiler/include/concretelang/Support/V0Parameters.h b/compiler/include/concretelang/Support/V0Parameters.h index c669fd22c..972eb1f60 100644 --- a/compiler/include/concretelang/Support/V0Parameters.h +++ b/compiler/include/concretelang/Support/V0Parameters.h @@ -6,13 +6,14 @@ #ifndef CONCRETELANG_SUPPORT_V0Parameter_H_ #define CONCRETELANG_SUPPORT_V0Parameter_H_ +#include "llvm/ADT/Optional.h" + #include "concretelang/Conversion/Utils/GlobalFHEContext.h" -#include namespace mlir { namespace concretelang { -const V0Parameter *getV0Parameter(V0FHEConstraint constraint); +llvm::Optional getV0Parameter(V0FHEConstraint constraint); } // namespace concretelang } // namespace mlir diff --git a/compiler/lib/Support/CMakeLists.txt b/compiler/lib/Support/CMakeLists.txt index 0f88a4e5a..1cc2d5205 100644 --- a/compiler/lib/Support/CMakeLists.txt +++ b/compiler/lib/Support/CMakeLists.txt @@ -1,3 +1,7 @@ +link_directories(${CONCRETE_OPTIMIZER_DIR}/target) +include_directories(${CONCRETE_OPTIMIZER_DIR}/target/include) +link_libraries(concrete_optimizer.a) + add_mlir_library(ConcretelangSupport Pipeline.cpp Jit.cpp diff --git a/compiler/lib/Support/CompilerEngine.cpp b/compiler/lib/Support/CompilerEngine.cpp index 11e521e18..a3ea85245 100644 --- a/compiler/lib/Support/CompilerEngine.cpp +++ b/compiler/lib/Support/CompilerEngine.cpp @@ -163,8 +163,7 @@ llvm::Error CompilerEngine::determineFHEParameters(CompilationResult &res) { if (!fheConstraintOrErr.get().hasValue()) { return llvm::Error::success(); } - const mlir::concretelang::V0Parameter *fheParams = - getV0Parameter(fheConstraintOrErr.get().getValue()); + auto fheParams = getV0Parameter(fheConstraintOrErr.get().getValue()); if (!fheParams) { return StreamStringError() @@ -173,7 +172,7 @@ llvm::Error CompilerEngine::determineFHEParameters(CompilationResult &res) { << (*fheConstraintOrErr)->p; } res.fheContext.emplace(mlir::concretelang::V0FHEContext{ - (*fheConstraintOrErr).getValue(), *fheParams}); + (*fheConstraintOrErr).getValue(), fheParams.getValue()}); return llvm::Error::success(); } diff --git a/compiler/lib/Support/V0Parameters.cpp b/compiler/lib/Support/V0Parameters.cpp index 8f2b866fd..60f16a98f 100644 --- a/compiler/lib/Support/V0Parameters.cpp +++ b/compiler/lib/Support/V0Parameters.cpp @@ -8,171 +8,43 @@ /// We should include this in our build system, but for moment it is just a cc /// from the optimizer output. +#include +#include +#include + +#include "concrete-optimizer.h" #include "concretelang/Support/V0Parameters.h" namespace mlir { namespace concretelang { -using namespace std; -const int NORM2_MAX = 31; -const int P_MAX = 8; +const double P_ERROR_4_SIGMA = 1.0 - 0.999936657516; -const static V0Parameter parameters[NORM2_MAX][P_MAX] = { - {V0Parameter(1, 10, 514, 2, 8, 5, 2), V0Parameter(1, 10, 564, 2, 8, 5, 2), - V0Parameter(1, 10, 599, 3, 6, 6, 2), V0Parameter(1, 10, 686, 3, 6, 7, 2), - V0Parameter(1, 11, 736, 1, 23, 5, 3), V0Parameter(1, 12, 830, 1, 23, 4, 4), - V0Parameter(1, 13, 869, 1, 23, 6, 3), - V0Parameter(1, 14, 933, 2, 15, 6, 3)}, - {V0Parameter(1, 10, 519, 2, 8, 5, 2), V0Parameter(1, 10, 558, 3, 6, 5, 2), - V0Parameter(1, 10, 610, 3, 6, 6, 2), V0Parameter(1, 11, 689, 1, 23, 4, 3), - V0Parameter(1, 11, 736, 1, 23, 5, 3), V0Parameter(1, 12, 832, 1, 23, 4, 4), - V0Parameter(1, 13, 860, 2, 15, 6, 3), - V0Parameter(1, 14, 934, 2, 15, 6, 3)}, - {V0Parameter(1, 10, 515, 3, 6, 5, 2), V0Parameter(1, 10, 569, 3, 6, 5, 2), - V0Parameter(1, 11, 638, 1, 23, 4, 3), V0Parameter(1, 11, 689, 1, 23, 4, 3), - V0Parameter(1, 11, 737, 1, 23, 5, 3), V0Parameter(1, 12, 842, 1, 23, 4, 4), - V0Parameter(1, 13, 860, 2, 15, 6, 3), - V0Parameter(1, 14, 934, 2, 15, 6, 3)}, - {V0Parameter(1, 10, 523, 3, 6, 5, 2), V0Parameter(1, 11, 598, 1, 23, 4, 3), - V0Parameter(1, 11, 639, 1, 23, 4, 3), V0Parameter(1, 11, 690, 1, 23, 4, 3), - V0Parameter(1, 11, 739, 1, 23, 5, 3), V0Parameter(1, 12, 806, 2, 15, 5, 3), - V0Parameter(1, 13, 860, 2, 15, 6, 3), - V0Parameter(1, 14, 934, 2, 15, 6, 3)}, - {V0Parameter(1, 11, 563, 1, 23, 3, 3), V0Parameter(1, 11, 598, 1, 23, 4, 3), - V0Parameter(1, 11, 639, 1, 23, 4, 3), V0Parameter(1, 11, 692, 1, 23, 4, 3), - V0Parameter(1, 11, 751, 1, 23, 5, 3), V0Parameter(1, 12, 806, 2, 15, 5, 3), - V0Parameter(1, 13, 860, 2, 16, 6, 3), - V0Parameter(1, 14, 936, 2, 15, 6, 3)}, - {V0Parameter(1, 11, 563, 1, 23, 3, 3), V0Parameter(1, 11, 598, 1, 23, 4, 3), - V0Parameter(1, 11, 640, 1, 23, 4, 3), V0Parameter(1, 11, 700, 1, 23, 4, 3), - V0Parameter(1, 11, 736, 2, 15, 5, 3), V0Parameter(1, 12, 806, 2, 16, 5, 3), - V0Parameter(1, 13, 860, 2, 16, 6, 3), - V0Parameter(1, 14, 958, 2, 15, 5, 4)}, - {V0Parameter(1, 11, 563, 1, 23, 3, 3), V0Parameter(1, 11, 599, 1, 23, 4, 3), - V0Parameter(1, 11, 644, 1, 23, 4, 3), V0Parameter(1, 11, 742, 1, 23, 5, 3), - V0Parameter(1, 11, 736, 2, 15, 5, 3), V0Parameter(1, 12, 806, 2, 15, 5, 3), - V0Parameter(1, 13, 861, 2, 15, 6, 3), - V0Parameter(1, 14, 957, 2, 15, 7, 3)}, - {V0Parameter(1, 11, 564, 1, 23, 3, 3), V0Parameter(1, 11, 603, 1, 23, 4, 3), - V0Parameter(1, 11, 683, 1, 23, 4, 3), V0Parameter(1, 11, 689, 2, 16, 4, 3), - V0Parameter(1, 11, 736, 2, 15, 5, 3), V0Parameter(1, 12, 807, 2, 16, 5, 3), - V0Parameter(1, 13, 866, 2, 15, 6, 3), - V0Parameter(1, 14, 936, 3, 12, 6, 3)}, - {V0Parameter(1, 11, 570, 1, 23, 3, 3), V0Parameter(1, 11, 629, 1, 23, 4, 3), - V0Parameter(1, 11, 638, 2, 16, 4, 3), V0Parameter(1, 11, 689, 2, 15, 4, 3), - V0Parameter(1, 11, 736, 2, 16, 5, 3), V0Parameter(1, 12, 809, 2, 15, 5, 3), - V0Parameter(1, 13, 914, 2, 15, 6, 3), - V0Parameter(1, 14, 928, 3, 12, 7, 3)}, - {V0Parameter(1, 11, 560, 1, 23, 5, 2), V0Parameter(1, 11, 579, 2, 16, 5, 2), - V0Parameter(1, 11, 638, 2, 15, 4, 3), V0Parameter(1, 11, 689, 2, 15, 4, 3), - V0Parameter(1, 11, 737, 2, 16, 5, 3), V0Parameter(1, 12, 819, 2, 15, 5, 3), - V0Parameter(1, 13, 861, 3, 12, 6, 3), - V0Parameter(1, 14, 964, 3, 12, 7, 3)}, - {V0Parameter(1, 11, 532, 2, 15, 5, 2), V0Parameter(1, 11, 579, 2, 16, 5, 2), - V0Parameter(1, 11, 638, 2, 16, 4, 3), V0Parameter(1, 11, 690, 2, 15, 4, 3), - V0Parameter(1, 11, 738, 2, 15, 5, 3), V0Parameter(1, 12, 807, 3, 12, 5, 3), - V0Parameter(1, 13, 864, 3, 12, 6, 3), V0Parameter(1, 14, 924, 4, 9, 7, 3)}, - {V0Parameter(1, 11, 532, 2, 15, 5, 2), V0Parameter(1, 11, 579, 2, 15, 5, 2), - V0Parameter(1, 11, 639, 2, 16, 4, 3), V0Parameter(1, 11, 691, 2, 15, 4, 3), - V0Parameter(1, 11, 747, 2, 16, 5, 3), V0Parameter(1, 12, 808, 3, 12, 5, 3), - V0Parameter(1, 13, 890, 3, 12, 6, 3), V0Parameter(1, 14, 932, 4, 9, 7, 3)}, - {V0Parameter(1, 11, 532, 2, 15, 5, 2), V0Parameter(1, 11, 580, 2, 15, 5, 2), - V0Parameter(1, 11, 639, 2, 15, 4, 3), V0Parameter(1, 11, 698, 2, 15, 4, 3), - V0Parameter(1, 11, 736, 3, 12, 5, 3), V0Parameter(1, 12, 813, 3, 12, 5, 3), - V0Parameter(1, 13, 862, 4, 9, 6, 3), V0Parameter(1, 14, 905, 5, 8, 10, 2)}, - {V0Parameter(1, 11, 533, 2, 16, 5, 2), V0Parameter(1, 11, 581, 2, 15, 5, 2), - V0Parameter(1, 11, 643, 2, 16, 4, 3), V0Parameter(1, 11, 713, 2, 16, 5, 3), - V0Parameter(1, 11, 737, 3, 12, 5, 3), V0Parameter(1, 12, 795, 3, 12, 8, 2), - V0Parameter(1, 13, 869, 4, 9, 6, 3), V0Parameter(1, 14, 907, 6, 7, 10, 2)}, - {V0Parameter(1, 11, 533, 2, 16, 5, 2), V0Parameter(1, 11, 585, 2, 15, 5, 2), - V0Parameter(1, 11, 665, 2, 16, 4, 3), V0Parameter(1, 11, 665, 3, 12, 6, 2), - V0Parameter(1, 11, 740, 3, 12, 5, 3), V0Parameter(1, 12, 775, 4, 9, 8, 2), - V0Parameter(1, 13, 842, 5, 8, 9, 2), V0Parameter(1, 14, 910, 7, 6, 10, 2)}, - {V0Parameter(1, 11, 536, 2, 15, 5, 2), V0Parameter(1, 11, 618, 2, 16, 4, 3), - V0Parameter(1, 11, 614, 3, 12, 6, 2), V0Parameter(1, 11, 667, 3, 12, 6, 2), - V0Parameter(1, 11, 754, 3, 12, 5, 3), V0Parameter(1, 12, 784, 4, 9, 8, 2), - V0Parameter(1, 13, 844, 6, 7, 9, 2), V0Parameter(1, 14, 916, 8, 5, 10, 2)}, - {V0Parameter(1, 11, 551, 2, 16, 5, 2), V0Parameter(1, 11, 580, 3, 12, 5, 2), - V0Parameter(1, 11, 615, 3, 12, 6, 2), V0Parameter(1, 11, 664, 3, 12, 7, 2), - V0Parameter(1, 11, 716, 4, 9, 7, 2), V0Parameter(1, 12, 780, 5, 8, 8, 2), - V0Parameter(1, 13, 846, 7, 6, 9, 2), - V0Parameter(1, 14, 945, 10, 4, 10, 2)}, - {V0Parameter(1, 11, 533, 3, 12, 5, 2), V0Parameter(1, 11, 581, 3, 12, 5, 2), - V0Parameter(1, 11, 620, 3, 12, 6, 2), V0Parameter(1, 11, 658, 4, 9, 7, 2), - V0Parameter(1, 11, 728, 4, 9, 7, 2), V0Parameter(1, 12, 781, 6, 7, 8, 2), - V0Parameter(1, 13, 852, 8, 5, 9, 2), - V0Parameter(1, 14, 908, 21, 2, 10, 2)}, - {V0Parameter(1, 11, 534, 3, 12, 5, 2), V0Parameter(1, 11, 588, 3, 12, 5, 2), - V0Parameter(1, 11, 665, 3, 12, 7, 2), V0Parameter(1, 11, 663, 4, 9, 7, 2), - V0Parameter(1, 11, 719, 5, 8, 7, 2), V0Parameter(1, 12, 783, 7, 6, 8, 2), - V0Parameter(1, 13, 895, 10, 4, 10, 2), - V0Parameter(1, 14, 915, 43, 1, 20, 1)}, - {V0Parameter(1, 11, 538, 3, 12, 5, 2), V0Parameter(1, 11, 613, 3, 12, 6, 2), - V0Parameter(1, 11, 619, 4, 9, 6, 2), V0Parameter(1, 11, 710, 4, 9, 7, 2), - V0Parameter(1, 11, 750, 5, 8, 8, 2), V0Parameter(1, 12, 792, 8, 5, 8, 2), - V0Parameter(1, 13, 880, 14, 3, 10, 2), V0Parameter(0, 0, 0, 0, 0, 0, 0)}, - {V0Parameter(1, 11, 571, 3, 12, 5, 2), V0Parameter(1, 11, 578, 4, 9, 6, 2), - V0Parameter(1, 11, 654, 4, 9, 6, 2), V0Parameter(1, 11, 670, 5, 8, 7, 2), - V0Parameter(1, 11, 722, 7, 6, 7, 2), V0Parameter(1, 12, 792, 11, 4, 8, 2), - V0Parameter(1, 13, 849, 43, 1, 18, 1), V0Parameter(0, 0, 0, 0, 0, 0, 0)}, - {V0Parameter(1, 11, 537, 4, 9, 5, 2), V0Parameter(1, 11, 602, 4, 9, 6, 2), - V0Parameter(1, 11, 626, 5, 8, 6, 2), V0Parameter(1, 11, 673, 6, 7, 7, 2), - V0Parameter(1, 11, 732, 8, 5, 8, 2), V0Parameter(1, 12, 811, 14, 3, 9, 2), - V0Parameter(0, 0, 0, 0, 0, 0, 0), V0Parameter(0, 0, 0, 0, 0, 0, 0)}, - {V0Parameter(1, 11, 560, 4, 9, 5, 2), V0Parameter(1, 11, 583, 5, 8, 6, 2), - V0Parameter(1, 11, 629, 6, 7, 6, 2), V0Parameter(1, 11, 678, 7, 6, 7, 2), - V0Parameter(1, 11, 729, 11, 4, 7, 2), - V0Parameter(1, 12, 785, 43, 1, 17, 1), V0Parameter(0, 0, 0, 0, 0, 0, 0), - V0Parameter(0, 0, 0, 0, 0, 0, 0)}, - {V0Parameter(1, 11, 542, 5, 8, 5, 2), V0Parameter(1, 11, 586, 6, 7, 6, 2), - V0Parameter(1, 11, 633, 7, 6, 6, 2), V0Parameter(1, 11, 690, 9, 5, 7, 2), - V0Parameter(1, 11, 751, 14, 3, 8, 2), V0Parameter(0, 0, 0, 0, 0, 0, 0), - V0Parameter(0, 0, 0, 0, 0, 0, 0), V0Parameter(0, 0, 0, 0, 0, 0, 0)}, - {V0Parameter(1, 11, 544, 6, 7, 5, 2), V0Parameter(1, 11, 589, 7, 6, 6, 2), - V0Parameter(1, 11, 642, 9, 5, 6, 2), V0Parameter(1, 11, 724, 11, 4, 7, 2), - V0Parameter(1, 11, 761, 22, 2, 16, 1), V0Parameter(0, 0, 0, 0, 0, 0, 0), - V0Parameter(0, 0, 0, 0, 0, 0, 0), V0Parameter(0, 0, 0, 0, 0, 0, 0)}, - {V0Parameter(1, 11, 547, 7, 6, 5, 2), V0Parameter(1, 11, 595, 9, 5, 6, 2), - V0Parameter(1, 11, 652, 11, 4, 7, 2), V0Parameter(1, 11, 673, 22, 2, 7, 2), - V0Parameter(0, 0, 0, 0, 0, 0, 0), V0Parameter(0, 0, 0, 0, 0, 0, 0), - V0Parameter(0, 0, 0, 0, 0, 0, 0), V0Parameter(0, 0, 0, 0, 0, 0, 0)}, - {V0Parameter(1, 11, 586, 8, 5, 6, 2), V0Parameter(1, 11, 604, 11, 4, 6, 2), - V0Parameter(1, 11, 628, 22, 2, 6, 2), - V0Parameter(1, 11, 732, 45, 1, 16, 1), V0Parameter(0, 0, 0, 0, 0, 0, 0), - V0Parameter(0, 0, 0, 0, 0, 0, 0), V0Parameter(0, 0, 0, 0, 0, 0, 0), - V0Parameter(0, 0, 0, 0, 0, 0, 0)}, - {V0Parameter(1, 11, 563, 11, 4, 5, 2), - V0Parameter(1, 11, 597, 21, 2, 12, 1), - V0Parameter(1, 11, 656, 44, 1, 13, 1), V0Parameter(0, 0, 0, 0, 0, 0, 0), - V0Parameter(0, 0, 0, 0, 0, 0, 0), V0Parameter(0, 0, 0, 0, 0, 0, 0), - V0Parameter(0, 0, 0, 0, 0, 0, 0), V0Parameter(0, 0, 0, 0, 0, 0, 0)}, - {V0Parameter(1, 11, 579, 15, 3, 11, 1), - V0Parameter(1, 11, 603, 44, 1, 12, 1), V0Parameter(0, 0, 0, 0, 0, 0, 0), - V0Parameter(0, 0, 0, 0, 0, 0, 0), V0Parameter(0, 0, 0, 0, 0, 0, 0), - V0Parameter(0, 0, 0, 0, 0, 0, 0), V0Parameter(0, 0, 0, 0, 0, 0, 0), - V0Parameter(0, 0, 0, 0, 0, 0, 0)}, - {V0Parameter(1, 11, 556, 44, 1, 11, 1), V0Parameter(0, 0, 0, 0, 0, 0, 0), - V0Parameter(0, 0, 0, 0, 0, 0, 0), V0Parameter(0, 0, 0, 0, 0, 0, 0), - V0Parameter(0, 0, 0, 0, 0, 0, 0), V0Parameter(0, 0, 0, 0, 0, 0, 0), - V0Parameter(0, 0, 0, 0, 0, 0, 0), V0Parameter(0, 0, 0, 0, 0, 0, 0)}, - {V0Parameter(0, 0, 0, 0, 0, 0, 0), V0Parameter(0, 0, 0, 0, 0, 0, 0), - V0Parameter(0, 0, 0, 0, 0, 0, 0), V0Parameter(0, 0, 0, 0, 0, 0, 0), - V0Parameter(0, 0, 0, 0, 0, 0, 0), V0Parameter(0, 0, 0, 0, 0, 0, 0), - V0Parameter(0, 0, 0, 0, 0, 0, 0), V0Parameter(0, 0, 0, 0, 0, 0, 0)}}; +llvm::Optional getV0Parameter(V0FHEConstraint constraint) { + int security = 128; + // the norm2 0 is equivalent to a maximum noise_factor of 2.0 + // norm2 = 0 ==> 1.0 =< noise_factor < 2.0 + // norm2 = k ==> 2^norm2 =< noise_factor < 2.0^norm2 + 1 + double noise_factor = std::exp2(constraint.norm2 + 1); + // https://github.com/zama-ai/concrete-optimizer/blob/prototype/python/optimizer/V0Parameters/tabulation.py#L58 + double p_error = P_ERROR_4_SIGMA; + auto sol = concrete_optimizer::optimise_bootstrap(constraint.p, security, + noise_factor, p_error); -const V0Parameter *getV0Parameter(V0FHEConstraint constraint) { - if (constraint.norm2 > NORM2_MAX) { - return nullptr; + if (sol.p_error == 1.0) { + // The optimizer return a p_error = 1 if there is no solution + return llvm::None; } - if (constraint.p > P_MAX) { - return nullptr; - } - // - 1 is an offset as p is in [1, ...] and not [0, ...] - auto param = ¶meters[constraint.norm2][constraint.p - 1]; - if (param->glweDimension == 0) { - return nullptr; - } - return param; + + return mlir::concretelang::V0Parameter{ + sol.glwe_dimension, + (size_t)std::log2l(sol.glwe_polynomial_size), + sol.internal_ks_output_lwe_dimension, + sol.br_decomposition_level_count, + sol.br_decomposition_base_log, + sol.ks_decomposition_level_count, + sol.ks_decomposition_base_log, + }; } } // namespace concretelang diff --git a/compiler/src/main.cpp b/compiler/src/main.cpp index a41d4b90d..4c82b1d0b 100644 --- a/compiler/src/main.cpp +++ b/compiler/src/main.cpp @@ -203,8 +203,7 @@ llvm::Expected buildFHEContext( overrideMaxEintPrecision.hasValue() ? overrideMaxEintPrecision.getValue() : autoFHEConstraints.getValue().p}; - const mlir::concretelang::V0Parameter *parameter = - getV0Parameter(fheConstraints); + auto parameter = getV0Parameter(fheConstraints); if (!parameter) { return mlir::concretelang::StreamStringError() @@ -212,7 +211,7 @@ llvm::Expected buildFHEContext( << fheConstraints.norm2 << " and p of " << fheConstraints.p; } - return mlir::concretelang::V0FHEContext{fheConstraints, *parameter}; + return mlir::concretelang::V0FHEContext{fheConstraints, parameter.getValue()}; } namespace llvm { diff --git a/compiler/tests/TestLib/CMakeLists.txt b/compiler/tests/TestLib/CMakeLists.txt index fab708bd0..37044a900 100644 --- a/compiler/tests/TestLib/CMakeLists.txt +++ b/compiler/tests/TestLib/CMakeLists.txt @@ -2,6 +2,13 @@ enable_testing() include_directories(${PROJECT_SOURCE_DIR}/include) +if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + link_libraries( + # usefull for old gcc versions + -Wl,--allow-multiple-definition # static concrete-optimizer and concrete shares some code + ) +endif() + add_executable( testlib_unit_test testlib_unit_test.cpp diff --git a/compiler/tests/stress_tests/test_stress.py b/compiler/tests/stress_tests/test_stress.py index 4c0aa60f3..de6f36232 100644 --- a/compiler/tests/stress_tests/test_stress.py +++ b/compiler/tests/stress_tests/test_stress.py @@ -12,10 +12,7 @@ from stress_tests.experiment import ( ) from stress_tests import read_mlir from stress_tests.utils import CONCRETECOMPILER, run -from stress_tests.v0_parameters import ( - LOG2_MANP_MAX, P_MAX, - v0_parameter -) +from stress_tests.v0_parameters import P_MAX, LOG2_MANP_MAX POSSIBLE_BITWIDTH = range(1, P_MAX+1) POSSIBLE_SIZE = range(1, 128) @@ -121,10 +118,6 @@ def basic_setup(bitwidth, size, const, retry=10): # Read various value from compiler log_manp_max = read_mlir.log_manp_max(path) params = read_mlir.v0_param(path) - # From CPP source - expected_params = v0_parameter(log_manp_max, bitwidth) - expected_log_poly_size = expected_params.logPolynomialSize - expected_glwe_dim = expected_params.glweDimension conditions_details = [] def msg(m, append_here=None, space=' '): @@ -136,10 +129,6 @@ def basic_setup(bitwidth, size, const, retry=10): msg('HIGH-MANP', conditions_details) if 2 ** bitwidth <= expected: msg(f'OVERFLOW', conditions_details) - if params.log_poly_size != expected_log_poly_size: - msg(f'BAD_LOGPOLYSIZE({params.log_poly_size} vs {expected_log_poly_size})', conditions_details) - if params.glwe_dim != expected_glwe_dim: - msg(f'BAD_GLWEDIM ({params.glwe_dim} vs {expected_glwe_dim})', conditions_details) cmd = (CONCRETECOMPILER, path) + JIT_INVOKE_MAIN + jit_args(*args) compilers_calls = [executor.submit(run, *cmd) for _ in range(retry)] diff --git a/compiler/tests/stress_tests/v0_parameters.py b/compiler/tests/stress_tests/v0_parameters.py index a38277086..9c93dd32a 100644 --- a/compiler/tests/stress_tests/v0_parameters.py +++ b/compiler/tests/stress_tests/v0_parameters.py @@ -38,13 +38,5 @@ def read_CPP_decl(name, cpp_filepath): # print(f'{name} = {value}', decl.group()) return value -LOG2_MANP_MAX = int(read_CPP_decl('NORM2_MAX', V0Parameters_PATH)) -P_MAX = int(read_CPP_decl('P_MAX', V0Parameters_PATH)) - -try: - parameters_cpp = read_CPP_decl('parameters', V0Parameters_PATH) -except (FileNotFoundError, NameError) as exc: - print(exc) - assert False - -v0_parameters = eval(parameters_cpp.replace('{', '[').replace('}', ']')) +LOG2_MANP_MAX = 31 +P_MAX = 8 diff --git a/compiler/tests/unittest/CMakeLists.txt b/compiler/tests/unittest/CMakeLists.txt index 67aafed65..28074dbc2 100644 --- a/compiler/tests/unittest/CMakeLists.txt +++ b/compiler/tests/unittest/CMakeLists.txt @@ -2,6 +2,13 @@ enable_testing() include_directories(${PROJECT_SOURCE_DIR}/include) +if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + link_libraries( + # usefull for old gcc versions + -Wl,--allow-multiple-definition # static concrete-optimizer and concrete shares some code + ) +endif() + if(CONCRETELANG_PARALLEL_EXECUTION_ENABLED) add_compile_options( -DCONCRETELANG_PARALLEL_TESTING_ENABLED