mirror of
https://github.com/zama-ai/concrete.git
synced 2026-04-17 03:00:54 -04:00
e8ef48ffd8
This commit: + Adds support for a protocol which enables inter-op between concrete, tfhe-rs and potentially other contributors to the fhe ecosystem. + Gets rid of hand-made serialization in the compiler, and client/server libs. + Refactors client/server libs to allow more pre/post processing of circuit inputs/outputs. The protocol is supported by a definition in the shape of a capnp file, which defines different types of objects among which: + ProgramInfo object, which is a precise description of a set of fhe circuit coming from the same compilation (understand function type information), and the associated key set. + *Key objects, which represent secret/public keys used to encrypt/execute fhe circuits. + Value object, which represent values that can be transferred between client and server to support calls to fhe circuits. The hand-rolled serialization that was previously used is completely dropped in favor of capnp in the whole codebase. The client/server libs, are refactored to introduce a modular design for pre-post processing. Reading the ProgramInfo file associated with a compilation, the client and server libs assemble a pipeline of transformers (functions) for pre and post processing of values coming in and out of a circuit. This design properly decouples various aspects of the processing, and allows these capabilities to be safely extended. In practice this commit includes the following: + Defines the specification in a concreteprotocol package + Integrate the compilation of this package as a compiler dependency via cmake + Modify the compiler to use the Encodings objects defined in the protocol + Modify the compiler to emit ProgramInfo files as compilation artifact, and gets rid of the bloated ClientParameters. + Introduces a new Common library containing the functionalities shared between the compiler and the client/server libs. + Introduces a functional pre-post processing pipeline to this common library + Modify the client/server libs to support loading ProgramInfo objects, and calling circuits using Value messages. + Drops support of JIT. + Drops support of C-api. + Drops support of Rust bindings. Co-authored-by: Nikita Frolov <nf@mkmks.org>
172 lines
6.0 KiB
CMake
172 lines
6.0 KiB
CMake
include(AddMLIRPython)
|
|
|
|
# Python bindings need to throw exceptions for proper handling of errors on the python-side
|
|
add_compile_options(-fexceptions)
|
|
|
|
# ######################################################################################################################
|
|
# Support wrapper library for Python
|
|
# ######################################################################################################################
|
|
set(LLVM_OPTIONAL_SOURCES CompilerAPIModule.cpp ConcretelangModule.cpp FHEModule.cpp)
|
|
|
|
add_mlir_public_c_api_library(
|
|
CONCRETELANGPySupport
|
|
CompilerEngine.cpp
|
|
LINK_LIBS
|
|
PUBLIC
|
|
MLIRCAPIIR
|
|
ConcretelangSupport
|
|
ConcretelangCommon)
|
|
|
|
# ######################################################################################################################
|
|
# Decalare native Python extension
|
|
# ######################################################################################################################
|
|
|
|
declare_mlir_python_sources(ConcretelangBindingsPythonExtension)
|
|
|
|
declare_mlir_python_extension(
|
|
ConcretelangBindingsPythonExtension.Core
|
|
MODULE_NAME
|
|
_concretelang
|
|
ADD_TO_PARENT
|
|
ConcretelangBindingsPythonExtension
|
|
SOURCES
|
|
ConcretelangModule.cpp
|
|
FHEModule.cpp
|
|
CompilerAPIModule.cpp
|
|
EMBED_CAPI_LINK_LIBS
|
|
MLIRCAPIRegisterEverything
|
|
CONCRETELANGPySupport)
|
|
|
|
# ######################################################################################################################
|
|
# Declare python sources
|
|
# ######################################################################################################################
|
|
|
|
declare_mlir_python_sources(
|
|
ConcretelangBindingsPythonSources
|
|
ROOT_DIR
|
|
"${CMAKE_CURRENT_SOURCE_DIR}"
|
|
SOURCES
|
|
concrete/compiler/__init__.py
|
|
concrete/compiler/client_parameters.py
|
|
concrete/compiler/client_support.py
|
|
concrete/compiler/compilation_context.py
|
|
concrete/compiler/compilation_feedback.py
|
|
concrete/compiler/compilation_options.py
|
|
concrete/compiler/key_set_cache.py
|
|
concrete/compiler/key_set.py
|
|
concrete/compiler/lambda_argument.py
|
|
concrete/compiler/library_compilation_result.py
|
|
concrete/compiler/library_support.py
|
|
concrete/compiler/library_lambda.py
|
|
concrete/compiler/parameter.py
|
|
concrete/compiler/public_arguments.py
|
|
concrete/compiler/public_result.py
|
|
concrete/compiler/evaluation_keys.py
|
|
concrete/compiler/simulated_value_decrypter.py
|
|
concrete/compiler/simulated_value_exporter.py
|
|
concrete/compiler/utils.py
|
|
concrete/compiler/value.py
|
|
concrete/compiler/value_decrypter.py
|
|
concrete/compiler/value_exporter.py
|
|
concrete/compiler/wrapper.py
|
|
concrete/__init__.py
|
|
concrete/lang/__init__.py
|
|
concrete/lang/dialects/__init__.py
|
|
concrete/lang/dialects/_ods_common.py)
|
|
|
|
# ######################################################################################################################
|
|
# Declare dialect-specific bindings.
|
|
# ######################################################################################################################
|
|
|
|
declare_mlir_python_sources(ConcretelangBindingsPythonSources.Dialects ADD_TO_PARENT ConcretelangBindingsPythonSources)
|
|
|
|
declare_mlir_dialect_python_bindings(
|
|
ADD_TO_PARENT
|
|
ConcretelangBindingsPythonSources.Dialects
|
|
ROOT_DIR
|
|
"${CMAKE_CURRENT_SOURCE_DIR}"
|
|
CONCRETELANGBindingsPythonFHEOps
|
|
TD_FILE
|
|
concrete/lang/dialects/FHEOps.td
|
|
SOURCES
|
|
concrete/lang/dialects/fhe.py
|
|
DIALECT_NAME
|
|
FHE)
|
|
|
|
declare_mlir_dialect_python_bindings(
|
|
ADD_TO_PARENT
|
|
ConcretelangBindingsPythonSources.Dialects
|
|
ROOT_DIR
|
|
"${CMAKE_CURRENT_SOURCE_DIR}"
|
|
CONCRETELANGBindingsPythonFHELinalgOps
|
|
TD_FILE
|
|
concrete/lang/dialects/FHELinalgOps.td
|
|
SOURCES
|
|
concrete/lang/dialects/fhelinalg.py
|
|
DIALECT_NAME
|
|
FHELinalg)
|
|
|
|
declare_mlir_dialect_python_bindings(
|
|
ADD_TO_PARENT
|
|
ConcretelangBindingsPythonSources.Dialects
|
|
ROOT_DIR
|
|
"${CMAKE_CURRENT_SOURCE_DIR}"
|
|
CONCRETELANGBindingsPythonTracingOps
|
|
TD_FILE
|
|
concrete/lang/dialects/TracingOps.td
|
|
SOURCES
|
|
concrete/lang/dialects/tracing.py
|
|
DIALECT_NAME
|
|
Tracing)
|
|
|
|
# ######################################################################################################################
|
|
# Build composite binaries
|
|
# ######################################################################################################################
|
|
|
|
# Bundle our own, self-contained CAPI library with all of our deps.
|
|
add_mlir_python_common_capi_library(
|
|
ConcretelangBindingsPythonCAPI
|
|
INSTALL_COMPONENT
|
|
ConcretelangBindingsPythonModules
|
|
INSTALL_DESTINATION
|
|
python_packages/concretelang_core/mlir/_mlir_libs
|
|
# NOTE: When the MLIR API is relocated under concretelang, this would change to .../concretelang/_mlir_libs
|
|
OUTPUT_DIRECTORY
|
|
"${CONCRETELANG_PYTHON_PACKAGES_DIR}/concretelang_core/mlir/_mlir_libs"
|
|
RELATIVE_INSTALL_ROOT
|
|
"../../../.."
|
|
DECLARED_SOURCES
|
|
# TODO: This can be chopped down significantly for size.
|
|
MLIRPythonSources
|
|
ConcretelangBindingsPythonSources
|
|
ConcretelangBindingsPythonExtension)
|
|
|
|
# Bundle the MLIR python sources into our package. The MLIR API is position independent, so we explicitly output it to
|
|
# the mlir/ folder as a temporary measure. It will eventually migrate under the concretelang/ folder and be accessible
|
|
# under the unified "import concretelang..." namespace.
|
|
add_mlir_python_modules(
|
|
ConcretelangMLIRPythonModules
|
|
ROOT_PREFIX
|
|
"${CONCRETELANG_PYTHON_PACKAGES_DIR}/concretelang_core/mlir"
|
|
INSTALL_PREFIX
|
|
"python_packages/concretelang_core/mlir"
|
|
DECLARED_SOURCES
|
|
MLIRPythonSources
|
|
# We need the circt extensions co-located with the MLIR extensions. When the namespace is unified, this moves to the
|
|
# below.
|
|
ConcretelangBindingsPythonExtension
|
|
COMMON_CAPI_LINK_LIBS
|
|
ConcretelangBindingsPythonCAPI)
|
|
|
|
# Bundle the CONCRETELANG python sources into our package.
|
|
add_mlir_python_modules(
|
|
ConcretelangPythonModules
|
|
ROOT_PREFIX
|
|
"${CONCRETELANG_PYTHON_PACKAGES_DIR}/concretelang_core"
|
|
INSTALL_PREFIX
|
|
"python_packages/concretelang_core"
|
|
DECLARED_SOURCES
|
|
ConcretelangBindingsPythonSources
|
|
COMMON_CAPI_LINK_LIBS
|
|
ConcretelangBindingsPythonCAPI)
|