Files
concrete/compilers/concrete-compiler/compiler/lib/Support/Encodings.cpp
Alexandre Péré e8ef48ffd8 feat(compiler): introduce concrete-protocol
This commit:
 + Adds support for a protocol which enables inter-op between concrete,
   tfhe-rs and potentially other contributors to the fhe ecosystem.
 + Gets rid of hand-made serialization in the compiler, and
   client/server libs.
 + Refactors client/server libs to allow more pre/post processing of
   circuit inputs/outputs.

The protocol is supported by a definition in the shape of a capnp file,
which defines different types of objects among which:
 + ProgramInfo object, which is a precise description of a set of fhe
   circuit coming from the same compilation (understand function type
   information), and the associated key set.
 + *Key objects, which represent secret/public keys used to
   encrypt/execute fhe circuits.
 + Value object, which represent values that can be transferred between
   client and server to support calls to fhe circuits.

The hand-rolled serialization that was previously used is completely
dropped in favor of capnp in the whole codebase.

The client/server libs, are refactored to introduce a modular design for
pre-post processing. Reading the ProgramInfo file associated with a
compilation, the client and server libs assemble a pipeline of
transformers (functions) for pre and post processing of values coming in
and out of a circuit. This design properly decouples various aspects of
the processing, and allows these capabilities to be safely extended.

In practice this commit includes the following:
 + Defines the specification in a concreteprotocol package
 + Integrate the compilation of this package as a compiler dependency
   via cmake
 + Modify the compiler to use the Encodings objects defined in the
   protocol
 + Modify the compiler to emit ProgramInfo files as compilation
   artifact, and gets rid of the bloated ClientParameters.
 + Introduces a new Common library containing the functionalities shared
   between the compiler and the client/server libs.
 + Introduces a functional pre-post processing pipeline to this common
   library
 + Modify the client/server libs to support loading ProgramInfo objects,
   and calling circuits using Value messages.
 + Drops support of JIT.
 + Drops support of C-api.
 + Drops support of Rust bindings.

Co-authored-by: Nikita Frolov <nf@mkmks.org>
2023-11-09 17:09:04 +01:00

177 lines
6.5 KiB
C++

// Part of the Concrete Compiler Project, under the BSD3 License with Zama
// Exceptions. See
// https://github.com/zama-ai/concrete-compiler-internal/blob/main/LICENSE.txt
// for license information.
#include "concretelang/Support/Encodings.h"
#include "concrete-protocol.capnp.h"
#include "concretelang/Common/Protocol.h"
#include "concretelang/Dialect/FHE/IR/FHETypes.h"
#include "concretelang/Support/Error.h"
#include "concretelang/Support/Utils.h"
#include "concretelang/Support/V0Parameters.h"
#include "concretelang/Support/Variants.h"
#include "kj/common.h"
#include <functional>
#include <memory>
#include <optional>
#include <variant>
namespace FHE = mlir::concretelang::FHE;
using concretelang::protocol::Message;
namespace mlir {
namespace concretelang {
namespace encodings {
llvm::Expected<Message<concreteprotocol::EncodingInfo>>
encodingFromType(mlir::Type ty) {
if (auto eintTy = ty.dyn_cast<FHE::FheIntegerInterface>()) {
auto output = Message<concreteprotocol::EncodingInfo>();
auto encodingBuilder =
output.asBuilder().getEncoding().initIntegerCiphertext();
encodingBuilder.setIsSigned(eintTy.isSigned());
encodingBuilder.setWidth(eintTy.getWidth());
output.asBuilder().getShape().initDimensions(0);
return std::move(output);
} else if (auto eboolTy = ty.dyn_cast<FHE::EncryptedBooleanType>()) {
auto output = Message<concreteprotocol::EncodingInfo>();
output.asBuilder().getEncoding().initBooleanCiphertext();
output.asBuilder().getShape().initDimensions(0);
return std::move(output);
} else if (auto intTy = ty.dyn_cast<mlir::IntegerType>()) {
auto output = Message<concreteprotocol::EncodingInfo>();
output.asBuilder().getEncoding().initPlaintext();
output.asBuilder().getShape().initDimensions(0);
return std::move(output);
} else if (auto indexTy = ty.dyn_cast<mlir::IndexType>()) {
auto output = Message<concreteprotocol::EncodingInfo>();
output.asBuilder().getEncoding().initIndex();
output.asBuilder().getShape().initDimensions(0);
return std::move(output);
} else if (auto tensorTy = ty.dyn_cast<mlir::RankedTensorType>()) {
auto maybeElementEncoding = encodingFromType(tensorTy.getElementType());
if (!maybeElementEncoding) {
return maybeElementEncoding.takeError();
}
auto output = std::move(*maybeElementEncoding);
auto shapeBuilder =
output.asBuilder().initShape().initDimensions(tensorTy.getRank());
for (int64_t i = 0; i < tensorTy.getRank(); i++) {
shapeBuilder.set(i, tensorTy.getShape()[i]);
}
return std::move(output);
}
return StreamStringError("Failed to recognize encoding for type : ") << ty;
}
llvm::Expected<Message<concreteprotocol::CircuitEncodingInfo>>
getCircuitEncodings(llvm::StringRef functionName, mlir::ModuleOp module) {
// Find the input function
auto rangeOps = module.getOps<mlir::func::FuncOp>();
auto funcOp = llvm::find_if(rangeOps, [&](mlir::func::FuncOp op) {
return op.getName() == functionName;
});
if (funcOp == rangeOps.end()) {
return StreamStringError("Function not found, name='")
<< functionName << "', cannot get circuit encodings";
}
auto funcType = (*funcOp).getFunctionType();
// Retrieve input/output encodings
auto circuitEncodings = Message<concreteprotocol::CircuitEncodingInfo>();
auto inputsBuilder =
circuitEncodings.asBuilder().initInputs(funcType.getNumInputs());
for (size_t i = 0; i < funcType.getNumInputs(); i++) {
auto ty = funcType.getInputs()[i];
auto maybeEncoding = encodingFromType(ty);
if (!maybeEncoding) {
return maybeEncoding.takeError();
}
inputsBuilder.setWithCaveats(i, maybeEncoding->asReader());
}
auto outputsBuilder =
circuitEncodings.asBuilder().initOutputs(funcType.getNumResults());
for (size_t i = 0; i < funcType.getNumResults(); i++) {
auto ty = funcType.getResults()[i];
auto maybeEncoding = encodingFromType(ty);
if (!maybeEncoding) {
return maybeEncoding.takeError();
}
outputsBuilder.setWithCaveats(i, maybeEncoding->asReader());
}
return std::move(circuitEncodings);
}
void setCircuitEncodingModes(
Message<concreteprotocol::CircuitEncodingInfo> &info,
std::optional<
Message<concreteprotocol::IntegerCiphertextEncodingInfo::ChunkedMode>>
maybeChunk,
std::optional<V0FHEContext> maybeFheContext) {
auto setMode = [&](concreteprotocol::EncodingInfo::Builder enc) {
if (!enc.getEncoding().hasIntegerCiphertext()) {
return;
}
auto integerEncodingBuilder = enc.getEncoding().getIntegerCiphertext();
// Chunks wanted. Setting encoding mode to chunks ...
if (maybeChunk) {
integerEncodingBuilder.getMode().setChunked(
maybeChunk.value().asReader());
return;
}
// Got v0 solution with crt decomposition. Setting encoding mode to crt.
if (maybeFheContext.has_value()) {
if (std::holds_alternative<V0Parameter>(maybeFheContext->solution)) {
auto v0ParameterSol = std::get<V0Parameter>(maybeFheContext->solution);
if (v0ParameterSol.largeInteger.has_value()) {
auto moduli = v0ParameterSol.largeInteger->crtDecomposition;
auto moduliBuilder =
integerEncodingBuilder.getMode().initCrt().initModuli(
moduli.size());
for (size_t i = 0; i < moduli.size(); i++) {
moduliBuilder.set(i, moduli[i]);
}
return;
}
}
}
// Got circuit solution with crt decomposition. Setting encoding mode to
// crt.
if (maybeFheContext.has_value()) {
if (std::holds_alternative<optimizer::CircuitSolution>(
maybeFheContext->solution)) {
auto circuitSol =
std::get<optimizer::CircuitSolution>(maybeFheContext->solution);
if (!circuitSol.crt_decomposition.empty()) {
auto moduli = circuitSol.crt_decomposition;
auto moduliBuilder =
integerEncodingBuilder.getMode().initCrt().initModuli(
moduli.size());
for (size_t i = 0; i < moduli.size(); i++) {
moduliBuilder.set(i, moduli[i]);
}
return;
}
}
}
// Got nothing particular. Setting encoding mode to native.
integerEncodingBuilder.getMode().initNative();
};
for (auto encInfoBuilder : info.asBuilder().getInputs()) {
setMode(encInfoBuilder);
}
for (auto encInfoBuilder : info.asBuilder().getOutputs()) {
setMode(encInfoBuilder);
}
}
} // namespace encodings
} // namespace concretelang
} // namespace mlir