mirror of
https://github.com/zama-ai/concrete.git
synced 2026-01-10 13:27:57 -05:00
e8ef48ffd8
This commit: + Adds support for a protocol which enables inter-op between concrete, tfhe-rs and potentially other contributors to the fhe ecosystem. + Gets rid of hand-made serialization in the compiler, and client/server libs. + Refactors client/server libs to allow more pre/post processing of circuit inputs/outputs. The protocol is supported by a definition in the shape of a capnp file, which defines different types of objects among which: + ProgramInfo object, which is a precise description of a set of fhe circuit coming from the same compilation (understand function type information), and the associated key set. + *Key objects, which represent secret/public keys used to encrypt/execute fhe circuits. + Value object, which represent values that can be transferred between client and server to support calls to fhe circuits. The hand-rolled serialization that was previously used is completely dropped in favor of capnp in the whole codebase. The client/server libs, are refactored to introduce a modular design for pre-post processing. Reading the ProgramInfo file associated with a compilation, the client and server libs assemble a pipeline of transformers (functions) for pre and post processing of values coming in and out of a circuit. This design properly decouples various aspects of the processing, and allows these capabilities to be safely extended. In practice this commit includes the following: + Defines the specification in a concreteprotocol package + Integrate the compilation of this package as a compiler dependency via cmake + Modify the compiler to use the Encodings objects defined in the protocol + Modify the compiler to emit ProgramInfo files as compilation artifact, and gets rid of the bloated ClientParameters. + Introduces a new Common library containing the functionalities shared between the compiler and the client/server libs. + Introduces a functional pre-post processing pipeline to this common library + Modify the client/server libs to support loading ProgramInfo objects, and calling circuits using Value messages. + Drops support of JIT. + Drops support of C-api. + Drops support of Rust bindings. Co-authored-by: Nikita Frolov <nf@mkmks.org>
48 lines
1.9 KiB
Docker
48 lines
1.9 KiB
Docker
FROM quay.io/pypa/manylinux_2_28_x86_64:2022-11-19-1b19e81
|
|
|
|
# epel-release is for install ccache
|
|
RUN dnf install -y epel-release
|
|
RUN dnf update -y
|
|
RUN dnf install -y ninja-build hwloc-devel ccache ncurses-devel
|
|
RUN dnf install -y openssh-clients
|
|
RUN dnf clean all
|
|
RUN mkdir -p ~/.ssh/ && ssh-keyscan -t ecdsa github.com >> ~/.ssh/known_hosts
|
|
# setup ccache with an unlimited amount of files and storage
|
|
RUN ccache -M 0
|
|
RUN ccache -F 0
|
|
# Install Rust
|
|
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
|
|
ENV PATH=/root/.cargo/bin:$PATH
|
|
RUN rustup install nightly
|
|
SHELL ["/bin/bash", "-c"]
|
|
# Install boost
|
|
ADD https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/boost_1_71_0.tar.gz /boost_1_71_0.tar.gz
|
|
RUN tar -xzvf /boost_1_71_0.tar.gz
|
|
WORKDIR /boost_1_71_0
|
|
RUN ./bootstrap.sh && ./b2 --with-filesystem install
|
|
# Setup HPX
|
|
COPY --from=ghcr.io/zama-ai/hpx:latest /hpx /hpx
|
|
ENV HPX_INSTALL_DIR=/hpx/build
|
|
# Setup CUDA
|
|
COPY --from=ghcr.io/zama-ai/cuda:11-7 /usr/local/cuda-11.7/ /usr/local/cuda-11.7/
|
|
COPY --from=ghcr.io/zama-ai/cuda:11-7 /usr/lib64/libcuda.so* /usr/lib64/
|
|
ENV PATH "$PATH:/usr/local/cuda-11.7/bin"
|
|
# Set the python path. Options: [cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310]
|
|
# Links and env would be available to use the appropriate python version
|
|
ARG python_tag=cp38-cp38
|
|
RUN ln -s /opt/python/${python_tag}/bin/pip /bin/pip
|
|
RUN ln -s /opt/python/${python_tag}/bin/python /bin/python
|
|
ENV PYTHON_EXEC=/opt/python/${python_tag}/bin/python
|
|
# Install python deps
|
|
RUN pip install numpy pybind11==2.8 PyYAML pytest wheel auditwheel
|
|
# Setup and build compiler
|
|
COPY / /workdir
|
|
WORKDIR /workdir/compilers/concrete-compiler/compiler
|
|
RUN mkdir -p /build
|
|
RUN --mount=type=ssh make DATAFLOW_EXECUTION_ENABLED=ON BUILD_DIR=/build CCACHE=ON \
|
|
Python3_EXECUTABLE=${PYTHON_EXEC} \
|
|
concretecompiler python-bindings
|
|
ENV PYTHONPATH "$PYTHONPATH:/build/tools/concretelang/python_packages/concretelang_core"
|
|
ENV PATH "$PATH:/build/bin"
|
|
RUN ccache -z
|