github/concrete
1
1
Fork 0
mirror of https://github.com/zama-ai/concrete.git synced 2026-02-08 11:35:02 -05:00
Code Issues Packages Projects Releases Wiki Activity

chore(deps): bump compiler to 2022-01-04T16:08:11Z

Browse Source 
- disable PR automerge on compiler update (too risky)
- change init file to be a namespace package
- remove version.py (infos can still be retrieved through package meta)
this avoids potential conflicts with other packages we release
- various fixes for compiler compatibility

closes #1272
This commit is contained in:
IceTDrinker
2022-01-04 16:52:55 +00:00
committed by Arthur Meyre
parent fcbf92609f
commit a48f8cb4f1
16 changed files with 69 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
.github/workflows/package-watcher.yaml vendored
View File

@@ -58,19 +58,6 @@ jobs:
body: "Automatic PR for docker env compiler update"
labels: dependencies
delete-branch: true
- name: Self approve PR to attempt auto merge
if: ${{ fromJSON(env.SHOULD_RUN) && steps.cpr.outputs.pull-request-number != null }}
uses: hmarr/auto-approve-action@6a9ec7556f0a7fa5b49527a1eea4878b8a22d2e0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
pull-request-number: ${{ steps.cpr.outputs.pull-request-number }}
- name: Enable auto-merge for PR
if: ${{ fromJSON(env.SHOULD_RUN) && steps.cpr.outputs.pull-request-number != null }}
uses: peter-evans/enable-pull-request-automerge@d2ede5636b3febc92809259995e643565e675aab
with:
token: ${{ secrets.BOT_TOKEN }}
pull-request-number: ${{ steps.cpr.outputs.pull-request-number }}
merge-method: rebase
- name: Send Slack Notification
if: ${{ always() && failure() }}
continue-on-error: true

8
concrete/__init__.py
View File

@@ -1,3 +1,5 @@
"""Package top import."""
from . import common, numpy, quantization, torch
from .version import __version__
"""Top level import."""
# Do not modify, this is to have a compatible namespace package
# https://packaging.python.org/en/latest/guides/packaging-namespace-packages/
# #pkg-resources-style-namespace-packages
__import__("pkg_resources").declare_namespace(__name__) # pragma: no cover

2
concrete/common/fhe_circuit.py
View File

@@ -4,7 +4,7 @@ from pathlib import Path
from typing import List, Optional, Union
import numpy
from zamalang import CompilerEngine
from concrete.compiler import CompilerEngine
from .debugging import draw_graph, format_operation_graph
from .operator_graph import OPGraph

2
concrete/common/mlir/conversion_helpers.py
View File

@@ -6,8 +6,8 @@
from typing import Optional
from concrete.lang.dialects.fhe import EncryptedIntegerType
from mlir.ir import Context, IntegerType, RankedTensorType, Type
from zamalang.dialects.hlfhe import EncryptedIntegerType
from ..data_types import Integer
from ..values import BaseValue, TensorValue

6
concrete/common/mlir/graph_converter.py
View File

@@ -7,8 +7,8 @@
from abc import ABC, abstractmethod
from typing import Any, Dict, List
import concrete.lang as concretelang
import networkx as nx
import zamalang
from mlir.dialects import builtin
from mlir.ir import Context, InsertionPoint, Location, Module
@@ -44,14 +44,14 @@ class OPGraphConverter(ABC):
# { node1: "%arg0", node2: "%0", node3: "%1" }
nodes_to_mlir_names: Dict[IntermediateNode, str] = {}
# { "%arg0": "i5", "%0": "tensor<2x3x!HLFHE.eint<4>>" }
# { "%arg0": "i5", "%0": "tensor<2x3x!FHE.eint<4>>" }
mlir_names_to_mlir_types: Dict[str, str] = {}
# { "%0": ["%c1_i5"] } == for %0 we need to convert %c1_i5 to 1d tensor
scalar_to_1d_tensor_conversion_hacks: Dict[str, List[str]] = {}
with Context() as ctx, Location.unknown():
zamalang.register_dialects(ctx)
concretelang.register_dialects(ctx)
module = Module.create()
with InsertionPoint(module.body):

34
concrete/common/mlir/node_converter.py
View File

@@ -7,6 +7,7 @@
from typing import Any, Dict, List, Tuple, cast
import numpy
from concrete.lang.dialects import fhe, fhelinalg
from mlir.dialects import arith, linalg, tensor
from mlir.ir import (
ArrayAttr,
@@ -19,7 +20,6 @@ from mlir.ir import (
OpResult,
RankedTensorType,
)
from zamalang.dialects import hlfhe, hlfhelinalg
from ..data_types import Integer
from ..debugging import assert_true
@@ -172,9 +172,9 @@ class IntermediateNodeConverter:
if self.all_of_the_inputs_are_encrypted:
if self.one_of_the_inputs_is_a_tensor:
result = hlfhelinalg.AddEintOp(resulting_type, *preds).result
result = fhelinalg.AddEintOp(resulting_type, *preds).result
else:
result = hlfhe.AddEintOp(resulting_type, *preds).result
result = fhe.AddEintOp(resulting_type, *preds).result
else:
if self.node.inputs[0].is_clear: # pragma: no cover
# this branch is not covered as it's impossible to get into due to how tracing works
@@ -182,9 +182,9 @@ class IntermediateNodeConverter:
preds = preds[::-1]
if self.one_of_the_inputs_is_a_tensor:
result = hlfhelinalg.AddEintIntOp(resulting_type, *preds).result
result = fhelinalg.AddEintIntOp(resulting_type, *preds).result
else:
result = hlfhe.AddEintIntOp(resulting_type, *preds).result
result = fhe.AddEintIntOp(resulting_type, *preds).result
return result
@@ -247,15 +247,15 @@ class IntermediateNodeConverter:
if self.all_of_the_inputs_are_tensors:
# numpy.dot(x, y) where x and y are both vectors = regular dot product
result = hlfhelinalg.Dot(resulting_type, *preds).result
result = fhelinalg.Dot(resulting_type, *preds).result
elif not self.one_of_the_inputs_is_a_tensor:
# numpy.dot(x, y) where x and y are both scalars = x * y
result = hlfhe.MulEintIntOp(resulting_type, *preds).result
result = fhe.MulEintIntOp(resulting_type, *preds).result
else:
# numpy.dot(x, y) where one of x or y is a scalar and the other one is a vector = x * y
result = hlfhelinalg.MulEintIntOp(resulting_type, *preds).result
result = fhelinalg.MulEintIntOp(resulting_type, *preds).result
return result
@@ -326,11 +326,11 @@ class IntermediateNodeConverter:
if self.one_of_the_inputs_is_a_tensor:
if len(tables) == 1:
result = hlfhelinalg.ApplyLookupTableEintOp(resulting_type, pred, lut).result
result = fhelinalg.ApplyLookupTableEintOp(resulting_type, pred, lut).result
else:
result = hlfhelinalg.ApplyMultiLookupTableEintOp(resulting_type, pred, lut).result
result = fhelinalg.ApplyMultiLookupTableEintOp(resulting_type, pred, lut).result
else:
result = hlfhe.ApplyLookupTableEintOp(resulting_type, pred, lut).result
result = fhe.ApplyLookupTableEintOp(resulting_type, pred, lut).result
return result
@@ -447,9 +447,9 @@ class IntermediateNodeConverter:
preds = self.preds
if self.node.inputs[0].is_clear:
result = hlfhelinalg.MatMulIntEintOp(resulting_type, *preds).result
result = fhelinalg.MatMulIntEintOp(resulting_type, *preds).result
else:
result = hlfhelinalg.MatMulEintIntOp(resulting_type, *preds).result
result = fhelinalg.MatMulEintIntOp(resulting_type, *preds).result
return result
@@ -479,9 +479,9 @@ class IntermediateNodeConverter:
preds = preds[::-1]
if self.one_of_the_inputs_is_a_tensor:
result = hlfhelinalg.MulEintIntOp(resulting_type, *preds).result
result = fhelinalg.MulEintIntOp(resulting_type, *preds).result
else:
result = hlfhe.MulEintIntOp(resulting_type, *preds).result
result = fhe.MulEintIntOp(resulting_type, *preds).result
return result
@@ -629,8 +629,8 @@ class IntermediateNodeConverter:
preds = self.preds
if self.one_of_the_inputs_is_a_tensor:
result = hlfhelinalg.SubIntEintOp(resulting_type, *preds).result
result = fhelinalg.SubIntEintOp(resulting_type, *preds).result
else:
result = hlfhe.SubIntEintOp(resulting_type, *preds).result
result = fhe.SubIntEintOp(resulting_type, *preds).result
return result

2
concrete/numpy/compile.py
View File

@@ -6,7 +6,7 @@ from copy import deepcopy
from typing import Any, Callable, Dict, Iterable, Optional, Tuple, Union, cast
import numpy
from zamalang import CompilerEngine
from concrete.compiler import CompilerEngine
from ..common.bounds_measurement.inputset_eval import eval_op_graph_bounds_on_inputset
from ..common.common_helpers import check_op_graph_is_integer_program

4
concrete/version.py
View File

@@ -1,4 +0,0 @@
"""Package version module."""
# Auto-generated by "make set_version" do not modify
__version__ = "0.2.0-rc5"

2
docker/Dockerfile.concrete-framework-env
View File

@@ -1,7 +1,7 @@
FROM ubuntu:20.04
# Do not change the line below it will be updated automatically when the docker is regenerated
# compiler timestamp: 2021-12-22T10:49:30Z
# compiler timestamp: 2022-01-04T16:08:11Z
# Remove once compiler is on PyPi https://github.com/zama-ai/concrete-framework-internal/issues/809
ARG WHEEL

18
docs/dev/explanation/compilation.md
View File

@@ -252,12 +252,12 @@ Data Types:
```
module {
func @main(%arg0: !HLFHE.eint<4>) -> !HLFHE.eint<4> {
func @main(%arg0: !FHE.eint<4>) -> !FHE.eint<4> {
%c3_i5 = constant 3 : i5
%c2_i5 = constant 2 : i5
%0 = "HLFHE.mul_eint_int"(%arg0, %c2_i5) : (!HLFHE.eint<4>, i5) -> !HLFHE.eint<4>
%1 = "HLFHE.add_eint_int"(%0, %c3_i5) : (!HLFHE.eint<4>, i5) -> !HLFHE.eint<4>
return %1 : !HLFHE.eint<4>
%0 = "FHE.mul_eint_int"(%arg0, %c2_i5) : (!FHE.eint<4>, i5) -> !FHE.eint<4>
%1 = "FHE.add_eint_int"(%0, %c3_i5) : (!FHE.eint<4>, i5) -> !FHE.eint<4>
return %1 : !FHE.eint<4>
}
}
```
@@ -349,13 +349,13 @@ Data Types:
```
module {
func @main(%arg0: !HLFHE.eint<6>, %arg1: !HLFHE.eint<6>) -> !HLFHE.eint<6> {
func @main(%arg0: !FHE.eint<6>, %arg1: !FHE.eint<6>) -> !FHE.eint<6> {
%c42_i7 = constant 42 : i7
%c2_i7 = constant 2 : i7
%0 = "HLFHE.sub_int_eint"(%c42_i7, %arg0) : (i7, !HLFHE.eint<6>) -> !HLFHE.eint<6>
%1 = "HLFHE.mul_eint_int"(%arg1, %c2_i7) : (!HLFHE.eint<6>, i7) -> !HLFHE.eint<6>
%2 = "HLFHE.add_eint"(%0, %1) : (!HLFHE.eint<6>, !HLFHE.eint<6>) -> !HLFHE.eint<6>
return %2 : !HLFHE.eint<6>
%0 = "FHE.sub_int_eint"(%c42_i7, %arg0) : (i7, !FHE.eint<6>) -> !FHE.eint<6>
%1 = "FHE.mul_eint_int"(%arg1, %c2_i7) : (!FHE.eint<6>, i7) -> !FHE.eint<6>
%2 = "FHE.add_eint"(%0, %1) : (!FHE.eint<6>, !FHE.eint<6>) -> !FHE.eint<6>
return %2 : !FHE.eint<6>
}
}
```

8
docs/dev/explanation/mlir.md
View File

@@ -3,9 +3,9 @@
MLIR is the intermediate representation used by the **Concrete** compiler, so we need to convert the operation graph to MLIR, which will look something like the following, for a graph performing the dot between two input tensors.
```
func @main(%arg0: tensor<4xi7>, %arg1: tensor<4x!HLFHE.eint<6>>) -> !HLFHE.eint<6> {
%0 = "HLFHE.dot_eint_int"(%arg1, %arg0) : (tensor<4x!HLFHE.eint<6>>, tensor<4xi7>) -> !HLFHE.eint<6>
return %0 : !HLFHE.eint<6>
func @main(%arg0: tensor<4xi7>, %arg1: tensor<4x!FHE.eint<6>>) -> !FHE.eint<6> {
%0 = "FHE.dot_eint_int"(%arg1, %arg0) : (tensor<4x!FHE.eint<6>>, tensor<4xi7>) -> !FHE.eint<6>
return %0 : !FHE.eint<6>
}
```
@@ -17,7 +17,7 @@ The conversion uses as input the operation graph to convert, as well as a dictio
## Define function signature
The first step would be to define the function signature (excluding return value at this point). We will convert input node's types to MLIR (e.g. convert `EncryptedTensor(Integer(64, is_signed=False), shape=(4,))` to `tensor<4xi64>`) and map their values to the argument of the function. So if we had an operation graph with one `EncryptedScalar(Integer(7, is_signed=False))`, we will get an MLIR function like `func @main(%arg0 : !HLFHE.eint<7>) -> (<ret-type>)`. Note that the return type would be detected automatically later on when returning MLIR values.
The first step would be to define the function signature (excluding return value at this point). We will convert input node's types to MLIR (e.g. convert `EncryptedTensor(Integer(64, is_signed=False), shape=(4,))` to `tensor<4xi64>`) and map their values to the argument of the function. So if we had an operation graph with one `EncryptedScalar(Integer(7, is_signed=False))`, we will get an MLIR function like `func @main(%arg0 : !FHE.eint<7>) -> (<ret-type>)`. Note that the return type would be detected automatically later on when returning MLIR values.
## Convert nodes in the OpGraph

8
docs/user/tutorial/compilation_artifacts.md
View File

@@ -234,12 +234,12 @@ This file contains information about the MLIR of the function you are trying to
```
module {
func @main(%arg0: !HLFHE.eint<7>) -> !HLFHE.eint<7> {
func @main(%arg0: !FHE.eint<7>) -> !FHE.eint<7> {
%c127_i8 = arith.constant 127 : i8
%cst = arith.constant dense<"..."> : tensor<128xi64>
%0 = "HLFHE.apply_lookup_table"(%arg0, %cst) : (!HLFHE.eint<7>, tensor<128xi64>) -> !HLFHE.eint<7>
%1 = "HLFHE.sub_int_eint"(%c127_i8, %0) : (i8, !HLFHE.eint<7>) -> !HLFHE.eint<7>
return %1 : !HLFHE.eint<7>
%0 = "FHE.apply_lookup_table"(%arg0, %cst) : (!FHE.eint<7>, tensor<128xi64>) -> !FHE.eint<7>
%1 = "FHE.sub_int_eint"(%c127_i8, %0) : (i8, !FHE.eint<7>) -> !FHE.eint<7>
return %1 : !FHE.eint<7>
}
}

15
poetry.lock generated
View File

@@ -498,7 +498,7 @@ python-versions = "*"
[[package]]
name = "ipykernel"
version = "6.6.0"
version = "6.6.1"
description = "IPython Kernel for Jupyter"
category = "dev"
optional = false
@@ -510,6 +510,7 @@ debugpy = ">=1.0.0,<2.0"
ipython = ">=7.23.1"
jupyter-client = "<8.0"
matplotlib-inline = ">=0.1.0,<0.2.0"
nest-asyncio = "*"
tornado = ">=4.2,<7.0"
traitlets = ">=5.1.0,<6.0"
@@ -964,7 +965,7 @@ test = ["codecov", "coverage", "ipython", "ipykernel", "ipywidgets", "pytest (>=
[[package]]
name = "nbconvert"
version = "6.3.0"
version = "6.4.0"
description = "Converting Jupyter Notebooks"
category = "dev"
optional = false
@@ -2247,7 +2248,7 @@ full = ["pygraphviz"]
[metadata]
lock-version = "1.1"
python-versions = ">=3.8,<3.10"
content-hash = "2ce4daeb16628c833d97527e34108b84bce5c5167d6a6a78d62517bcbb5938bc"
content-hash = "4fbcb00a68b38cf9bfc977c6c750f4f76852bdd704f5346bc19752ca7c4b6905"
[metadata.files]
alabaster = [
@@ -2568,8 +2569,8 @@ invoke = [
{file = "invoke-1.6.0.tar.gz", hash = "sha256:374d1e2ecf78981da94bfaf95366216aaec27c2d6a7b7d5818d92da55aa258d3"},
]
ipykernel = [
{file = "ipykernel-6.6.0-py3-none-any.whl", hash = "sha256:82ded8919fa7f5483be2b6219c3b13380d93faab1fc49cc2cfcd10e9e24cc158"},
{file = "ipykernel-6.6.0.tar.gz", hash = "sha256:3a227788216b43982d9ac28195949467627b0d16e6b8af9741d95dcaa8c41a89"},
{file = "ipykernel-6.6.1-py3-none-any.whl", hash = "sha256:de99f6c1caa72578305cc96122ee3a19669e9c1958694a2b564ed1be28240ab9"},
{file = "ipykernel-6.6.1.tar.gz", hash = "sha256:91ff0058b45660aad4a68088041059c0d378cd53fc8aff60e5abc91bcc049353"},
]
ipython = [
{file = "ipython-7.30.1-py3-none-any.whl", hash = "sha256:fc60ef843e0863dd4e24ab2bb5698f071031332801ecf8d1aeb4fb622056545c"},
@@ -2931,8 +2932,8 @@ nbclient = [
{file = "nbclient-0.5.9.tar.gz", hash = "sha256:99e46ddafacd0b861293bf246fed8540a184adfa3aa7d641f89031ec070701e0"},
]
nbconvert = [
{file = "nbconvert-6.3.0-py3-none-any.whl", hash = "sha256:8f23fbeabda4a500685d788ee091bf22cf34119304314304fb39f16e2fc32f37"},
{file = "nbconvert-6.3.0.tar.gz", hash = "sha256:5e77d6203854944520105e38f2563a813a4a3708e8563aa598928a3b5ee1081a"},
{file = "nbconvert-6.4.0-py3-none-any.whl", hash = "sha256:f5ec6a1fad9e3aa2bee7c6a1c4ad3e0fafaa7ff64f29ba56d9da7e1669f8521c"},
{file = "nbconvert-6.4.0.tar.gz", hash = "sha256:5412ec774c6db4fccecb8c4ba07ec5d37d6dcf5762593cb3d6ecbbeb562ebbe5"},
]
nbformat = [
{file = "nbformat-5.1.3-py3-none-any.whl", hash = "sha256:eb8447edd7127d043361bc17f2f5a807626bc8e878c7709a1c647abda28a9171"},

4
pylintrc
View File

@@ -9,7 +9,7 @@ extension-pkg-allow-list=
# be loaded. Extensions are loading into the active Python interpreter and may
# run arbitrary code. (This is an alternative name to extension-pkg-allow-list
# for backward compatibility.)
extension-pkg-whitelist=zamalang,mlir
extension-pkg-whitelist=concrete,mlir
# Return non-zero exit code if any of these messages/categories are detected,
# even if score is above --fail-under value. Syntax same as enable. Messages
@@ -610,7 +610,7 @@ int-import-graph=
known-standard-library=
# Force import order to recognize a module as part of a third party library.
known-third-party=enchant
known-third-party=enchant,concrete,mlir
# Couples of modules and preferred modules, separated by a comma.
preferred-modules=

3
pyproject.toml
View File

@@ -16,6 +16,7 @@ pygraphviz = { version = "^1.7", optional = true }
Pillow = "^8.3.2"
loguru = "^0.5.3"
torch = "^1.10.0"
setuptools = "*"
[tool.poetry.extras]
full = ["pygraphviz"]
@@ -67,5 +68,5 @@ filterwarnings = [
[tool.semantic_release]
version_toml = "pyproject.toml:tool.poetry.version"
version_variable = "concrete/version.py:__version__,docs/conf.py:release"
version_variable = "docs/conf.py:release"
upload_to_pypi = "False"

24
tests/common/mlir/test_conversion_helpers.py
View File

@@ -4,8 +4,8 @@
# pylint: disable=no-name-in-module
import concrete.lang as concretelang
import pytest
import zamalang
from mlir.ir import Context, Location
from concrete.common.data_types import Float, SignedInteger, UnsignedInteger
@@ -22,15 +22,15 @@ from concrete.common.values import ClearScalar, ClearTensor, EncryptedScalar, En
pytest.param(UnsignedInteger(5), False, "i5"),
pytest.param(SignedInteger(32), False, "i32"),
pytest.param(UnsignedInteger(32), False, "i32"),
pytest.param(SignedInteger(5), True, "!HLFHE.eint<5>"),
pytest.param(UnsignedInteger(5), True, "!HLFHE.eint<5>"),
pytest.param(SignedInteger(5), True, "!FHE.eint<5>"),
pytest.param(UnsignedInteger(5), True, "!FHE.eint<5>"),
],
)
def test_integer_to_mlir_type(integer, is_encrypted, expected_mlir_type_str):
"""Test function for integer to MLIR type conversion."""
with Context() as ctx, Location.unknown():
zamalang.register_dialects(ctx)
concretelang.register_dialects(ctx)
assert str(integer_to_mlir_type(ctx, integer, is_encrypted)) == expected_mlir_type_str
@@ -46,7 +46,7 @@ def test_fail_integer_to_mlir_type(integer, is_encrypted, expected_error_message
with pytest.raises(ValueError) as excinfo:
with Context() as ctx, Location.unknown():
zamalang.register_dialects(ctx)
concretelang.register_dialects(ctx)
integer_to_mlir_type(ctx, integer, is_encrypted)
assert str(excinfo.value) == expected_error_message
@@ -57,21 +57,19 @@ def test_fail_integer_to_mlir_type(integer, is_encrypted, expected_error_message
[
pytest.param(ClearScalar(SignedInteger(5)), "i5"),
pytest.param(ClearTensor(SignedInteger(5), shape=(2, 3)), "tensor<2x3xi5>"),
pytest.param(EncryptedScalar(SignedInteger(5)), "!HLFHE.eint<5>"),
pytest.param(EncryptedTensor(SignedInteger(5), shape=(2, 3)), "tensor<2x3x!HLFHE.eint<5>>"),
pytest.param(EncryptedScalar(SignedInteger(5)), "!FHE.eint<5>"),
pytest.param(EncryptedTensor(SignedInteger(5), shape=(2, 3)), "tensor<2x3x!FHE.eint<5>>"),
pytest.param(ClearScalar(UnsignedInteger(5)), "i5"),
pytest.param(ClearTensor(UnsignedInteger(5), shape=(2, 3)), "tensor<2x3xi5>"),
pytest.param(EncryptedScalar(UnsignedInteger(5)), "!HLFHE.eint<5>"),
pytest.param(
EncryptedTensor(UnsignedInteger(5), shape=(2, 3)), "tensor<2x3x!HLFHE.eint<5>>"
),
pytest.param(EncryptedScalar(UnsignedInteger(5)), "!FHE.eint<5>"),
pytest.param(EncryptedTensor(UnsignedInteger(5), shape=(2, 3)), "tensor<2x3x!FHE.eint<5>>"),
],
)
def test_value_to_mlir_type(value, expected_mlir_type_str):
"""Test function for value to MLIR type conversion."""
with Context() as ctx, Location.unknown():
zamalang.register_dialects(ctx)
concretelang.register_dialects(ctx)
assert str(value_to_mlir_type(ctx, value)) == expected_mlir_type_str
@@ -109,7 +107,7 @@ def test_fail_value_to_mlir_type(value, expected_error_message):
with pytest.raises(TypeError) as excinfo:
with Context() as ctx, Location.unknown():
zamalang.register_dialects(ctx)
concretelang.register_dialects(ctx)
value_to_mlir_type(ctx, value)
assert str(excinfo.value) == expected_error_message