github/concrete
1
1
Fork 0
mirror of https://github.com/zama-ai/concrete.git synced 2026-02-10 04:35:03 -05:00
Code Issues Packages Projects Releases Wiki Activity
Files
94d68515d462145eea1e7cab0145cdd0caf41930
concrete/compiler/Makefile
David Testé d1db4a5e45 tests(benchmarks): reduce number of test cases to speed-up execution 
Bench just one compilation option for automatic benchmarks. Only 'loop'
option is tested to take advantage of hardware with a lot of available
CPUs. Running benchmarks with 'default' option is suboptimal for this
kind of hardware since it uses only one CPU.

This also remove time consuming MNIST test, as it should be in ML benchmarks.

Moreover Makefile is fixed to use provided Python executable instead of
relying on system one to generate MLIR Yaml files.
2022-11-03 19:07:39 +01:00

360 lines
12 KiB
Makefile
Raw Blame History

BUILD_TYPE?=Release
BUILD_DIR?=./build
Python3_EXECUTABLE?=python3
BINDINGS_PYTHON_ENABLED=ON
DATAFLOW_EXECUTION_ENABLED=OFF
TIMING_ENABLED=OFF
CC_COMPILER=
CXX_COMPILER=
CUDA_SUPPORT=OFF
CONCRETE_CORE_PATH?=
CONCRETE_OPTIMIZER_DIR ?= $(shell pwd)/concrete-optimizer
KEYSETCACHEDEV=/tmp/KeySetCache
KEYSETCACHECI ?= ../KeySetCache
KEYSETCACHENAME ?= KeySetCacheV2
HPX_VERSION?=1.7.1
HPX_URL=https://github.com/STEllAR-GROUP/hpx/archive/refs/tags/$(HPX_VERSION).tar.gz
HPX_TARBALL=$(shell pwd)/hpx-$(HPX_VERSION).tar.gz
HPX_LOCAL_DIR=$(shell pwd)/hpx-$(HPX_VERSION)
HPX_INSTALL_DIR?=$(HPX_LOCAL_DIR)/build
CONCRETE_CORE_FFI_VERSION?=0.2.0-rc.5
ifeq ($(shell uname), Linux)
CONCRETE_CORE_FFI_TARBALL=concrete-core-ffi_$(CONCRETE_CORE_FFI_VERSION)_linux_amd64.tar.gz
else
CONCRETE_CORE_FFI_TARBALL=concrete-core-ffi_$(CONCRETE_CORE_FFI_VERSION)_darwin_amd64.tar.gz
endif
export PATH := $(abspath $(BUILD_DIR))/bin:$(PATH)
ifeq ($(shell which ccache),)
CCACHE=OFF
else
CCACHE=ON
endif
ifeq ($(CCACHE),ON)
CMAKE_CCACHE_OPTIONS=-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
else
CMAKE_CCACHE_OPTIONS=
endif
ifneq ($(CC_COMPILER),)
CC_COMPILER_OPTION=-DCMAKE_C_COMPILER=$(CC_COMPILER)
else
CC_COMPILER_OPTION=
endif
ifneq ($(CXX_COMPILER),)
CXX_COMPILER_OPTION=-DCMAKE_CXX_COMPILER=$(CXX_COMPILER)
else
CXX_COMPILER_OPTION=
endif
# don't run parallel python tests if compiler doesn't support it
ifeq ($(DATAFLOW_EXECUTION_ENABLED),ON)
PYTHON_TESTS_MARKER=""
else
PYTHON_TESTS_MARKER="not parallel"
endif
ifneq ($(CONCRETE_CORE_PATH),)
CONCRETE_CORE_PATH_OPTION=-DCONCRETE_CORE_PATH=$(CONCRETE_CORE_PATH)
endif
all: concretecompiler python-bindings build-tests build-benchmarks build-mlbench doc
# concrete-core-ffi #######################################
CONCRETE_CORE_FFI_FOLDER = $(shell pwd)/concrete-core-ffi-$(CONCRETE_CORE_FFI_VERSION)
CONCRETE_CORE_FFI_ARTIFACTS=$(CONCRETE_CORE_FFI_FOLDER)/libconcrete_core_ffi.a $(CONCRETE_CORE_FFI_FOLDER)/concrete-core-ffi.h
concrete-core-ffi: $(CONCRETE_CORE_FFI_ARTIFACTS)
$(CONCRETE_CORE_FFI_ARTIFACTS): $(CONCRETE_CORE_FFI_FOLDER)
$(CONCRETE_CORE_FFI_FOLDER): $(CONCRETE_CORE_FFI_TARBALL)
mkdir -p $(CONCRETE_CORE_FFI_FOLDER)
tar -xvzf $(CONCRETE_CORE_FFI_TARBALL) --directory $(CONCRETE_CORE_FFI_FOLDER)
$(CONCRETE_CORE_FFI_TARBALL):
curl -L https://github.com/zama-ai/concrete-core/releases/download/concrete-core-ffi-$(CONCRETE_CORE_FFI_VERSION)/$(CONCRETE_CORE_FFI_TARBALL) -o $(CONCRETE_CORE_FFI_TARBALL)
# concrete-optimizer ######################################
LIB_CONCRETE_OPTIMIZER_CPP = $(CONCRETE_OPTIMIZER_DIR)/target/libconcrete_optimizer_cpp.a
concrete-optimizer-lib:
make -C $(CONCRETE_OPTIMIZER_DIR)/concrete-optimizer-cpp $(LIB_CONCRETE_OPTIMIZER_CPP)
# HPX #####################################################
install-hpx-from-source: $(HPX_LOCAL_DIR)
mkdir -p $(HPX_LOCAL_DIR)/build
cd $(HPX_LOCAL_DIR)/build && cmake \
-DHPX_WITH_FETCH_ASIO=on \
-DHPX_FILESYSTEM_WITH_BOOST_FILESYSTEM_COMPATIBILITY=ON \
-DHPX_WITH_MALLOC=system ..
cd $(HPX_LOCAL_DIR)/build && make -j2
$(HPX_TARBALL):
curl -L $(HPX_URL) -o $(HPX_TARBALL)
$(HPX_LOCAL_DIR): $(HPX_TARBALL)
tar xzvf $(HPX_TARBALL)
$(BUILD_DIR)/configured.stamp:
mkdir -p $(BUILD_DIR)
cmake -B $(BUILD_DIR) -GNinja ../llvm-project/llvm/ \
$(CMAKE_CCACHE_OPTIONS) \
$(CC_COMPILER_OPTION) \
$(CXX_COMPILER_OPTION) \
-DLLVM_ENABLE_PROJECTS="mlir;clang;openmp" \
-DLLVM_BUILD_EXAMPLES=OFF \
-DLLVM_TARGETS_TO_BUILD="host" \
-DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \
-DLLVM_ENABLE_ASSERTIONS=ON \
-DMLIR_ENABLE_BINDINGS_PYTHON=$(BINDINGS_PYTHON_ENABLED) \
-DCONCRETELANG_BINDINGS_PYTHON_ENABLED=$(BINDINGS_PYTHON_ENABLED) \
-DCONCRETELANG_DATAFLOW_EXECUTION_ENABLED=$(DATAFLOW_EXECUTION_ENABLED) \
-DCONCRETELANG_TIMING_ENABLED=$(TIMING_ENABLED) \
-DCONCRETE_FFI_RELEASE=$(CONCRETE_CORE_FFI_FOLDER) \
-DHPX_DIR=${HPX_INSTALL_DIR}/lib/cmake/HPX \
-DLLVM_EXTERNAL_PROJECTS=concretelang \
-DLLVM_EXTERNAL_CONCRETELANG_SOURCE_DIR=. \
-DPython3_EXECUTABLE=${Python3_EXECUTABLE} \
-DCONCRETE_OPTIMIZER_DIR=${CONCRETE_OPTIMIZER_DIR} \
$(CONCRETE_CORE_PATH_OPTION) \
-DCONCRETELANG_CUDA_SUPPORT=${CUDA_SUPPORT}
touch $@
build-initialized: concrete-optimizer-lib concrete-core-ffi $(BUILD_DIR)/configured.stamp
doc: build-initialized
cmake --build $(BUILD_DIR) --target mlir-doc
concretecompiler: build-initialized
cmake --build $(BUILD_DIR) --target concretecompiler
python-bindings: build-initialized
cmake --build $(BUILD_DIR) --target ConcretelangMLIRPythonModules
cmake --build $(BUILD_DIR) --target ConcretelangPythonModules
# MLIRCAPIRegistration is currently required for linking rust bindings
# This may fade if we can represent it somewhere else, mainly, we may be able to define a single
# lib that the rust bindings need to link to, while that lib contains all necessary libs
rust-bindings: build-initialized concretecompiler
cmake --build $(BUILD_DIR) --target MLIRCAPIRegistration
cd lib/Bindings/Rust && CONCRETE_COMPILER_BUILD_DIR=$(abspath $(BUILD_DIR)) cargo build --release
clientlib: build-initialized
cmake --build $(BUILD_DIR) --target ConcretelangClientLib
serverlib: build-initialized
cmake --build $(BUILD_DIR) --target ConcretelangServerLib
GITHUB_URL=https://api.github.com/repos/zama-ai/concrete-compiler-internal
GITHUB_URL_LIST_ARTIFACTS="${GITHUB_URL}/actions/artifacts"
CURL=curl -H"Accept: application/vnd.github.v3+json" -H"authorization: Bearer ${GITHUB_TOKEN}"
keysetcache.zip: REDIRECT_URL = $(shell ${CURL} -s ${GITHUB_URL_LIST_ARTIFACTS} | grep -A 10 ${KEYSETCACHENAME} | grep archive_download_url | head -n 1 | grep -o 'http[^"]\+')
keysetcache.zip:
${CURL} --location -o keysetcache.zip ${REDIRECT_URL}
du -h keysetcache.zip
keysetcache_ci_populated: keysetcache.zip
unzip keysetcache.zip -d ${KEYSETCACHECI}
du -sh ${KEYSETCACHECI}
rm keysetcache.zip
keysetcache_populated: keysetcache.zip
unzip keysetcache.zip -d ${KEYSETCACHEDEV}
du -sh ${KEYSETCACHEDEV}
rm keysetcache.zip
# test
build-tests: build-unit-tests build-end-to-end-tests
run-tests: run-check-tests run-unit-tests run-end-to-end-tests run-python-tests
## check-tests
run-check-tests: concretecompiler file-check not
$(BUILD_DIR)/bin/llvm-lit -v tests/
## unit-tests
build-unit-tests: build-initialized
cmake --build $(BUILD_DIR) --target ConcretelangUnitTests
run-unit-tests: build-unit-tests
find $(BUILD_DIR)/tools/concretelang/tests/unit_tests -name unit_tests_concretelang* -executable -type f | xargs -n1 ./run_test_bin.sh
## python-tests
run-python-tests: python-bindings concretecompiler
PYTHONPATH=${PYTHONPATH}:$(BUILD_DIR)/tools/concretelang/python_packages/concretelang_core LD_PRELOAD=$(BUILD_DIR)/lib/libConcretelangRuntime.so pytest -vs -m $(PYTHON_TESTS_MARKER) tests/python
test-compiler-file-output: concretecompiler
pytest -vs tests/test_compiler_file_output
## rust-tests
run-rust-tests: rust-bindings
cd lib/Bindings/Rust && CONCRETE_COMPILER_BUILD_DIR=$(abspath $(BUILD_DIR)) cargo test --release
## end-to-end-tests
build-end-to-end-tests: build-end-to-end-jit-test build-end-to-end-jit-fhe build-end-to-end-jit-encrypted-tensor build-end-to-end-jit-fhelinalg build-end-to-end-jit-lambda
run-end-to-end-tests: build-end-to-end-tests
find $(BUILD_DIR)/tools/concretelang/tests/end_to_end_tests -name end_to_end* -executable -type f | xargs -n1 ./run_test_bin.sh
build-end-to-end-jit-test: build-initialized
cmake --build $(BUILD_DIR) --target end_to_end_jit_test
build-end-to-end-jit-fhe: build-initialized
cmake --build $(BUILD_DIR) --target end_to_end_jit_fhe
build-end-to-end-jit-encrypted-tensor: build-initialized
cmake --build $(BUILD_DIR) --target end_to_end_jit_encrypted_tensor
build-end-to-end-jit-fhelinalg: build-initialized
cmake --build $(BUILD_DIR) --target end_to_end_jit_fhelinalg
build-end-to-end-jit-lambda: build-initialized
cmake --build $(BUILD_DIR) --target end_to_end_jit_lambda
## end-to-end-dataflow-tests
build-end-to-end-dataflow-tests: build-initialized
cmake --build $(BUILD_DIR) --target end_to_end_jit_auto_parallelization
cmake --build $(BUILD_DIR) --target end_to_end_jit_distributed
cmake --build $(BUILD_DIR) --target end_to_end_jit_aes_short
run-end-to-end-dataflow-tests: build-end-to-end-dataflow-tests
$(BUILD_DIR)/tools/concretelang/tests/end_to_end_tests/end_to_end_jit_auto_parallelization
$(BUILD_DIR)/tools/concretelang/tests/end_to_end_tests/end_to_end_jit_distributed
## GPU tests
build-end-to-end-gpu-tests: build-initialized
cmake --build $(BUILD_DIR) --target end_to_end_gpu_test
run-end-to-end-gpu-tests: build-end-to-end-gpu-tests
$(BUILD_DIR)/tools/concretelang/tests/end_to_end_tests/end_to_end_gpu_test
# benchmark
generate-benchmarks:
$(Python3_EXECUTABLE) ./tests/end_to_end_fixture/end_to_end_linalg_apply_lookup_table_gen.py > tests/end_to_end_fixture/end_to_end_linalg_apply_lookup_table.yaml
build-benchmarks: build-initialized
cmake --build $(BUILD_DIR) --target end_to_end_benchmark
run-benchmarks: build-benchmarks generate-benchmarks
$(BUILD_DIR)/bin/end_to_end_benchmark --benchmark_out=benchmarks_results.json --benchmark_out_format=json
build-mlbench: build-initialized
cmake --build $(BUILD_DIR) --target end_to_end_mlbench
generate-mlbench:
mkdir -p tests/end_to_end_benchmarks/mlbench
rm -rf tests/end_to_end_benchmarks/mlbench/*
unzip tests/end_to_end_benchmarks/mlbench.zip -d tests/end_to_end_benchmarks/mlbench
rm -f tests/end_to_end_benchmarks/mlbench/**/*\=*
find tests/end_to_end_benchmarks/mlbench -name "*.mlir" -exec sed -e '1d' -e 's/ func / func.func /g' -e 's/ linalg.tensor_/ tensor./g' -e '$$d' -i {} \;
$(Python3_EXECUTABLE) tests/end_to_end_benchmarks/generate_bench_yaml.py tests/end_to_end_benchmarks/mlbench tests/end_to_end_benchmarks/mlbench/end_to_end_mlbench
run-mlbench: build-mlbench generate-mlbench
tests/end_to_end_benchmarks/end_to_end_mlbench.sh tests/end_to_end_benchmarks/mlbench/ $(BUILD_DIR)/bin/end_to_end_mlbench
show-stress-tests-summary:
@echo '------ Stress tests summary ------'
@echo
@echo 'Rates:'
@cd tests/stress_tests/trace && grep success_rate -R
@echo
@echo 'Parameters issues:'
@cd tests/stress_tests/trace && grep BAD -R || echo 'No issues'
stress-tests: concretecompiler
pytest -vs tests/stress_tests
# useful for faster cache generation, need pytest-parallel
stress-tests-fast-cache: concretecompiler
pytest --workers auto -vs tests/stress_tests
# LLVM/MLIR dependencies
all-deps: file-check not
file-check: build-initialized
cmake --build $(BUILD_DIR) --target FileCheck
not: build-initialized
cmake --build $(BUILD_DIR) --target not
mlir-cpu-runner: build-initialized
cmake --build $(BUILD_DIR) --target mlir-cpu-runner
opt: build-initialized
cmake --build $(BUILD_DIR) --target opt
mlir-opt: build-initialized
cmake --build $(BUILD_DIR) --target mlir-opt
mlir-translate: build-initialized
cmake --build $(BUILD_DIR) --target mlir-translate
release-tarballs:
docker image build -t concrete-compiler-manylinux:linux_x86_64_tarball -f ../builders/Dockerfile.release_tarball_linux_x86_64 ..
docker container run --rm -v ${PWD}/../tarballs:/tarballs_volume concrete-compiler-manylinux:linux_x86_64_tarball cp -r /tarballs/. /tarballs_volume/.
update-python-version:
echo "__version__ = \"`git describe --tags --abbrev=0 | grep -e '[0-9].*' -o`\"" > lib/Bindings/Python/version.txt
check-python-format:
black --check tests/python/ lib/Bindings/Python/concrete/
python-format:
black tests/python/ lib/Bindings/Python/concrete/
python-lint:
pylint --rcfile=../pylintrc lib/Bindings/Python/concrete/compiler
.PHONY: build-initialized \
build-end-to-end-jit \
concretecompiler \
python-bindings \
add-deps \
file-check \
not \
release-tarballs \
update-python-version \
python-lint \
python-format \
check-python-format \
concrete-optimizer-lib \
concrete-core-ffi \
build-tests \
run-tests \
run-check-tests \
build-unit-tests \
run-unit-tests \
run-python-tests \
build-end-to-end-tests \
build-end-to-end-dataflow-tests \
run-end-to-end-dataflow-tests \
concrete-core-ffi \
opt \
mlir-opt \
mlir-cpu-runner \
mlir-translate
Reference in New Issue View Git Blame Copy Permalink