github/concrete
1
1
Fork 0
mirror of https://github.com/zama-ai/concrete.git synced 2026-02-08 19:44:57 -05:00
Code Issues Packages Projects Releases Wiki Activity

chore(benchmarks): Refactor the benchmark tools

Browse Source
This commit is contained in:
Quentin Bourgerie
2022-12-07 09:58:52 +01:00
parent fa1b2dc056
commit 07aa334d8d
7 changed files with 293 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
.github/workflows/benchmark.yml vendored
View File

@@ -112,17 +112,20 @@ jobs:
run: |
echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}"
- name: Export CUDA variables
- name: Export specific variables (CPU)
if: ${{ inputs.backend == 'cpu' }}
run: |
echo "CUDA_SUPPORT=OFF" >> "${GITHUB_ENV}"
echo "BENCHMARK_TARGET=run-cpu-benchmarks" >> "${GITHUB_ENV}"
- name: Export specific variables (GPU)
if: ${{ inputs.backend == 'gpu' }}
run: |
echo "CUDA_SUPPORT=ON" >> "${GITHUB_ENV}"
echo "BENCHMARK_TARGET=run-gpu-benchmarks" >> "${GITHUB_ENV}"
echo "CUDA_PATH=$CUDA_PATH" >> "${GITHUB_ENV}"
echo "$CUDA_PATH/bin" >> "${GITHUB_PATH}"
echo "LD_LIBRARY_PATH=$CUDA_PATH/lib:$LD_LIBRARY_PATH" >> "${GITHUB_ENV}"
# Specify the correct host compilers
- name: Export gcc and g++ variables
if: ${{ inputs.backend == 'gpu' }}
run: |
echo "CC=/usr/bin/gcc-${{ env.GCC_VERSION }}" >> "${GITHUB_ENV}"
echo "CXX=/usr/bin/g++-${{ env.GCC_VERSION }}" >> "${GITHUB_ENV}"
echo "CUDAHOSTCXX=/usr/bin/g++-${{ env.GCC_VERSION }}" >> "${GITHUB_ENV}"
@@ -134,19 +137,11 @@ jobs:
toolchain: stable
override: true
- name: Build compiler and end-to-end benchmarks (CPU)
if: ${{ inputs.backend == 'cpu' }}
- name: Build compiler benchmarks
run: |
set -e
cd compiler
make BINDINGS_PYTHON_ENABLED=OFF build-benchmarks generate-cpu-benchmarks
- name: Build compiler and end-to-end benchmarks (GPU)
if: ${{ inputs.backend == 'gpu' }}
run: |
set -e
cd compiler
make BINDINGS_PYTHON_ENABLED=OFF CUDA_SUPPORT=ON build-benchmarks generate-gpu-benchmarks
make BINDINGS_PYTHON_ENABLED=OFF CUDA_SUPPORT=${{ env.CUDA_SUPPORT }} build-benchmarks
- name: Download KeySetCache
if: ${{ !contains(github.head_ref, 'newkeysetcache') }}
@@ -155,15 +150,11 @@ jobs:
cd compiler
GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} make keysetcache_ci_populated
- name: Mark KeySetCache
run: |
touch keysetcache.timestamp
- name: Run end-to-end benchmarks
run: |
set -e
cd compiler
make BINDINGS_PYTHON_ENABLED=OFF run-benchmarks
make ${{ env.BENCHMARK_TARGET }}
- name: Upload raw results artifact
uses: actions/upload-artifact@v3

51
compiler/Makefile
View File

@@ -265,7 +265,8 @@ run-end-to-end-tests: build-end-to-end-tests generate-cpu-tests
$(BUILD_DIR)/tools/concretelang/tests/end_to_end_tests/end_to_end_jit_encrypted_tensor
$(BUILD_DIR)/tools/concretelang/tests/end_to_end_tests/end_to_end_jit_fhelinalg
$(BUILD_DIR)/tools/concretelang/tests/end_to_end_tests/end_to_end_jit_lambda
$(BUILD_DIR)/tools/concretelang/tests/end_to_end_tests/end_to_end_test --loop-parallelize --jit $(FIXTURE_CPU_DIR)/*.yaml
$(BUILD_DIR)/tools/concretelang/tests/end_to_end_tests/end_to_end_test \
--backend=cpu --jit $(FIXTURE_CPU_DIR)/*.yaml
### end-to-end-tests GPU
@@ -284,10 +285,9 @@ generate-gpu-tests: $(FIXTURE_GPU_DIR) $(FIXTURE_GPU_DIR)/end_to_end_apply_looku
run-end-to-end-tests-gpu: build-end-to-end-test generate-gpu-tests
$(BUILD_DIR)/tools/concretelang/tests/end_to_end_tests/end_to_end_test \
--emit-gpu-ops --batch-concrete-ops --library /tmp/concrete_compiler/gpu_tests/ \
--backend=gpu --library /tmp/concrete_compiler/gpu_tests/ \
$(FIXTURE_GPU_DIR)/*.yaml
## end-to-end-dataflow-tests
build-end-to-end-dataflow-tests: build-initialized
@@ -301,30 +301,51 @@ run-end-to-end-dataflow-tests: build-end-to-end-dataflow-tests
# benchmark
build-benchmarks: build-initialized
cmake --build $(BUILD_DIR) --target end_to_end_benchmark
## benchmark CPU
BENCHMARK_CPU_DIR=tests/end_to_end_fixture/benchmarks_cpu
$(BENCHMARK_CPU_DIR):
mkdir -p $@
$(BENCHMARK_CPU_DIR)/end_to_end_linalg_apply_lookup_table.yaml: tests/end_to_end_fixture/end_to_end_linalg_apply_lookup_table_gen.py
$(Python3_EXECUTABLE) $< --n-ct 64 128 1024 > $@
$(BENCHMARK_CPU_DIR)/%.yaml: tests/end_to_end_fixture/%_gen.py
mkdir -p $(FIXTURE_CPU_DIR)
$(Python3_EXECUTABLE) $< > $@
$(BENCHMARK_CPU_DIR):
mkdir -p $@
generate-cpu-benchmarks: $(BENCHMARK_CPU_DIR) $(BENCHMARK_CPU_DIR)/end_to_end_linalg_apply_lookup_table.yaml $(BENCHMARK_CPU_DIR)/end_to_end_apply_lookup_table.yaml $(BENCHMARK_CPU_DIR)/end_to_end_leveled.yaml
generate-gpu-benchmarks:
$(Python3_EXECUTABLE) ./tests/end_to_end_fixture/end_to_end_linalg_apply_lookup_table_gen.py \
--min_bitwidth 1 --max_bitwidth 7 --n_ct 1 128 1024 2048 8192 \
> tests/end_to_end_fixture/end_to_end_linalg_apply_lookup_table.yaml
run-cpu-benchmarks: build-benchmarks generate-cpu-benchmarks
$(BUILD_DIR)/bin/end_to_end_benchmark \
--backend=cpu \
--benchmark_out=benchmarks_results.json --benchmark_out_format=json \
$(BENCHMARK_CPU_DIR)/*.yaml
## benchmark GPU
BENCHMARK_GPU_DIR=tests/end_to_end_fixture/benchmarks_gpu
$(BENCHMARK_GPU_DIR):
mkdir -p $@
$(BENCHMARK_GPU_DIR)/end_to_end_linalg_apply_lookup_table.yaml: tests/end_to_end_fixture/end_to_end_linalg_apply_lookup_table_gen.py
$(Python3_EXECUTABLE) $< \
--bitwidth 1 2 3 4 5 6 7 --n-ct 1 128 1024 2048 8192
generate-gpu-benchmarks: $(BENCHMARK_GPU_DIR) $(BENCHMARK_GPU_DIR)/end_to_end_linalg_apply_lookup_table.yaml
run-gpu-benchmarks: build-benchmarks generate-cpu-benchmarks
$(BUILD_DIR)/bin/end_to_end_benchmark \
--backend=gpu \
--benchmark_out=benchmarks_results.json --benchmark_out_format=json \
$(BENCHMARK_CPU_DIR)/*.yaml
build-benchmarks: build-initialized
cmake --build $(BUILD_DIR) --target end_to_end_benchmark
run-benchmarks: build-benchmarks generate-cpu-benchmarks
$(BUILD_DIR)/bin/end_to_end_benchmark --benchmark_out=benchmarks_results.json --benchmark_out_format=json
build-mlbench: build-initialized
cmake --build $(BUILD_DIR) --target end_to_end_mlbench

22
compiler/include/concretelang/Support/CompilerEngine.h
View File

@@ -38,6 +38,11 @@ protected:
llvm::LLVMContext *llvmContext;
};
enum Backend {
CPU,
GPU,
};
/// Compilation options allows to configure the compilation pipeline.
struct CompilationOptions {
llvm::Optional<mlir::concretelang::V0FHEConstraint> v0FHEConstraints;
@@ -75,6 +80,23 @@ struct CompilationOptions {
CompilationOptions(std::string funcname) : CompilationOptions() {
clientParametersFuncName = funcname;
}
/// @brief Constructor for CompilationOptions with default parameters for a
/// specific backend.
/// @param funcname The name of the function to compile.
/// @param backend The backend to target.
CompilationOptions(std::string funcname, enum Backend backend)
: CompilationOptions(funcname) {
switch (backend) {
case Backend::CPU:
loopParallelize = true;
break;
case Backend::GPU:
batchConcreteOps = true;
emitGPUOps = true;
break;
}
}
};
class CompilerEngine {

186
compiler/tests/end_to_end_benchmarks/end_to_end_benchmark.cpp
View File

@@ -1,7 +1,9 @@
#include "end_to_end_fixture/EndToEndFixture.h"
#include "../end_to_end_tests/end_to_end_test.h"
#include <benchmark/benchmark.h>
#define BENCHMARK_HAS_CXX11
#include "llvm/Support/Path.h"
#include <benchmark/benchmark.h>
#include "tests_tools/StackSize.h"
#include "tests_tools/keySetCache.h"
@@ -100,82 +102,112 @@ static void BM_Evaluate(benchmark::State &state, EndToEndDesc description,
}
}
static int registerEndToEndTestFromFile(std::string prefix, std::string path,
size_t stackSizeRequirement = 0,
bool only_evaluate = false) {
auto registe = [&](std::string optionsName,
mlir::concretelang::CompilationOptions options) {
llvm::for_each(loadEndToEndDesc(path), [&](EndToEndDesc &description) {
options.clientParametersFuncName = "main";
mlir::concretelang::JITSupport support;
auto benchName = [&](std::string name) {
std::ostringstream s;
s << prefix << "/" << name << "/" << optionsName << "/"
<< description.description;
return s.str();
};
benchmark::RegisterBenchmark(
benchName("Evaluate").c_str(), [=](::benchmark::State &st) {
BM_Evaluate(st, description, support, options);
});
if (!only_evaluate) {
benchmark::RegisterBenchmark(
benchName("Compile").c_str(), [=](::benchmark::State &st) {
BM_Compile(st, description, support, options);
});
benchmark::RegisterBenchmark(
benchName("KeyGen").c_str(), [=](::benchmark::State &st) {
BM_KeyGen(st, description, support, options);
});
benchmark::RegisterBenchmark(
benchName("ExportArguments").c_str(), [=](::benchmark::State &st) {
BM_ExportArguments(st, description, support, options);
});
}
return;
});
};
setCurrentStackLimit(stackSizeRequirement);
#ifndef CONCRETELANG_CUDA_SUPPORT
// Run only parallelized benchmarks to take advantage of hardware with lots of
// CPU cores.
mlir::concretelang::CompilationOptions cpu;
registe("cpu", cpu);
cpu.loopParallelize = true;
#else
mlir::concretelang::CompilationOptions gpu;
gpu.batchConcreteOps = true;
gpu.emitGPUOps = true;
gpu.loopParallelize = true;
registe("gpu", gpu);
#endif
return 1;
std::string getOptionsName(mlir::concretelang::CompilationOptions options) {
std::ostringstream os;
if (options.loopParallelize)
os << "_loop";
if (options.dataflowParallelize)
os << "_dataflow";
if (options.emitGPUOps)
os << "_gpu";
auto ostr = os.str();
if (ostr.size() == 0) {
os << "_default";
}
return os.str().substr(1);
}
auto stackSizeRequirement = 0;
auto _ = {
registerEndToEndTestFromFile("FHELinalgLeveled",
"tests/end_to_end_fixture/benchmarks_cpu/"
"end_to_end_leveled.yaml",
stackSizeRequirement,
/* only_evaluate = */ false),
// For lookup table bench we only bench the keygen time on simple lookup
// table bench, to avoid
// bench the same keygen several times as it take times
registerEndToEndTestFromFile("FHELinalg",
"tests/end_to_end_fixture/benchmarks_cpu/"
"end_to_end_apply_lookup_table.yaml",
stackSizeRequirement,
/* only_evaluate = */ false),
// So for the other lookup table benchmarks we only test the evaluataion
// times
registerEndToEndTestFromFile("FHELinalgTLU",
"tests/end_to_end_fixture/benchmarks_cpu/"
"end_to_end_linalg_apply_lookup_table.yaml",
stackSizeRequirement,
/* only_evaluate = */ true),
enum Action {
COMPILE,
KEYGEN,
ENCRYPT,
EVALUATE,
};
BENCHMARK_MAIN();
void registerEndToEndBenchmark(std::string suiteName,
std::vector<EndToEndDesc> descriptions,
mlir::concretelang::CompilationOptions options,
std::vector<enum Action> actions,
size_t stackSizeRequirement = 0) {
auto optionsName = getOptionsName(options);
for (auto description : descriptions) {
options.clientParametersFuncName = "main";
mlir::concretelang::JITSupport support;
auto benchName = [&](std::string name) {
std::ostringstream s;
s << suiteName << "/" << name << "/" << optionsName << "/"
<< description.description;
return s.str();
};
for (auto action : actions) {
switch (action) {
case Action::COMPILE:
benchmark::RegisterBenchmark(
benchName("compile").c_str(), [=](::benchmark::State &st) {
BM_Compile(st, description, support, options);
});
break;
case Action::KEYGEN:
benchmark::RegisterBenchmark(
benchName("keygen").c_str(), [=](::benchmark::State &st) {
BM_KeyGen(st, description, support, options);
});
break;
case Action::ENCRYPT:
benchmark::RegisterBenchmark(
benchName("encrypt").c_str(), [=](::benchmark::State &st) {
BM_ExportArguments(st, description, support, options);
});
break;
case Action::EVALUATE:
benchmark::RegisterBenchmark(
benchName("evaluate").c_str(), [=](::benchmark::State &st) {
BM_Evaluate(st, description, support, options);
});
break;
}
}
}
setCurrentStackLimit(stackSizeRequirement);
}
int main(int argc, char **argv) {
// Parse google benchmark options
::benchmark::Initialize(&argc, argv);
llvm::cl::list<enum Action> clActions(
"b", "bench",
llvm::cl::desc("Specify benchmark cases to run, if no benchmarks speci"),
llvm::cl::values(
clEnumValN(Action::COMPILE, "compile", "Run compile benchmark")),
llvm::cl::values(
clEnumValN(Action::KEYGEN, "keygen", "Run keygen benchmark")),
llvm::cl::values(
clEnumValN(Action::ENCRYPT, "encrypt", "Run encrypt benchmark")),
llvm::cl::values(
clEnumValN(Action::EVALUATE, "evaluate", "Run evaluate benchmark")));
// parse end to end test compiler options
auto options = parseEndToEndCommandLine(argc, argv);
auto compilationOptions = std::get<0>(options);
auto libpath = std::get<1>(options);
auto descriptionFiles = std::get<2>(options);
std::vector<enum Action> actions = clActions;
if (actions.empty()) {
actions = {Action::COMPILE, Action::KEYGEN, Action::ENCRYPT,
Action::EVALUATE};
}
auto stackSizeRequirement = 0;
for (auto descFile : descriptionFiles) {
auto suiteName = llvm::sys::path::stem(descFile.path).str();
registerEndToEndBenchmark(suiteName, descFile.descriptions,
compilationOptions, actions,
stackSizeRequirement);
}
::benchmark::RunSpecifiedBenchmarks();
::benchmark::Shutdown();
return 0;
}

5
compiler/tests/end_to_end_fixture/EndToEndFixture.h
View File

@@ -56,6 +56,11 @@ struct EndToEndDesc {
std::vector<TestErrorRate> test_error_rates;
};
struct EndToEndDescFile {
std::string path;
std::vector<EndToEndDesc> descriptions;
};
llvm::Error checkResult(ValueDescription &desc,
mlir::concretelang::LambdaArgument &res);

71
compiler/tests/end_to_end_tests/end_to_end_test.cc
View File

@@ -226,78 +226,23 @@ int main(int argc, char **argv) {
// Parse google test options, update argc and argv by removing gtest options
::testing::InitGoogleTest(&argc, argv);
// Main command line options
llvm::cl::ResetCommandLineParser();
// parse end to end test compiler options
llvm::cl::list<std::string> descriptionFiles(
llvm::cl::Positional, llvm::cl::desc("<End to end description Files>"),
llvm::cl::OneOrMore);
auto options = parseEndToEndCommandLine(argc, argv);
// Compilation options
llvm::cl::opt<bool> loopParallelize(
"loop-parallelize",
llvm::cl::desc(
"Set the loopParallelize compilation options to run the tests"),
llvm::cl::init(false));
llvm::cl::opt<bool> dataflowParallelize(
"dataflow-parallelize",
llvm::cl::desc(
"Set the loopParallelize compilation options to run the tests"),
llvm::cl::init(false));
llvm::cl::opt<bool> emitGPUOps(
"emit-gpu-ops",
llvm::cl::desc("Set the emitGPUOps compilation options to run the tests"),
llvm::cl::init(false));
llvm::cl::opt<bool> batchConcreteOps(
"batch-concrete-ops",
llvm::cl::desc(
"Set the batchConcreteOps compilation options to run the tests"),
llvm::cl::init(false));
// Optimizer options
llvm::cl::opt<bool> optimizerDisplay(
"optimizer-display",
llvm::cl::desc("Set the optimizerConfig.display compilation options to "
"run the tests"),
llvm::cl::init(false));
// JIT or Library support
llvm::cl::opt<bool> jit(
"jit",
llvm::cl::desc("Use JIT support to run the tests (default, overwritten "
"if --library is set"),
llvm::cl::init(true));
llvm::cl::opt<std::string> library(
"library",
llvm::cl::desc("Use library support to run the tests and specify the "
"prefix for compilation artifacts"),
llvm::cl::init<std::string>(""));
llvm::cl::ParseCommandLineOptions(argc, argv);
// Build compilation options
mlir::concretelang::CompilationOptions compilationOptions("main");
compilationOptions.loopParallelize = loopParallelize.getValue();
compilationOptions.dataflowParallelize = dataflowParallelize.getValue();
compilationOptions.emitGPUOps = emitGPUOps.getValue();
compilationOptions.batchConcreteOps =
compilationOptions.optimizerConfig.display = optimizerDisplay.getValue();
auto compilationOptions = std::get<0>(options);
auto libpath = std::get<1>(options);
auto descriptionFiles = std::get<2>(options);
for (auto descFile : descriptionFiles) {
auto desc = loadEndToEndDesc(descFile);
auto suiteName = path::stem(descFile).str();
auto libpath = library.getValue();
if (libpath.empty() && !jit.getValue()) {
llvm::errs()
<< "You must specify the library path or use jit to run the test";
return 1;
}
auto suiteName = path::stem(descFile.path).str();
if (libpath.empty()) {
suiteName = suiteName + ".jit";
} else {
suiteName = suiteName + ".library";
}
registerEndToEndSuite(suiteName, libpath, desc, compilationOptions);
registerEndToEndSuite(suiteName, libpath, descFile.descriptions,
compilationOptions);
}
return RUN_ALL_TESTS();
}

102
compiler/tests/end_to_end_tests/end_to_end_test.h
View File

@@ -1,7 +1,10 @@
#ifndef END_TO_END_TEST_H
#define END_TO_END_TEST_H
#include <gtest/gtest.h>
#include "concretelang/Support/CompilerEngine.h"
#include "llvm/Support/CommandLine.h"
#include "end_to_end_fixture/EndToEndFixture.h"
// Shorthands to create integer literals of a specific type
static inline uint8_t operator"" _u8(unsigned long long int v) { return v; }
@@ -13,4 +16,101 @@ static inline uint64_t operator"" _u64(unsigned long long int v) { return v; }
// array
#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof(arr[0]))
/// @brief Parse the command line and return a tuple contains the compilation
/// options, the library path if the --library options has been specified and
/// the parsed description files
std::tuple<mlir::concretelang::CompilationOptions, std::string,
std::vector<EndToEndDescFile>>
parseEndToEndCommandLine(int argc, char **argv) {
// TODO - Well reset other llvm command line options registered but assert on
// --help
// llvm::cl::ResetCommandLineParser();
llvm::cl::list<std::string> descriptionFiles(
llvm::cl::Positional, llvm::cl::desc("<End to end description Files>"),
llvm::cl::OneOrMore);
// Compilation options
llvm::cl::opt<mlir::concretelang::Backend> backend(
"backend",
llvm::cl::desc("Specify benchmark cases to run, if no benchmarks speci"),
llvm::cl::values(clEnumValN(mlir::concretelang::Backend::CPU, "cpu",
"Target a CPU backend")),
llvm::cl::values(clEnumValN(mlir::concretelang::Backend::GPU, "gpu",
"Target a GPU backend")),
llvm::cl::init(mlir::concretelang::Backend::CPU));
llvm::cl::opt<llvm::Optional<bool>> loopParallelize(
"loop-parallelize",
llvm::cl::desc(
"Set the loopParallelize compilation options to run the tests"),
llvm::cl::init(llvm::None));
llvm::cl::opt<llvm::Optional<bool>> dataflowParallelize(
"dataflow-parallelize",
llvm::cl::desc(
"Set the loopParallelize compilation options to run the tests"),
llvm::cl::init(llvm::None));
llvm::cl::opt<llvm::Optional<bool>> emitGPUOps(
"emit-gpu-ops",
llvm::cl::desc("Set the emitGPUOps compilation options to run the tests"),
llvm::cl::init(llvm::None));
llvm::cl::opt<llvm::Optional<bool>> batchConcreteOps(
"batch-concrete-ops",
llvm::cl::desc(
"Set the batchConcreteOps compilation options to run the tests"),
llvm::cl::init(llvm::None));
// Optimizer options
llvm::cl::opt<bool> optimizerDisplay(
"optimizer-display",
llvm::cl::desc("Set the optimizerConfig.display compilation options to "
"run the tests"),
llvm::cl::init(false));
// JIT or Library support
llvm::cl::opt<bool> jit(
"jit",
llvm::cl::desc("Use JIT support to run the tests (default, overwritten "
"if --library is set"),
llvm::cl::init(true));
llvm::cl::opt<std::string> library(
"library",
llvm::cl::desc("Use library support to run the tests and specify the "
"prefix for compilation artifacts"),
llvm::cl::init<std::string>(""));
llvm::cl::ParseCommandLineOptions(argc, argv);
// Build compilation options
mlir::concretelang::CompilationOptions compilationOptions("main",
backend.getValue());
if (loopParallelize.hasValue())
compilationOptions.loopParallelize = loopParallelize.getValue().getValue();
if (dataflowParallelize.hasValue())
compilationOptions.dataflowParallelize =
dataflowParallelize.getValue().getValue();
if (emitGPUOps.hasValue())
compilationOptions.emitGPUOps = emitGPUOps.getValue().getValue();
if (batchConcreteOps.hasValue())
compilationOptions.batchConcreteOps =
batchConcreteOps.getValue().getValue();
compilationOptions.optimizerConfig.display = optimizerDisplay.getValue();
std::vector<EndToEndDescFile> parsedDescriptionFiles;
for (auto descFile : descriptionFiles) {
EndToEndDescFile f;
f.path = descFile;
f.descriptions = loadEndToEndDesc(descFile);
parsedDescriptionFiles.push_back(f);
}
auto libpath = library.getValue();
if (libpath.empty() && !jit.getValue()) {
llvm::errs()
<< "You must specify the library path or use jit to run the test";
exit(1);
}
return std::make_tuple(compilationOptions, libpath, parsedDescriptionFiles);
}
#endif