diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index ba3527ff5..a41e59518 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -359,7 +359,7 @@ jobs: pip install pytest rm -rf /build export PYTHONPATH="" - make PARALLEL_EXECUTION_ENABLED=ON CCACHE=ON BUILD_DIR=/build run-tests run-end-to-end-dataflow-tests + make DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON BUILD_DIR=/build run-tests run-end-to-end-dataflow-tests echo "Debug: ccache statistics (after the build):" ccache -s chmod -R ugo+rwx /tmp/KeySetCache diff --git a/builders/Dockerfile.concrete-compiler-df-env b/builders/Dockerfile.concrete-compiler-df-env index ef4740c3a..81e6ada05 100644 --- a/builders/Dockerfile.concrete-compiler-df-env +++ b/builders/Dockerfile.concrete-compiler-df-env @@ -2,9 +2,9 @@ FROM ubuntu:latest RUN apt-get update RUN DEBIAN_FRONTEND="noninteractive" apt-get install -y curl cmake g++ \ - build-essential python3 python3-pip \ - python3-setuptools ninja-build git \ - zlib1g-dev ccache libboost-filesystem-dev libhwloc-dev + build-essential python3 python3-pip \ + python3-setuptools ninja-build git \ + zlib1g-dev ccache libboost-filesystem-dev libhwloc-dev # setup ccache with an unlimited amount of files and storage RUN ccache -M 0 RUN ccache -F 0 @@ -24,8 +24,8 @@ COPY /llvm-project /llvm-project COPY /compiler /compiler WORKDIR /compiler RUN mkdir -p /build -RUN make PARALLEL_EXECUTION_ENABLED=ON BUILD_DIR=/build CCACHE=ON \ - concretecompiler python-bindings && \ +RUN make DATAFLOW_EXECUTION_ENABLED=ON BUILD_DIR=/build CCACHE=ON \ + concretecompiler python-bindings && \ mv /build/tools/concretelang/python_packages/concretelang_core /concretelang_core && \ mv /build/bin/concretecompiler /bin && \ mv /build/lib/libConcretelangRuntime.so /lib && \ diff --git a/compiler/CMakeLists.txt b/compiler/CMakeLists.txt index efe9dfb6c..935467e6f 100644 --- a/compiler/CMakeLists.txt +++ b/compiler/CMakeLists.txt @@ -8,12 +8,12 @@ set(CMAKE_CXX_STANDARD 14) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # Wouldn't be able to compile LLVM without this on Mac (using either Clang or AppleClang) -if (APPLE) +if(APPLE) add_definitions("-Wno-narrowing -Wno-dollar-in-identifier-extension") endif() # If we are trying to build the compiler with LLVM/MLIR as libraries -if( NOT DEFINED LLVM_EXTERNAL_CONCRETELANG_SOURCE_DIR ) +if(NOT DEFINED LLVM_EXTERNAL_CONCRETELANG_SOURCE_DIR) message(FATAL_ERROR "Concrete compiler requires a unified build with LLVM/MLIR") endif() @@ -24,13 +24,13 @@ set(CMAKE_PLATFORM_NO_VERSIONED_SONAME ON CACHE BOOL set(CMAKE_VISIBILITY_INLINES_HIDDEN ON CACHE BOOL "Hide inlines") # The -fvisibility=hidden option only works for static builds. -if (BUILD_SHARED_LIBS AND (CMAKE_CXX_VISIBILITY_PRESET STREQUAL "hidden")) +if(BUILD_SHARED_LIBS AND(CMAKE_CXX_VISIBILITY_PRESET STREQUAL "hidden")) message(FATAL_ERROR "CMAKE_CXX_VISIBILITY_PRESET=hidden is incompatible \ with BUILD_SHARED_LIBS.") endif() -set(MLIR_MAIN_SRC_DIR ${LLVM_MAIN_SRC_DIR}/../mlir ) # --src-root -set(MLIR_INCLUDE_DIR ${MLIR_MAIN_SRC_DIR}/include ) # --includedir +set(MLIR_MAIN_SRC_DIR ${LLVM_MAIN_SRC_DIR}/../mlir) # --src-root +set(MLIR_INCLUDE_DIR ${MLIR_MAIN_SRC_DIR}/include) # --includedir set(MLIR_TABLEGEN_OUTPUT_DIR ${LLVM_BINARY_DIR}/tools/mlir/include) set(MLIR_TABLEGEN_EXE $) include_directories(SYSTEM ${MLIR_INCLUDE_DIR}) @@ -48,17 +48,16 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules") include(AddConcretelangDoc) set(CONCRETELANG_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}) -#------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------- # Concrete FFI Configuration -#------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------- include_directories(${CONCRETE_FFI_RELEASE}) add_library(Concrete STATIC IMPORTED) -set_target_properties(Concrete PROPERTIES IMPORTED_LOCATION ${CONCRETE_FFI_RELEASE}/libconcrete_ffi.a ) +set_target_properties(Concrete PROPERTIES IMPORTED_LOCATION ${CONCRETE_FFI_RELEASE}/libconcrete_ffi.a) -#------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------- # Python Configuration -#------------------------------------------------------------------------------- - +# ------------------------------------------------------------------------------- option(CONCRETELANG_BINDINGS_PYTHON_ENABLED "Enables ConcreteLang Python bindings." ON) if(CONCRETELANG_BINDINGS_PYTHON_ENABLED) @@ -71,26 +70,25 @@ else() message(STATUS "ConcreteLang Python bindings are disabled.") endif() -#------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------- # DFR - parallel execution configuration -#------------------------------------------------------------------------------- - -option(CONCRETELANG_PARALLEL_EXECUTION_ENABLED "Enables parallel execution for ConcreteLang." ON) +# ------------------------------------------------------------------------------- +option(CONCRETELANG_DATAFLOW_EXECUTION_ENABLED "Enables dataflow execution for ConcreteLang." ON) option(CONCRETELANG_TIMING_ENABLED "Enables execution timing." ON) -if(CONCRETELANG_PARALLEL_EXECUTION_ENABLED) - message(STATUS "ConcreteLang parallel execution enabled.") +if(CONCRETELANG_DATAFLOW_EXECUTION_ENABLED) + message(STATUS "ConcreteLang dataflow execution enabled.") find_package(HPX REQUIRED CONFIG) include_directories(SYSTEM ${HPX_INCLUDE_DIRS}) list(APPEND CMAKE_MODULE_PATH "${HPX_CMAKE_DIR}") add_compile_options( - -DCONCRETELANG_PARALLEL_EXECUTION_ENABLED + -DCONCRETELANG_DATAFLOW_EXECUTION_ENABLED -DHPX_DEFAULT_CONFIG_FILE="${PROJECT_SOURCE_DIR}/hpx.ini" ) else() - message(STATUS "ConcreteLang parallel execution disabled.") + message(STATUS "ConcreteLang dataflow execution disabled.") endif() if(CONCRETELANG_TIMING_ENABLED) @@ -101,21 +99,19 @@ else() message(STATUS "ConcreteLang execution timing disabled.") endif() -#------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------- # Unit tests -#------------------------------------------------------------------------------- - +# ------------------------------------------------------------------------------- option(CONCRETELANG_UNIT_TESTS "Enables the build of unittests" ON) -#------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------- # Benchmarks -#------------------------------------------------------------------------------- - +# ------------------------------------------------------------------------------- option(CONCRETELANG_BENCHMARK "Enables the build of benchmarks" ON) -#------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------- # Handling sub dirs -#------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------- include_directories(${CONCRETE_OPTIMIZER_DIR}/concrete-optimizer-cpp/src/cpp) add_subdirectory(include) diff --git a/compiler/Makefile b/compiler/Makefile index 9d7d11f4a..82b8a7b84 100644 --- a/compiler/Makefile +++ b/compiler/Makefile @@ -1,7 +1,7 @@ BUILD_DIR=./build Python3_EXECUTABLE?= BINDINGS_PYTHON_ENABLED=ON -PARALLEL_EXECUTION_ENABLED=OFF +DATAFLOW_EXECUTION_ENABLED=OFF TIMING_ENABLED=OFF CC_COMPILER= CXX_COMPILER= @@ -45,7 +45,7 @@ else endif # don't run parallel python tests if compiler doesn't support it -ifeq ($(PARALLEL_EXECUTION_ENABLED),ON) +ifeq ($(DATAFLOW_EXECUTION_ENABLED),ON) PYTHON_TESTS_MARKER="" else PYTHON_TESTS_MARKER="not parallel" @@ -66,7 +66,7 @@ $(BUILD_DIR)/configured.stamp: -DLLVM_ENABLE_ASSERTIONS=ON \ -DMLIR_ENABLE_BINDINGS_PYTHON=$(BINDINGS_PYTHON_ENABLED) \ -DCONCRETELANG_BINDINGS_PYTHON_ENABLED=$(BINDINGS_PYTHON_ENABLED) \ - -DCONCRETELANG_PARALLEL_EXECUTION_ENABLED=$(PARALLEL_EXECUTION_ENABLED) \ + -DCONCRETELANG_DATAFLOW_EXECUTION_ENABLED=$(DATAFLOW_EXECUTION_ENABLED) \ -DCONCRETELANG_TIMING_ENABLED=$(TIMING_ENABLED) \ -DCONCRETE_FFI_RELEASE=${CONCRETE_PROJECT}/target/release \ -DHPX_DIR=${HPX_INSTALL_DIR}/lib/cmake/HPX \ diff --git a/compiler/include/concretelang/Runtime/time_util.h b/compiler/include/concretelang/Runtime/time_util.h index ddb84172a..394082fe9 100644 --- a/compiler/include/concretelang/Runtime/time_util.h +++ b/compiler/include/concretelang/Runtime/time_util.h @@ -24,7 +24,7 @@ static inline int timespec_diff(struct timespec *, const struct timespec *, assert(clock_gettime(TIME_UTIL_CLOCK, (p)) == 0); \ } while (0) -#if CONCRETELANG_PARALLEL_EXECUTION_ENABLED +#if CONCRETELANG_DATAFLOW_EXECUTION_ENABLED #define END_TIME(p, m) \ do { \ struct timespec _end_time_tv; \ diff --git a/compiler/lib/CAPI/Support/CompilerEngine.cpp b/compiler/lib/CAPI/Support/CompilerEngine.cpp index b40f66bdb..b00d2c6df 100644 --- a/compiler/lib/CAPI/Support/CompilerEngine.cpp +++ b/compiler/lib/CAPI/Support/CompilerEngine.cpp @@ -234,7 +234,7 @@ clientParametersSerialize(mlir::concretelang::ClientParameters ¶ms) { } void terminateParallelization() { -#ifdef CONCRETELANG_PARALLEL_EXECUTION_ENABLED +#ifdef CONCRETELANG_DATAFLOW_EXECUTION_ENABLED _dfr_terminate(); #endif } diff --git a/compiler/lib/Runtime/CMakeLists.txt b/compiler/lib/Runtime/CMakeLists.txt index b15790ebf..0aaf8ba82 100644 --- a/compiler/lib/Runtime/CMakeLists.txt +++ b/compiler/lib/Runtime/CMakeLists.txt @@ -4,7 +4,7 @@ add_library(ConcretelangRuntime SHARED DFRuntime.cpp ) -if(CONCRETELANG_PARALLEL_EXECUTION_ENABLED) +if(CONCRETELANG_DATAFLOW_EXECUTION_ENABLED) target_link_libraries( ConcretelangRuntime PRIVATE diff --git a/compiler/lib/Runtime/DFRuntime.cpp b/compiler/lib/Runtime/DFRuntime.cpp index f055f2d54..321ac561b 100644 --- a/compiler/lib/Runtime/DFRuntime.cpp +++ b/compiler/lib/Runtime/DFRuntime.cpp @@ -9,7 +9,7 @@ /// This hides the details of implementation, including of the HPX /// framework currently used, from the code generation side. -#ifdef CONCRETELANG_PARALLEL_EXECUTION_ENABLED +#ifdef CONCRETELANG_DATAFLOW_EXECUTION_ENABLED #include #include @@ -1287,7 +1287,7 @@ void _dfr_print_debug(size_t val) { hpx::cout << "_dfr_print_debug : " << val << "\n" << std::flush; } -#else // CONCRETELANG_PARALLEL_EXECUTION_ENABLED +#else // CONCRETELANG_DATAFLOW_EXECUTION_ENABLED #include "concretelang/Runtime/DFRuntime.hpp" #include "concretelang/Runtime/time_util.h" diff --git a/compiler/lib/Support/Jit.cpp b/compiler/lib/Support/Jit.cpp index 81ab0d7b6..7ae82089d 100644 --- a/compiler/lib/Support/Jit.cpp +++ b/compiler/lib/Support/Jit.cpp @@ -84,7 +84,7 @@ uint64_t numArgOfRankedMemrefCallingConvention(uint64_t rank) { llvm::Expected> JITLambda::call(clientlib::PublicArguments &args, clientlib::EvaluationKeys &evaluationKeys) { -#ifndef CONCRETELANG_PARALLEL_EXECUTION_ENABLED +#ifndef CONCRETELANG_DATAFLOW_EXECUTION_ENABLED if (this->useDataflow) { return StreamStringError( "call: current runtime doesn't support dataflow execution, while " diff --git a/compiler/tests/end_to_end_tests/CMakeLists.txt b/compiler/tests/end_to_end_tests/CMakeLists.txt index 3213621cb..3589c1c73 100644 --- a/compiler/tests/end_to_end_tests/CMakeLists.txt +++ b/compiler/tests/end_to_end_tests/CMakeLists.txt @@ -17,9 +17,9 @@ if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin") ) endif() -if(CONCRETELANG_PARALLEL_EXECUTION_ENABLED) +if(CONCRETELANG_DATAFLOW_EXECUTION_ENABLED) add_compile_options( - -DCONCRETELANG_PARALLEL_TESTING_ENABLED + -DCONCRETELANG_DATAFLOW_TESTING_ENABLED ) endif() @@ -59,7 +59,7 @@ add_concretecompiler_unittest( globals.cc ) -if(CONCRETELANG_PARALLEL_EXECUTION_ENABLED) +if(CONCRETELANG_DATAFLOW_EXECUTION_ENABLED) add_concretecompiler_unittest( end_to_end_jit_auto_parallelization end_to_end_jit_auto_parallelization.cc diff --git a/compiler/tests/end_to_end_tests/end_to_end_jit_fhe.cc b/compiler/tests/end_to_end_tests/end_to_end_jit_fhe.cc index 9fb3495ea..028b8e9c7 100644 --- a/compiler/tests/end_to_end_tests/end_to_end_jit_fhe.cc +++ b/compiler/tests/end_to_end_tests/end_to_end_jit_fhe.cc @@ -25,7 +25,7 @@ void compile_and_run(EndToEndDesc desc, LambdaSupport support) { } /* 0 - Enable parallel testing where required */ -#ifdef CONCRETELANG_PARALLEL_TESTING_ENABLED +#ifdef CONCRETELANG_DATAFLOW_TESTING_ENABLED options.dataflowParallelize = true; options.loopParallelize = true; #endif diff --git a/compiler/tests/end_to_end_tests/end_to_end_jit_test.h b/compiler/tests/end_to_end_tests/end_to_end_jit_test.h index 00d84b295..1e41dd3c2 100644 --- a/compiler/tests/end_to_end_tests/end_to_end_jit_test.h +++ b/compiler/tests/end_to_end_tests/end_to_end_jit_test.h @@ -30,8 +30,8 @@ internalCheckedJit(llvm::StringRef src, llvm::StringRef func = "main", // Allow loop parallelism in all cases options.loopParallelize = loopParallelize; -#ifdef CONCRETELANG_PARALLEL_EXECUTION_ENABLED -#ifdef CONCRETELANG_PARALLEL_TESTING_ENABLED +#ifdef CONCRETELANG_DATAFLOW_EXECUTION_ENABLED +#ifdef CONCRETELANG_DATAFLOW_TESTING_ENABLED options.dataflowParallelize = true; options.loopParallelize = true; #else diff --git a/compiler/tests/unit_tests/concretelang/TestLib/CMakeLists.txt b/compiler/tests/unit_tests/concretelang/TestLib/CMakeLists.txt index ce8ba2665..3f9dbcb28 100644 --- a/compiler/tests/unit_tests/concretelang/TestLib/CMakeLists.txt +++ b/compiler/tests/unit_tests/concretelang/TestLib/CMakeLists.txt @@ -16,9 +16,9 @@ if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin") ) endif() -if(CONCRETELANG_PARALLEL_EXECUTION_ENABLED) +if(CONCRETELANG_DATAFLOW_EXECUTION_ENABLED) add_compile_options( - -DCONCRETELANG_PARALLEL_TESTING_ENABLED + -DCONCRETELANG_DATAFLOW_TESTING_ENABLED ) endif() diff --git a/compiler/tests/unit_tests/concretelang/TestLib/testlib_unit_test.cpp b/compiler/tests/unit_tests/concretelang/TestLib/testlib_unit_test.cpp index 98f836436..d27157e36 100644 --- a/compiler/tests/unit_tests/concretelang/TestLib/testlib_unit_test.cpp +++ b/compiler/tests/unit_tests/concretelang/TestLib/testlib_unit_test.cpp @@ -44,8 +44,8 @@ compile(std::string outputLib, std::string source, mlir::concretelang::CompilationContext::createShared(); mlir::concretelang::CompilerEngine ce{ccx}; mlir::concretelang::CompilationOptions options(funcname); -#ifdef CONCRETELANG_PARALLEL_TESTING_ENABLED - options.autoParallelize = true; +#ifdef CONCRETELANG_DATAFLOW_TESTING_ENABLED + options.dataflowParallelize = true; #endif ce.setCompilationOptions(options); auto result = ce.compile(sources, outputLib);