diff --git a/.github/workflows/compiler_build_and_test_cpu.yml b/.github/workflows/compiler_build_and_test_cpu.yml index c4aee2713..a68ae4ca5 100644 --- a/.github/workflows/compiler_build_and_test_cpu.yml +++ b/.github/workflows/compiler_build_and_test_cpu.yml @@ -83,11 +83,20 @@ jobs: shell: bash run: | set -e + + cd /concrete/frontends/concrete-python + make venv + source .venv/bin/activate + cd /concrete/compilers/concrete-compiler/compiler rm -rf /build/* - make DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build all python-package build-end-to-end-dataflow-tests + make DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$(which python3) BUILD_DIR=/build all python-package build-end-to-end-dataflow-tests echo "Debug: ccache statistics (after the build):" ccache -s + + cd /concrete/frontends/concrete-python + export COMPILER_BUILD_DIRECTORY=/build + make whl - name: Test compiler uses: addnab/docker-run-action@v3 @@ -103,11 +112,18 @@ jobs: shell: bash run: | set -e + + cd /concrete/frontends/concrete-python + make venv + source .venv/bin/activate + export COMPILER_BUILD_DIRECTORY=/build + export CONCRETE_PYTHON_KEY_CACHE_DIRECTORY=/tmp/KeySetCache + make pytest + cd /concrete/compilers/concrete-compiler/compiler - pip install pytest sed "s/pytest/python -m pytest/g" -i Makefile mkdir -p /tmp/concrete_compiler/gpu_tests/ - make DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build run-tests run-end-to-end-dataflow-tests + make DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$(which python3) BUILD_DIR=/build run-tests run-end-to-end-dataflow-tests chmod -R ugo+rwx /tmp/KeySetCache - name: Archive python package @@ -117,6 +133,13 @@ jobs: path: build/wheels/concrete_compiler-*-manylinux_{{ env.GLIB_VER }}_x86_64.whl retention-days: 14 + - name: Archive Concrete Python Wheel + uses: actions/upload-artifact@v3 + with: + name: concrete-python + path: /concrete/frontends/concrete-python/dist/ + retention-days: 14 + - name: Build the documentation id: build-doc if: ${{ steps.build-compiler.outcome == 'success' && !cancelled() }} diff --git a/.github/workflows/concrete_python_tests.yml b/.github/workflows/concrete_python_tests.yml new file mode 100644 index 000000000..78e8bde77 --- /dev/null +++ b/.github/workflows/concrete_python_tests.yml @@ -0,0 +1,122 @@ +name: Concrete Python Tests + +on: + workflow_dispatch: + inputs: + instance_id: + description: 'Instance ID' + type: string + instance_image_id: + description: 'Instance AMI ID' + type: string + instance_type: + description: 'Instance product type' + type: string + runner_name: + description: 'Action runner name' + type: string + request_id: + description: 'Slab request ID' + type: string + +env: + DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler + GLIB_VER: 2_28 + +jobs: + BuildConcreteCompilerAndTestConcretePython: + name: Build Concrete Compiler and Test Concrete Python in EC2 + runs-on: ${{ github.event.inputs.runner_name }} + if: ${{ !cancelled() }} + steps: + - name: Instance configuration used + run: | + echo "IDs: ${{ inputs.instance_id }}" + echo "AMI: ${{ inputs.instance_image_id }}" + echo "Type: ${{ inputs.instance_type }}" + echo "Request ID: ${{ inputs.request_id }}" + + - name: Set up SSH agent + uses: webfactory/ssh-agent@v0.7.0 + with: + ssh-private-key: ${{ secrets.CONCRETE_CI_SSH_PRIVATE }} + + - name: Set up env + run: | + echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" + #echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" + echo "SSH_AUTH_SOCK_DIR=$(dirname $SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" + + - name: Fetch repository + uses: actions/checkout@v3 + with: + submodules: recursive + token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} + + - name: Create build directory + run: mkdir build + + - name: Build Concrete Compiler + uses: addnab/docker-run-action@v3 + id: build-compiler + with: + registry: ghcr.io + image: ${{ env.DOCKER_IMAGE_TEST }} + username: ${{ secrets.GHCR_LOGIN }} + password: ${{ secrets.GHCR_PASSWORD }} + options: >- + -v ${{ github.workspace }}:/concrete + -v ${{ github.workspace }}/build:/build + -v ${{ github.workspace }}/wheels:/wheels + -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket + -e SSH_AUTH_SOCK=/ssh.socket + ${{ env.DOCKER_GPU_OPTION }} + shell: bash + run: | + set -e + + cd /concrete/frontends/concrete-python + make venv + source .venv/bin/activate + + cd /concrete/compilers/concrete-compiler/compiler + rm -rf /build/* + make DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$(which python3) BUILD_DIR=/build python-bindings + echo "Debug: ccache statistics (after the build):" + ccache -s + + cd /concrete/frontends/concrete-python + export COMPILER_BUILD_DIRECTORY=/build + make whl + + - name: Test Concrete Python + uses: addnab/docker-run-action@v3 + with: + registry: ghcr.io + image: ${{ env.DOCKER_IMAGE_TEST }} + username: ${{ secrets.GHCR_LOGIN }} + password: ${{ secrets.GHCR_PASSWORD }} + options: >- + -v ${{ github.workspace }}:/concrete + -v ${{ github.workspace }}/build:/build + ${{ env.DOCKER_GPU_OPTION }} + shell: bash + run: | + set -e + cd /concrete/frontends/concrete-python + + make venv + source .venv/bin/activate + + export COMPILER_BUILD_DIRECTORY=/build + export CONCRETE_PYTHON_KEY_CACHE_DIRECTORY=/tmp/KeySetCache + make pytest + + chmod -R ugo+rwx /tmp/KeySetCache + + - name: Archive Concrete Python Wheel + uses: actions/upload-artifact@v3 + with: + name: concrete-python + path: /concrete/frontends/concrete-python/dist/ + retention-days: 14 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 923a013b5..062c197c8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -147,3 +147,11 @@ jobs: if: needs.file-change.outputs.concrete-python == 'true' || needs.file-change.outputs.push-main uses: ./.github/workflows/concrete_python_checks.yml secrets: inherit + + concrete-python-tests: + needs: file-change + if: needs.file-change.outputs.concrete-python == 'true' && needs.file-change.outputs.compiler == 'false' && !needs.file-change.outputs.push-main + uses: ./.github/workflows/start_slab.yml + secrets: inherit + with: + command: concrete-python-tests diff --git a/ci/slab.toml b/ci/slab.toml index a88b9eb5e..1df5985c6 100644 --- a/ci/slab.toml +++ b/ci/slab.toml @@ -70,3 +70,12 @@ workflow = "ml_benchmark_subset.yml" profile = "m6i-cpu-bench" matrix = [0,1,2,3,4,5,6,7,8,9,10] max_parallel_jobs = 2 + +################################################# +# Concrete Python Commands +################################################# + +[command.concrete-python-tests] +workflow = "concrete_python_tests.yml" +profile = "m6i-cpu-test" +check_run_name = "Concrete Python Tests" diff --git a/frontends/concrete-python/Makefile b/frontends/concrete-python/Makefile index 8b30a57d6..79c9046e5 100644 --- a/frontends/concrete-python/Makefile +++ b/frontends/concrete-python/Makefile @@ -29,7 +29,8 @@ pytest: pytest tests -svv -n auto \ --cov=concrete \ --cov-fail-under=100 \ - --cov-report=term-missing:skip-covered + --cov-report=term-missing:skip-covered \ + --key-cache "${CONCRETE_PYTHON_KEY_CACHE_DIRECTORY}" # ========== # Formatting diff --git a/frontends/concrete-python/tests/conftest.py b/frontends/concrete-python/tests/conftest.py index 29748099d..01b93f24f 100644 --- a/frontends/concrete-python/tests/conftest.py +++ b/frontends/concrete-python/tests/conftest.py @@ -50,7 +50,7 @@ def pytest_sessionstart(session): # pylint: enable=global-statement key_cache_location = session.config.getoption("--key-cache", default=None) - if key_cache_location is not None: + if key_cache_location is not None and key_cache_location != "": if key_cache_location.lower() == "disable": key_cache_location = None else: