diff --git a/.github/workflows/cargo_build.yml b/.github/workflows/cargo_build.yml index a239031a1..73a77c3bb 100644 --- a/.github/workflows/cargo_build.yml +++ b/.github/workflows/cargo_build.yml @@ -18,17 +18,95 @@ permissions: contents: read jobs: - cargo-builds: - name: cargo_build/cargo-builds (bpr) - runs-on: ${{ matrix.os }} + prepare-parallel-pcc-matrix: + name: cargo_build/prepare-parallel-pcc-matrix + runs-on: ubuntu-latest + outputs: + matrix_command: ${{ steps.set-pcc-commands-matrix.outputs.commands }} + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 + with: + persist-credentials: "false" + token: ${{ env.CHECKOUT_TOKEN }} + # Fetch all the Make recipes that start with `pcc_batch_` + - name: Set pcc commands matrix + id: set-pcc-commands-matrix + run: | + COMMANDS=$(grep -oE '^pcc_batch_[^:]*:' Makefile | sed 's/:/\"/; s/^/\"/' | paste -sd,) + echo "commands=[${COMMANDS}]" >> "$GITHUB_OUTPUT" + + parallel-pcc-cpu: + name: cargo_build/parallel-pcc-cpu + needs: prepare-parallel-pcc-matrix + runs-on: large_ubuntu_16 + strategy: + matrix: + command: ${{fromJson(needs.prepare-parallel-pcc-matrix.outputs.matrix_command)}} + fail-fast: false + steps: + - name: Checkout tfhe-rs repo + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 + with: + persist-credentials: 'false' + token: ${{ env.CHECKOUT_TOKEN }} + + - name: Install latest stable + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # zizmor: ignore[stale-action-refs] this action doesn't create releases + with: + toolchain: stable + + - name: Run pcc checks batch + run: | + make "${COMMAND}" + env: + COMMAND: ${{ matrix.command }} + + pcc-hpu: + name: cargo_build/pcc-hpu + runs-on: large_ubuntu_16 + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 + with: + persist-credentials: 'false' + token: ${{ env.CHECKOUT_TOKEN }} + + - name: Install latest stable + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # zizmor: ignore[stale-action-refs] this action doesn't create releases + with: + toolchain: stable + + - name: Run Hpu pcc checks + run: | + make pcc_hpu + + build-tfhe-full: + name: cargo_build/build-tfhe-full + runs-on: ${{ matrix.os }} strategy: matrix: # GitHub macos-latest are now M1 macs, so use ours, we limit what runs so it will be fast # even with a few PRs os: [large_ubuntu_16, macos-latest-xlarge, large_windows_16_latest] fail-fast: false + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 + with: + persist-credentials: 'false' + token: ${{ env.CHECKOUT_TOKEN }} + - name: Install latest stable + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # zizmor: ignore[stale-action-refs] this action doesn't create releases + with: + toolchain: stable + + - name: Build Release tfhe full + run: | + make build_tfhe_full + + build: + name: cargo_build/build + runs-on: large_ubuntu_16 steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 with: @@ -41,7 +119,6 @@ jobs: toolchain: stable - name: Install and run newline linter checks - if: ${{ contains(matrix.os, 'ubuntu') }} run: | wget https://github.com/fernandrone/linelint/releases/download/0.0.6/linelint-linux-amd64 echo "16b70fb7b471d6f95cbdc0b4e5dc2b0ac9e84ba9ecdc488f7bdf13df823aca4b linelint-linux-amd64" > checksum @@ -50,60 +127,93 @@ jobs: mv linelint-linux-amd64 /usr/local/bin/linelint make check_newline - - name: Run pcc checks - if: ${{ contains(matrix.os, 'ubuntu') }} - run: | - make pcc - - name: Build tfhe-csprng - if: ${{ contains(matrix.os, 'ubuntu') }} run: | make build_tfhe_csprng - name: Build with MSRV - if: ${{ contains(matrix.os, 'ubuntu') }} run: | make build_tfhe_msrv + - name: Build coverage tests + run: | + make build_tfhe_coverage + + build-layers: + name: cargo_build/build-layers + runs-on: large_ubuntu_16 + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 + with: + persist-credentials: 'false' + token: ${{ env.CHECKOUT_TOKEN }} + + - name: Install latest stable + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # zizmor: ignore[stale-action-refs] this action doesn't create releases + with: + toolchain: stable + - name: Build Release core - if: ${{ contains(matrix.os, 'ubuntu') }} run: | make build_core AVX512_SUPPORT=ON make build_core_experimental AVX512_SUPPORT=ON - name: Build Release boolean - if: ${{ contains(matrix.os, 'ubuntu') }} run: | make build_boolean - name: Build Release shortint - if: ${{ contains(matrix.os, 'ubuntu') }} run: | make build_shortint - name: Build Release integer - if: ${{ contains(matrix.os, 'ubuntu') }} run: | make build_integer - - name: Build Release tfhe full - run: | - make build_tfhe_full + build-c-api: + name: cargo_build/build-c-api + runs-on: large_ubuntu_16 + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 + with: + persist-credentials: 'false' + token: ${{ env.CHECKOUT_TOKEN }} + + - name: Install latest stable + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # zizmor: ignore[stale-action-refs] this action doesn't create releases + with: + toolchain: stable - name: Build Release c_api - if: ${{ contains(matrix.os, 'ubuntu') }} run: | make build_c_api - - name: Build coverage tests - if: ${{ contains(matrix.os, 'ubuntu') }} - run: | - make build_tfhe_coverage - - - name: Run Hpu pcc checks - if: ${{ contains(matrix.os, 'ubuntu') }} - run: | - make pcc_hpu - # The wasm build check is a bit annoying to set-up here and is done during the tests in # aws_tfhe_tests.yml + + cargo-builds: + name: cargo_build/cargo-builds (bpr) + needs: [ parallel-pcc-cpu, pcc-hpu, build-tfhe-full, build, build-layers, build-c-api ] + if: ${{ always() }} + runs-on: ubuntu-latest + steps: + - name: Check all builds success + if: needs.parallel-pcc-cpu.result == 'success' && + needs.pcc-hpu.result == 'success' && + needs.build-tfhe-full.result == 'success' && + needs.build.result == 'success' && + needs.build-layers.result == 'success' && + needs.build-c-api.result == 'success' + run: | + echo "All tfhe-rs build checks passed" + + - name: Check builds failure + if: needs.parallel-pcc-cpu.result != 'success' || + needs.pcc-hpu.result != 'success' || + needs.build-tfhe-full.result != 'success' || + needs.build.result != 'success' || + needs.build-layers.result != 'success' || + needs.build-c-api.result != 'success' + run: | + echo "Some tfhe-rs build checks failed" + exit 1 diff --git a/Makefile b/Makefile index 132f58b1f..908ca58de 100644 --- a/Makefile +++ b/Makefile @@ -1189,6 +1189,8 @@ check_compile_tests: install_rs_build_toolchain --features=experimental,boolean,shortint,integer,internal-keycache \ -p tfhe +.PHONY: check_compile_tests_c_api # Build C API tests without running them +check_compile_tests_c_api: install_rs_build_toolchain @if [[ "$(OS)" == "Linux" || "$(OS)" == "Darwin" ]]; then \ "$(MAKE)" build_c_api && \ ./scripts/c_api_tests.sh --build-only --cargo-profile "$(CARGO_PROFILE)"; \ @@ -1662,11 +1664,38 @@ sha256_bool: install_rs_check_toolchain RUSTFLAGS="$(RUSTFLAGS)" cargo $(CARGO_RS_CHECK_TOOLCHAIN) run --profile $(CARGO_PROFILE) \ --example sha256_bool --features=boolean -.PHONY: pcc # pcc stands for pre commit checks (except GPU) -pcc: no_tfhe_typo no_dbg_log check_parameter_export_ok check_fmt check_typos lint_doc \ -check_md_docs_are_tested check_intra_md_links check_doc_paths_use_dash \ -clippy_all check_compile_tests test_tfhe_lints \ -tfhe_lints +.PHONY: pcc # pcc stands for pre commit checks for CPU compilation +pcc: pcc_batch_1 pcc_batch_2 pcc_batch_3 pcc_batch_4 pcc_batch_5 pcc_batch_6 pcc_batch_7 + +# +# PCC split into several batches to speed-up CI feedback. +# Each batch have roughly the same execution time. +# Durations are given from GitHub Ubuntu large runner with 16 CPU. +# + +.PHONY: pcc_batch_1 # duration: 6'10'' +pcc_batch_1: no_tfhe_typo no_dbg_log check_parameter_export_ok check_fmt check_typos lint_doc \ +check_md_docs_are_tested check_intra_md_links check_doc_paths_use_dash test_tfhe_lints tfhe_lints \ +clippy_rustdoc + +.PHONY: pcc_batch_2 # duration: 6'10'' +pcc_batch_2: clippy clippy_all_targets + +.PHONY: pcc_batch_3 # duration: 6'50'' +pcc_batch_3: clippy_shortint clippy_integer + +.PHONY: pcc_batch_4 # duration: 7'40'' +pcc_batch_4: clippy_core clippy_js_wasm_api clippy_ws_tests clippy_bench + +.PHONY: pcc_batch_5 # duration: 7'20'' +pcc_batch_5: clippy_tfhe_lints check_compile_tests clippy_backward_compat_data + +.PHONY: pcc_batch_6 # duration: 4'50'' (shortest one, extend it with further checks) +pcc_batch_6: clippy_boolean clippy_c_api clippy_tasks clippy_tfhe_csprng clippy_zk_pok \ +clippy_trivium clippy_versionable clippy_param_dedup + +.PHONY: pcc_batch_7 # duration: 7'50'' (currently PCC execution bottleneck) +pcc_batch_7: check_compile_tests_c_api .PHONY: pcc_gpu # pcc stands for pre commit checks for GPU compilation pcc_gpu: check_rust_bindings_did_not_change clippy_rustdoc_gpu \