chore(ci): split cargo builds into several jobs

Post-commit checks was the bottleneck regarding running duration.
It's now split into 7 batches to improve parallelism.
Builds that are specific to Ubuntu are run in their own jobs, so
that only build_tfhe_full recipe call remains in the os matrix.
A final check is performed to ensure all the checks have passed,
this very job is used as branch protection rule.

.github/workflows/cargo_build.yml

@@ -18,17 +18,95 @@ permissions:
   contents: read
 
 jobs:
-  cargo-builds:
-    name: cargo_build/cargo-builds (bpr)
-    runs-on: ${{ matrix.os }}
+  prepare-parallel-pcc-matrix:
+    name: cargo_build/prepare-parallel-pcc-matrix
+    runs-on: ubuntu-latest
+    outputs:
+      matrix_command: ${{ steps.set-pcc-commands-matrix.outputs.commands }}
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
+        with:
+          persist-credentials: "false"
+          token: ${{ env.CHECKOUT_TOKEN }}
 
+      # Fetch all the Make recipes that start with `pcc_batch_`
+      - name: Set pcc commands matrix
+        id: set-pcc-commands-matrix
+        run: |
+          COMMANDS=$(grep -oE '^pcc_batch_[^:]*:' Makefile | sed 's/:/\"/; s/^/\"/' | paste -sd,)
+          echo "commands=[${COMMANDS}]" >> "$GITHUB_OUTPUT"
+
+  parallel-pcc-cpu:
+    name: cargo_build/parallel-pcc-cpu
+    needs: prepare-parallel-pcc-matrix
+    runs-on: large_ubuntu_16
+    strategy:
+      matrix:
+        command: ${{fromJson(needs.prepare-parallel-pcc-matrix.outputs.matrix_command)}}
+      fail-fast: false
+    steps:
+      - name: Checkout tfhe-rs repo
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
+        with:
+          persist-credentials: 'false'
+          token: ${{ env.CHECKOUT_TOKEN }}
+
+      - name: Install latest stable
+        uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # zizmor: ignore[stale-action-refs] this action doesn't create releases
+        with:
+          toolchain: stable
+
+      - name: Run pcc checks batch
+        run: |
+          make "${COMMAND}"
+        env:
+          COMMAND: ${{ matrix.command }}
+
+  pcc-hpu:
+    name: cargo_build/pcc-hpu
+    runs-on: large_ubuntu_16
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
+        with:
+          persist-credentials: 'false'
+          token: ${{ env.CHECKOUT_TOKEN }}
+
+      - name: Install latest stable
+        uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # zizmor: ignore[stale-action-refs] this action doesn't create releases
+        with:
+          toolchain: stable
+
+      - name: Run Hpu pcc checks
+        run: |
+          make pcc_hpu
+
+  build-tfhe-full:
+    name: cargo_build/build-tfhe-full
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         # GitHub macos-latest are now M1 macs, so use ours, we limit what runs so it will be fast
         # even with a few PRs
         os: [large_ubuntu_16, macos-latest-xlarge, large_windows_16_latest]
       fail-fast: false
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
+        with:
+          persist-credentials: 'false'
+          token: ${{ env.CHECKOUT_TOKEN }}
 
+      - name: Install latest stable
+        uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # zizmor: ignore[stale-action-refs] this action doesn't create releases
+        with:
+          toolchain: stable
+
+      - name: Build Release tfhe full
+        run: |
+          make build_tfhe_full
+
+  build:
+    name: cargo_build/build
+    runs-on: large_ubuntu_16
     steps:
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
         with:
@@ -41,7 +119,6 @@ jobs:
           toolchain: stable
 
       - name: Install and run newline linter checks
-        if: ${{ contains(matrix.os, 'ubuntu') }}
         run: |
           wget https://github.com/fernandrone/linelint/releases/download/0.0.6/linelint-linux-amd64
           echo "16b70fb7b471d6f95cbdc0b4e5dc2b0ac9e84ba9ecdc488f7bdf13df823aca4b linelint-linux-amd64" > checksum
@@ -50,60 +127,93 @@ jobs:
           mv linelint-linux-amd64 /usr/local/bin/linelint
           make check_newline
 
-      - name: Run pcc checks
-        if: ${{ contains(matrix.os, 'ubuntu') }}
-        run: |
-          make pcc
-
       - name: Build tfhe-csprng
-        if: ${{ contains(matrix.os, 'ubuntu') }}
         run: |
           make build_tfhe_csprng
 
       - name: Build with MSRV
-        if: ${{ contains(matrix.os, 'ubuntu') }}
         run: |
           make build_tfhe_msrv
 
+      - name: Build coverage tests
+        run: |
+          make build_tfhe_coverage
+
+  build-layers:
+    name: cargo_build/build-layers
+    runs-on: large_ubuntu_16
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
+        with:
+          persist-credentials: 'false'
+          token: ${{ env.CHECKOUT_TOKEN }}
+
+      - name: Install latest stable
+        uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # zizmor: ignore[stale-action-refs] this action doesn't create releases
+        with:
+          toolchain: stable
+
       - name: Build Release core
-        if: ${{ contains(matrix.os, 'ubuntu') }}
         run: |
           make build_core AVX512_SUPPORT=ON
           make build_core_experimental AVX512_SUPPORT=ON
 
       - name: Build Release boolean
-        if: ${{ contains(matrix.os, 'ubuntu') }}
         run: |
           make build_boolean
 
       - name: Build Release shortint
-        if: ${{ contains(matrix.os, 'ubuntu') }}
         run: |
           make build_shortint
 
       - name: Build Release integer
-        if: ${{ contains(matrix.os, 'ubuntu') }}
         run: |
           make build_integer
 
-      - name: Build Release tfhe full
-        run: |
-          make build_tfhe_full
+  build-c-api:
+    name: cargo_build/build-c-api
+    runs-on: large_ubuntu_16
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
+        with:
+          persist-credentials: 'false'
+          token: ${{ env.CHECKOUT_TOKEN }}
+
+      - name: Install latest stable
+        uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # zizmor: ignore[stale-action-refs] this action doesn't create releases
+        with:
+          toolchain: stable
 
       - name: Build Release c_api
-        if: ${{ contains(matrix.os, 'ubuntu') }}
         run: |
           make build_c_api
 
-      - name: Build coverage tests
-        if: ${{ contains(matrix.os, 'ubuntu') }}
-        run: |
-          make build_tfhe_coverage
-
-      - name: Run Hpu pcc checks
-        if: ${{ contains(matrix.os, 'ubuntu') }}
-        run: |
-          make pcc_hpu
-
       # The wasm build check is a bit annoying to set-up here and is done during the tests in
       # aws_tfhe_tests.yml
+
+  cargo-builds:
+    name: cargo_build/cargo-builds (bpr)
+    needs: [ parallel-pcc-cpu, pcc-hpu, build-tfhe-full, build, build-layers, build-c-api ]
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check all builds success
+        if: needs.parallel-pcc-cpu.result == 'success' &&
+          needs.pcc-hpu.result == 'success' &&
+          needs.build-tfhe-full.result == 'success' &&
+          needs.build.result == 'success' &&
+          needs.build-layers.result == 'success' &&
+          needs.build-c-api.result == 'success'
+        run: |
+          echo "All tfhe-rs build checks passed"
+
+      - name: Check builds failure
+        if: needs.parallel-pcc-cpu.result != 'success' ||
+          needs.pcc-hpu.result != 'success' ||
+          needs.build-tfhe-full.result != 'success' ||
+          needs.build.result != 'success' ||
+          needs.build-layers.result != 'success' ||
+          needs.build-c-api.result != 'success'
+        run: |
+          echo "Some tfhe-rs build checks failed"
+          exit 1

Makefile

@@ -1189,6 +1189,8 @@ check_compile_tests: install_rs_build_toolchain
 		--features=experimental,boolean,shortint,integer,internal-keycache \
 		-p tfhe
 
+.PHONY: check_compile_tests_c_api # Build C API tests without running them
+check_compile_tests_c_api: install_rs_build_toolchain
 	@if [[ "$(OS)" == "Linux" || "$(OS)" == "Darwin" ]]; then \
 		"$(MAKE)" build_c_api && \
 		./scripts/c_api_tests.sh --build-only --cargo-profile "$(CARGO_PROFILE)"; \
@@ -1662,11 +1664,38 @@ sha256_bool: install_rs_check_toolchain
 	RUSTFLAGS="$(RUSTFLAGS)" cargo $(CARGO_RS_CHECK_TOOLCHAIN) run --profile $(CARGO_PROFILE) \
 	--example sha256_bool --features=boolean
 
-.PHONY: pcc # pcc stands for pre commit checks (except GPU)
-pcc: no_tfhe_typo no_dbg_log check_parameter_export_ok check_fmt check_typos lint_doc \
-check_md_docs_are_tested check_intra_md_links check_doc_paths_use_dash \
-clippy_all check_compile_tests test_tfhe_lints \
-tfhe_lints
+.PHONY: pcc # pcc stands for pre commit checks for CPU compilation
+pcc: pcc_batch_1 pcc_batch_2 pcc_batch_3 pcc_batch_4 pcc_batch_5 pcc_batch_6 pcc_batch_7
+
+#
+# PCC split into several batches to speed-up CI feedback.
+# Each batch have roughly the same execution time.
+# Durations are given from GitHub Ubuntu large runner with 16 CPU.
+#
+
+.PHONY: pcc_batch_1 # duration: 6'10''
+pcc_batch_1: no_tfhe_typo no_dbg_log check_parameter_export_ok check_fmt check_typos lint_doc \
+check_md_docs_are_tested check_intra_md_links check_doc_paths_use_dash test_tfhe_lints tfhe_lints \
+clippy_rustdoc
+
+.PHONY: pcc_batch_2 # duration: 6'10''
+pcc_batch_2: clippy clippy_all_targets
+
+.PHONY: pcc_batch_3 # duration: 6'50''
+pcc_batch_3: clippy_shortint clippy_integer
+
+.PHONY: pcc_batch_4 # duration: 7'40''
+pcc_batch_4: clippy_core clippy_js_wasm_api clippy_ws_tests clippy_bench
+
+.PHONY: pcc_batch_5 # duration: 7'20''
+pcc_batch_5: clippy_tfhe_lints check_compile_tests clippy_backward_compat_data
+
+.PHONY: pcc_batch_6  # duration: 4'50'' (shortest one, extend it with further checks)
+pcc_batch_6: clippy_boolean clippy_c_api clippy_tasks clippy_tfhe_csprng clippy_zk_pok \
+clippy_trivium clippy_versionable clippy_param_dedup
+
+.PHONY: pcc_batch_7 # duration: 7'50'' (currently PCC execution bottleneck)
+pcc_batch_7: check_compile_tests_c_api
 
 .PHONY: pcc_gpu # pcc stands for pre commit checks for GPU compilation
 pcc_gpu: check_rust_bindings_did_not_change clippy_rustdoc_gpu \