chore(tfhe): bump version to 0.2.4

Bumps [JS-DevTools/npm-publish](https://github.com/JS-DevTools/npm-publish) from 2.0.0 to 2.1.0.
- [Release notes](https://github.com/JS-DevTools/npm-publish/releases)
- [Changelog](https://github.com/JS-DevTools/npm-publish/blob/main/CHANGELOG.md)
- [Commits](0be441d808...541aa6b21b)

---
updated-dependencies:
- dependency-name: JS-DevTools/npm-publish
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

.github/workflows/make_release.yml

@@ -0,0 +1,52 @@
+# Publish new release of tfhe-rs on various platform.
+name: Publish release
+
+on:
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: "Dry-run"
+        type: boolean
+        default: true
+
+jobs:
+  publish_release:
+    name: Publish Release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab
+        with:
+          fetch-depth: 0
+
+      - name: Publish crate.io package
+        env:
+          CRATES_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+          DRY_RUN: ${{ inputs.dry_run && '--dry-run' || '' }}
+        run: |
+          cargo publish -p tfhe --token ${{ env.CRATES_TOKEN }} ${{ env.DRY_RUN }}
+
+      - name: Build web package
+        run: |
+          make build_web_js_api
+
+      - name: Publish web package
+        uses: JS-DevTools/npm-publish@541aa6b21b4a1e9990c95a92c21adc16b35e9551
+        with:
+          token: ${{ secrets.NPM_TOKEN }}
+          package: tfhe/pkg/package.json
+          dry-run: ${{ inputs.dry_run }}
+
+      - name: Build Node package
+        run: |
+          rm -rf tfhe/pkg
+
+          make build_node_js_api
+          sed -i 's/"tfhe"/"node-tfhe"/g' tfhe/pkg/package.json
+
+      - name: Publish Node package
+        uses: JS-DevTools/npm-publish@541aa6b21b4a1e9990c95a92c21adc16b35e9551
+        with:
+          token: ${{ secrets.NPM_TOKEN }}
+          package: tfhe/pkg/package.json
+          dry-run: ${{ inputs.dry_run }}

Makefile

@@ -52,6 +52,12 @@ install_cargo_nextest: install_rs_build_toolchain
 	cargo $(CARGO_RS_BUILD_TOOLCHAIN) install cargo-nextest --locked || \
 	( echo "Unable to install cargo nextest, unknown error." && exit 1 )
 
+.PHONY: install_wasm_pack # Install wasm-pack to build JS packages
+install_wasm_pack: install_rs_build_toolchain
+	@wasm-pack --version > /dev/null 2>&1 || \
+	cargo $(CARGO_RS_BUILD_TOOLCHAIN) install wasm-pack || \
+	( echo "Unable to install cargo wasm-pack, unknown error." && exit 1 )
+
 .PHONY: fmt # Format rust code
 fmt: install_rs_check_toolchain
 	cargo "$(CARGO_RS_CHECK_TOOLCHAIN)" fmt
@@ -174,14 +180,14 @@ build_c_api: install_rs_check_toolchain
 		-p tfhe
 
 .PHONY: build_web_js_api # Build the js API targeting the web browser
-build_web_js_api: install_rs_build_toolchain
+build_web_js_api: install_rs_build_toolchain install_wasm_pack
 	cd tfhe && \
 	RUSTFLAGS="$(WASM_RUSTFLAGS)" rustup run "$(RS_BUILD_TOOLCHAIN)" \
 		wasm-pack build --release --target=web \
 		-- --features=boolean-client-js-wasm-api,shortint-client-js-wasm-api
 
 .PHONY: build_node_js_api # Build the js API targeting nodejs
-build_node_js_api: install_rs_build_toolchain
+build_node_js_api: install_rs_build_toolchain install_wasm_pack
 	cd tfhe && \
 	RUSTFLAGS="$(WASM_RUSTFLAGS)" rustup run "$(RS_BUILD_TOOLCHAIN)" \
 		wasm-pack build --release --target=nodejs \

scripts/no_tfhe_typo.sh

@@ -6,7 +6,7 @@ THIS_SCRIPT_NAME="$(basename "$0")"
 
 TMP_FILE="$(mktemp)"
 
-COUNT="$(git grep -rniI "thfe" . | grep -v "${THIS_SCRIPT_NAME}" | \
+COUNT="$(git grep -rniI "thfe\|tfhr\|thfr" . | grep -v "${THIS_SCRIPT_NAME}" | \
     tee "${TMP_FILE}" | wc -l | tr -d '[:space:]')"
 
 cat "${TMP_FILE}"

tfhe/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tfhe"
-version = "0.2.2"
+version = "0.2.4"
 edition = "2021"
 readme = "../README.md"
 keywords = ["fully", "homomorphic", "encryption", "fhe", "cryptography"]

tfhe/docs/core_crypto/tutorial.md

@@ -9,7 +9,7 @@ Welcome to this tutorial about TFHE-rs `core_crypto` module.
 To use `TFHE-rs`, first it has to be added as a dependency in the `Cargo.toml`:
 
 ```toml
-tfhe = { version = "0.2.2", features = [ "x86_64-unix" ] }
+tfhe = { version = "0.2.4", features = [ "x86_64-unix" ] }
 ```
 
 This enables the `x86_64-unix` feature to have efficient implementations of various algorithms for `x86_64` CPUs on a Unix-like system. The 'unix' suffix indicates that the `UnixSeeder`, which uses `/dev/random` to generate random numbers, is activated as a fallback if no hardware number generator is available, like `rdseed` on `x86_64` or if the [`Randomization Services`](https://developer.apple.com/documentation/security/1399291-secrandomcopybytes?language=objc) on Apple platforms are not available. To avoid having the `UnixSeeder` as a potential fallback or to run on non-Unix systems (e.g., Windows), the `x86_64` feature is sufficient.
@@ -19,19 +19,19 @@ For Apple Silicon, the `aarch64-unix` or `aarch64` feature should be enabled. `a
 In short: For x86\_64-based machines running Unix-like OSes:
 
 ```toml
-tfhe = { version = "0.2.2", features = ["x86_64-unix"] }
+tfhe = { version = "0.2.4", features = ["x86_64-unix"] }
 ```
 
 For Apple Silicon or aarch64-based machines running Unix-like OSes:
 
 ```toml
-tfhe = { version = "0.2.2", features = ["aarch64-unix"] }
+tfhe = { version = "0.2.4", features = ["aarch64-unix"] }
 ```
 
 For x86\_64-based machines with the [`rdseed instruction`](https://en.wikipedia.org/wiki/RDRAND) running Windows:
 
 ```toml
-tfhe = { version = "0.2.2", features = ["x86_64"] }
+tfhe = { version = "0.2.4", features = ["x86_64"] }
 ```
 
 ### Commented code to double a 2-bits message in a leveled fashion and using a PBS with the `core_crypto` module.

tfhe/docs/getting_started/installation.md

@@ -5,7 +5,7 @@
 To use `TFHE-rs` in your project, you first need to add it as a dependency in your `Cargo.toml`:
 
 ```toml
-tfhe = { version = "0.2.2", features = [ "boolean", "shortint", "integer", "x86_64-unix" ] }
+tfhe = { version = "0.2.4", features = [ "boolean", "shortint", "integer", "x86_64-unix" ] }
 ```
 
 {% hint style="info" %}

tfhe/docs/high_level_api/serialization.md

@@ -11,7 +11,7 @@ To serialize our data, a [data format](https://serde.rs/#data-formats) should be
 
 [dependencies]
 # ...
-tfhe = { version = "0.2.2", features = ["integer","x86_64-unix"]}
+tfhe = { version = "0.2.4", features = ["integer","x86_64-unix"]}
 bincode = "1.3.3"
 ```
 

tfhe/docs/high_level_api/tutorial.md

@@ -46,7 +46,7 @@ fn main() {
 
 Default configuration for x86 Unix machines:
 ```toml
-tfhe = { version = "0.2.2", features = ["integer", "x86_64-unix"]}
+tfhe = { version = "0.2.4", features = ["integer", "x86_64-unix"]}
 ```
 
 Other configurations can be found [here](../getting_started/installation.md).
@@ -190,7 +190,7 @@ To use the `FheUint8` type, the `integer` feature must be activated:
 
 [dependencies]
 # Default configuration for x86 Unix machines:
-tfhe = { version = "0.2.2", features = ["integer", "x86_64-unix"]}
+tfhe = { version = "0.2.4", features = ["integer", "x86_64-unix"]}
 ```
 
 Other configurations can be found [here](../getting_started/installation.md).
@@ -319,7 +319,7 @@ To use Booleans, the `booleans` feature in our Cargo.toml must be enabled:
 # Cargo.toml
 
 # Default configuration for x86 Unix machines:
-tfhe = { version = "0.2.2", features = ["boolean", "x86_64-unix"]}
+tfhe = { version = "0.2.4", features = ["boolean", "x86_64-unix"]}
 ```
 
 Other configurations can be found [here](../getting_started/installation.md).

tfhe/src/core_crypto/algorithms/lwe_programmable_bootstrapping.rs

@@ -25,7 +25,8 @@ use dyn_stack::{PodStack, SizeOverflow, StackReq};
 
 /// Perform a blind rotation given an input [`LWE ciphertext`](`LweCiphertext`), modifying a look-up
 /// table passed as a [`GLWE ciphertext`](`GlweCiphertext`) and an [`LWE bootstrap
-/// key`](`LweBootstrapKey`) in the fourier domain.
+/// key`](`LweBootstrapKey`) in the fourier domain see [`fourier LWE bootstrap
+/// key`](`FourierLweBootstrapKey`).
 ///
 /// If you want to manage the computation memory manually you can use
 /// [`blind_rotate_assign_mem_optimized`].
@@ -811,7 +812,8 @@ pub fn cmux_assign_mem_optimized_requirement<Scalar>(
 
 /// Perform a programmable bootstrap given an input [`LWE ciphertext`](`LweCiphertext`), a
 /// look-up table passed as a [`GLWE ciphertext`](`GlweCiphertext`) and an [`LWE bootstrap
-/// key`](`LweBootstrapKey`) in the fourier domain. The result is written in the provided output
+/// key`](`LweBootstrapKey`) in the fourier domain see [`fourier LWE bootstrap
+/// key`](`FourierLweBootstrapKey`). The result is written in the provided output
 /// [`LWE ciphertext`](`LweCiphertext`).
 ///
 /// If you want to manage the computation memory manually you can use
@@ -1119,11 +1121,12 @@ pub fn programmable_bootstrap_lwe_ciphertext_mem_optimized_requirement<Scalar>(
 
 /// Perform a programmable bootstrap given an input [`LWE ciphertext`](`LweCiphertext`), a
 /// look-up table passed as a [`GLWE ciphertext`](`GlweCiphertext`) and an [`LWE bootstrap
-/// key`](`LweBootstrapKey`) in the fourier domain using f128. The result is written in the provided
+/// key`](`LweBootstrapKey`) in the fourier domain using f128 see [`fourier LWE bootstrap
+/// key`](`Fourier128LweBootstrapKey`). The result is written in the provided
 /// output [`LWE ciphertext`](`LweCiphertext`).
 ///
 /// If you want to manage the computation memory manually you can use
-/// [`programmable_bootstrap_lwe_ciphertext_mem_optimized`].
+/// [`programmable_bootstrap_f128_lwe_ciphertext_mem_optimized`].
 ///
 /// # Example
 ///

tfhe/src/core_crypto/commons/generators/encryption.rs

@@ -610,7 +610,18 @@ mod test {
         let bits = (Scalar::BITS / 2) as i32;
 
         for _ in 0..1000 {
-            let val: Scalar = gen.random_noise(StandardDev(2.0f64.powi(-bits)));
+            let mut retries = 100;
+
+            let mut val = Scalar::ZERO;
+            while retries >= 0 {
+                val = gen.random_noise(StandardDev(2.0f64.powi(-bits)));
+                if val != Scalar::ZERO {
+                    break;
+                }
+                retries -= 1;
+            }
+
+            assert!(retries != 0);
             assert!(val != Scalar::ZERO);
         }
     }
@@ -636,8 +647,19 @@ mod test {
         let bits = (Scalar::BITS / 2) as i32;
 
         for _ in 0..1000 {
-            let val: Scalar =
-                gen.random_noise_custom_mod(StandardDev(2.0f64.powi(-bits)), ciphertext_modulus);
+            let mut retries = 100;
+
+            let mut val = Scalar::ZERO;
+            while retries >= 0 {
+                val = gen
+                    .random_noise_custom_mod(StandardDev(2.0f64.powi(-bits)), ciphertext_modulus);
+                if val != Scalar::ZERO {
+                    break;
+                }
+                retries -= 1;
+            }
+
+            assert!(retries != 0);
             assert!(val != Scalar::ZERO);
         }
     }
@@ -677,9 +699,18 @@ mod test {
 
         let bits = (Scalar::BITS / 2) as i32;
 
-        let mut slice = vec![Scalar::ZERO; 1000];
-        gen.fill_slice_with_random_noise(&mut slice, StandardDev(2.0f64.powi(-bits)));
-        assert!(slice.iter().all(|&x| x != Scalar::ZERO))
+        let mut vec = vec![Scalar::ZERO; 1000];
+        let mut retries = 100;
+        while retries >= 0 {
+            gen.fill_slice_with_random_noise(&mut vec, StandardDev(2.0f64.powi(-bits)));
+            if vec.iter().all(|&x| x != Scalar::ZERO) {
+                break;
+            }
+
+            retries -= 1;
+        }
+        assert!(retries != 0);
+        assert!(vec.iter().all(|&x| x != Scalar::ZERO));
     }
 
     #[test]
@@ -704,13 +735,22 @@ mod test {
 
         let bits = (Scalar::BITS / 2) as i32;
 
-        let mut slice = vec![Scalar::ZERO; 1000];
-        gen.fill_slice_with_random_noise_custom_mod(
-            &mut slice,
-            StandardDev(2.0f64.powi(-bits)),
-            ciphertext_modulus,
-        );
-        assert!(slice.iter().all(|&x| x != Scalar::ZERO))
+        let mut vec = vec![Scalar::ZERO; 1000];
+        let mut retries = 100;
+        while retries >= 0 {
+            gen.fill_slice_with_random_noise_custom_mod(
+                &mut vec,
+                StandardDev(2.0f64.powi(-bits)),
+                ciphertext_modulus,
+            );
+            if vec.iter().all(|&x| x != Scalar::ZERO) {
+                break;
+            }
+
+            retries -= 1;
+        }
+        assert!(retries != 0);
+        assert!(vec.iter().all(|&x| x != Scalar::ZERO));
     }
 
     #[test]
@@ -746,9 +786,18 @@ mod test {
     fn mask_gen_slice_native<Scalar: UnsignedTorus>() {
         let mut gen = new_encryption_random_generator();
 
-        let mut slice = vec![Scalar::ZERO; 1000];
-        gen.fill_slice_with_random_mask(&mut slice);
-        assert!(slice.iter().all(|&x| x != Scalar::ZERO))
+        let mut vec = vec![Scalar::ZERO; 1000];
+        let mut retries = 100;
+        while retries >= 0 {
+            gen.fill_slice_with_random_mask(&mut vec);
+            if vec.iter().all(|&x| x != Scalar::ZERO) {
+                break;
+            }
+
+            retries -= 1;
+        }
+        assert!(retries != 0);
+        assert!(vec.iter().all(|&x| x != Scalar::ZERO));
     }
 
     #[test]
@@ -771,9 +820,18 @@ mod test {
     ) {
         let mut gen = new_encryption_random_generator();
 
-        let mut slice = vec![Scalar::ZERO; 1000];
-        gen.fill_slice_with_random_mask_custom_mod(&mut slice, ciphertext_modulus);
-        assert!(slice.iter().all(|&x| x != Scalar::ZERO))
+        let mut vec = vec![Scalar::ZERO; 1000];
+        let mut retries = 100;
+        while retries >= 0 {
+            gen.fill_slice_with_random_mask_custom_mod(&mut vec, ciphertext_modulus);
+            if vec.iter().all(|&x| x != Scalar::ZERO) {
+                break;
+            }
+
+            retries -= 1;
+        }
+        assert!(retries != 0);
+        assert!(vec.iter().all(|&x| x != Scalar::ZERO));
     }
 
     #[test]

tfhe/src/core_crypto/commons/math/random/generator.rs

@@ -177,7 +177,7 @@ impl<G: ByteRandomGenerator> RandomGenerator<G> {
     /// let mut generator = RandomGenerator::<SoftwareRandomGenerator>::new(Seed(0));
     /// let mut vec = vec![0u32; 1000];
     /// generator.fill_slice_with_random_uniform(&mut vec);
-    /// assert!(vec.iter().all(|&x| x != 0));
+    /// assert!(vec.iter().any(|&x| x != 0));
     /// ```
     pub fn fill_slice_with_random_uniform<Scalar>(&mut self, output: &mut [Scalar])
     where
@@ -202,7 +202,7 @@ impl<G: ByteRandomGenerator> RandomGenerator<G> {
     ///     &mut vec,
     ///     CiphertextModulus::try_new_power_of_2(31).unwrap(),
     /// );
-    /// assert!(vec.iter().all(|&x| x != 0));
+    /// assert!(vec.iter().any(|&x| x != 0));
     /// ```
     pub fn fill_slice_with_random_uniform_custom_mod<Scalar>(
         &mut self,
@@ -351,15 +351,11 @@ impl<G: ByteRandomGenerator> RandomGenerator<G> {
     /// // check that both samples are in 6 sigmas.
     /// assert!(g1.abs() <= 6.);
     /// assert!(g2.abs() <= 6.);
-    /// assert!(g1 != 0.);
-    /// assert!(g2 != 0.);
     /// // for f64
     /// let (g1, g2): (f64, f64) = generator.random_gaussian(0. as f64, 1. as f64);
     /// // check that both samples are in 6 sigmas.
     /// assert!(g1.abs() <= 6.);
     /// assert!(g2.abs() <= 6.);
-    /// assert!(g1 != 0.);
-    /// assert!(g2 != 0.);
     /// ```
     pub fn random_gaussian<Float, Scalar>(&mut self, mean: Float, std: Float) -> (Scalar, Scalar)
     where
@@ -380,7 +376,7 @@ impl<G: ByteRandomGenerator> RandomGenerator<G> {
     /// let mut generator = RandomGenerator::<SoftwareRandomGenerator>::new(Seed(0));
     /// let mut vec = vec![0f32; 1000];
     /// generator.fill_slice_with_random_gaussian(&mut vec, 0., 1.);
-    /// assert!(vec.iter().all(|&x| x != 0.));
+    /// assert!(vec.iter().any(|&x| x != 0.));
     /// ```
     pub fn fill_slice_with_random_gaussian<Float, Scalar>(
         &mut self,
@@ -419,7 +415,7 @@ impl<G: ByteRandomGenerator> RandomGenerator<G> {
     ///     1.,
     ///     CiphertextModulus::try_new_power_of_2(63).unwrap(),
     /// );
-    /// assert!(vec.iter().all(|&x| x != 0));
+    /// assert!(vec.iter().any(|&x| x != 0));
     /// ```
     pub fn fill_slice_with_random_gaussian_custom_mod<Float, Scalar>(
         &mut self,
@@ -464,7 +460,7 @@ impl<G: ByteRandomGenerator> RandomGenerator<G> {
     /// let mut generator = RandomGenerator::<SoftwareRandomGenerator>::new(Seed(0));
     /// let mut vec = vec![0u32; 1000];
     /// generator.unsigned_torus_slice_wrapping_add_random_gaussian_assign(&mut vec, 0., 1.);
-    /// assert!(vec.iter().all(|&x| x != 0));
+    /// assert!(vec.iter().any(|&x| x != 0));
     /// ```
     pub fn unsigned_torus_slice_wrapping_add_random_gaussian_assign<Float, Scalar>(
         &mut self,
@@ -503,7 +499,7 @@ impl<G: ByteRandomGenerator> RandomGenerator<G> {
     ///     1.,
     ///     CiphertextModulus::try_new_power_of_2(31).unwrap(),
     /// );
-    /// assert!(vec.iter().all(|&x| x != 0));
+    /// assert!(vec.iter().any(|&x| x != 0));
     /// ```
     pub fn unsigned_torus_slice_wrapping_add_random_gaussian_custom_mod_assign<Float, Scalar>(
         &mut self,

tfhe/src/core_crypto/fft_impl/fft128/crypto/bootstrap.rs

@@ -17,6 +17,8 @@ use crate::core_crypto::entities::*;
 use crate::core_crypto::fft_impl::common::{pbs_modulus_switch, FourierBootstrapKey};
 use crate::core_crypto::prelude::ContainerMut;
 use aligned_vec::{avec, ABox, CACHELINE_ALIGN};
+use core::any::TypeId;
+use core::mem::transmute;
 use dyn_stack::{PodStack, ReborrowMut, SizeOverflow, StackReq};
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
@@ -355,6 +357,14 @@ where
             fft: Fft128View<'_>,
             stack: PodStack<'_>,
         ) {
+            if TypeId::of::<Scalar>() == TypeId::of::<u128>() {
+                let mut lwe_out: LweCiphertext<&mut [u128]> = unsafe { transmute(lwe_out) };
+                let lwe_in: LweCiphertext<&[u128]> = unsafe { transmute(lwe_in) };
+                let accumulator: GlweCiphertext<&[u128]> = unsafe { transmute(accumulator) };
+
+                return this.bootstrap_u128(&mut lwe_out, &lwe_in, &accumulator, fft, stack);
+            }
+
             let (mut local_accumulator_data, stack) =
                 stack.collect_aligned(CACHELINE_ALIGN, accumulator.as_ref().iter().copied());
             let mut local_accumulator = GlweCiphertextMutView::from_container(

tfhe/src/core_crypto/fft_impl/fft128/math/fft/mod.rs

@@ -124,7 +124,7 @@ fn f128_floor(x: f128) -> f128 {
     let f128(x0, x1) = x;
     let x0_floor = x0.floor();
     if x0_floor == x0 {
-        f128(x0_floor, x1.floor())
+        f128::add_f64_f64(x0_floor, x1.floor())
     } else {
         f128(x0_floor, 0.0)
     }
@@ -225,7 +225,7 @@ fn u128_to_signed_to_f128(x: u128) -> f128 {
 
 #[inline(always)]
 fn u128_from_torus_f128(x: f128) -> u128 {
-    let mut x = x - f128_floor(x);
+    let mut x = f128::sub_estimate_f128_f128(x, f128_floor(x));
 
     let normalization = 2.0f64.powi(128);
     x.0 *= normalization;

tfhe/src/core_crypto/fft_impl/fft128_u128/crypto/bootstrap.rs

@@ -166,9 +166,7 @@ where
         )
     }
 
-    // TODO: investigate split version noise
-    #[allow(dead_code)]
-    pub(crate) fn bootstrap_u128<ContLweOut, ContLweIn, ContAcc>(
+    pub fn bootstrap_u128<ContLweOut, ContLweIn, ContAcc>(
         &self,
         lwe_out: &mut LweCiphertext<ContLweOut>,
         lwe_in: &LweCiphertext<ContLweIn>,

tfhe/src/core_crypto/fft_impl/fft128_u128/crypto/mod.rs

@@ -1,2 +1,5 @@
 pub mod bootstrap;
 pub mod ggsw;
+
+#[cfg(test)]
+mod tests;

tfhe/src/core_crypto/fft_impl/fft128_u128/crypto/tests.rs

@@ -0,0 +1,279 @@
+use dyn_stack::{GlobalPodBuffer, PodStack, ReborrowMut};
+
+use super::super::super::{fft128, fft128_u128};
+use super::super::math::fft::{Fft128, Fft128View};
+use crate::core_crypto::prelude::*;
+use aligned_vec::CACHELINE_ALIGN;
+
+fn sqr(x: f64) -> f64 {
+    x * x
+}
+
+#[test]
+fn test_split_external_product() {
+    let small_lwe_dimension = LweDimension(742);
+    let glwe_dimension = GlweDimension(1);
+    let polynomial_size = PolynomialSize(2048);
+    let pbs_base_log = DecompositionBaseLog(23);
+    let pbs_level = DecompositionLevelCount(1);
+    let glwe_modular_std_dev = StandardDev(sqr(0.00000000000000029403601535432533));
+    let ciphertext_modulus = CiphertextModulus::<u128>::new_native();
+    let ciphertext_modulus_split = CiphertextModulus::<u64>::new_native();
+
+    let mut glwe = GlweCiphertext::new(
+        0u128,
+        glwe_dimension.to_glwe_size(),
+        polynomial_size,
+        ciphertext_modulus,
+    );
+
+    for x in glwe.as_mut() {
+        *x = rand::random();
+    }
+
+    let mut boxed_seeder = new_seeder();
+    let seeder = boxed_seeder.as_mut();
+    let mut secret_generator =
+        SecretRandomGenerator::<ActivatedRandomGenerator>::new(seeder.seed());
+    let mut encryption_generator =
+        EncryptionRandomGenerator::<ActivatedRandomGenerator>::new(seeder.seed(), seeder);
+
+    let small_lwe_sk =
+        LweSecretKey::generate_new_binary(small_lwe_dimension, &mut secret_generator);
+    let glwe_sk =
+        GlweSecretKey::generate_new_binary(glwe_dimension, polynomial_size, &mut secret_generator);
+
+    let std_bootstrapping_key = par_allocate_and_generate_new_lwe_bootstrap_key(
+        &small_lwe_sk,
+        &glwe_sk,
+        pbs_base_log,
+        pbs_level,
+        glwe_modular_std_dev,
+        ciphertext_modulus,
+        &mut encryption_generator,
+    );
+
+    let mut fourier_bsk = Fourier128LweBootstrapKey::new(
+        std_bootstrapping_key.input_lwe_dimension(),
+        std_bootstrapping_key.glwe_size(),
+        std_bootstrapping_key.polynomial_size(),
+        std_bootstrapping_key.decomposition_base_log(),
+        std_bootstrapping_key.decomposition_level_count(),
+    );
+
+    let fft = Fft128::new(polynomial_size);
+    let fft = fft.as_view();
+    fourier_bsk.fill_with_forward_fourier(&std_bootstrapping_key, fft);
+    let ggsw = fourier_bsk.as_view().into_ggsw_iter().next().unwrap();
+
+    let mut glwe_lo = GlweCiphertext::new(
+        0u64,
+        glwe_dimension.to_glwe_size(),
+        polynomial_size,
+        ciphertext_modulus_split,
+    );
+    let mut glwe_hi = GlweCiphertext::new(
+        0u64,
+        glwe_dimension.to_glwe_size(),
+        polynomial_size,
+        ciphertext_modulus_split,
+    );
+
+    for ((lo, hi), val) in glwe_lo
+        .as_mut()
+        .iter_mut()
+        .zip(glwe_hi.as_mut())
+        .zip(glwe.as_ref())
+    {
+        *lo = *val as u64;
+        *hi = (*val >> 64) as u64;
+    }
+
+    let mut out = GlweCiphertext::new(
+        0u128,
+        glwe_dimension.to_glwe_size(),
+        polynomial_size,
+        ciphertext_modulus,
+    );
+
+    fft128::crypto::ggsw::add_external_product_assign(
+        &mut out,
+        &ggsw,
+        &glwe,
+        fft,
+        PodStack::new(&mut GlobalPodBuffer::new(
+            fft128::crypto::ggsw::add_external_product_assign_scratch::<u128>(
+                glwe_dimension.to_glwe_size(),
+                polynomial_size,
+                fft,
+            )
+            .unwrap(),
+        )),
+    );
+
+    let mut out_lo = GlweCiphertext::new(
+        0u64,
+        glwe_dimension.to_glwe_size(),
+        polynomial_size,
+        ciphertext_modulus_split,
+    );
+    let mut out_hi = GlweCiphertext::new(
+        0u64,
+        glwe_dimension.to_glwe_size(),
+        polynomial_size,
+        ciphertext_modulus_split,
+    );
+
+    fft128_u128::crypto::ggsw::add_external_product_assign_split(
+        &mut out_lo,
+        &mut out_hi,
+        &ggsw,
+        &glwe_lo,
+        &glwe_hi,
+        fft,
+        PodStack::new(&mut GlobalPodBuffer::new(
+            fft128::crypto::ggsw::add_external_product_assign_scratch::<u128>(
+                glwe_dimension.to_glwe_size(),
+                polynomial_size,
+                fft,
+            )
+            .unwrap(),
+        )),
+    );
+
+    for ((lo, hi), val) in out_lo
+        .as_ref()
+        .iter()
+        .zip(out_hi.as_ref())
+        .zip(out.as_ref())
+    {
+        assert_eq!(*val as u64, *lo);
+        assert_eq!((*val >> 64) as u64, *hi);
+    }
+}
+
+#[test]
+fn test_split_pbs() {
+    let small_lwe_dimension = LweDimension(742);
+    let glwe_dimension = GlweDimension(1);
+    let polynomial_size = PolynomialSize(2048);
+    let pbs_base_log = DecompositionBaseLog(23);
+    let pbs_level = DecompositionLevelCount(1);
+    let glwe_modular_std_dev = StandardDev(sqr(0.00000000000000029403601535432533));
+    let ciphertext_modulus = CiphertextModulus::<u128>::new_native();
+
+    let mut boxed_seeder = new_seeder();
+    let seeder = boxed_seeder.as_mut();
+    let mut secret_generator =
+        SecretRandomGenerator::<ActivatedRandomGenerator>::new(seeder.seed());
+    let mut encryption_generator =
+        EncryptionRandomGenerator::<ActivatedRandomGenerator>::new(seeder.seed(), seeder);
+
+    let small_lwe_sk =
+        LweSecretKey::generate_new_binary(small_lwe_dimension, &mut secret_generator);
+    let glwe_sk =
+        GlweSecretKey::generate_new_binary(glwe_dimension, polynomial_size, &mut secret_generator);
+
+    let std_bootstrapping_key = par_allocate_and_generate_new_lwe_bootstrap_key(
+        &small_lwe_sk,
+        &glwe_sk,
+        pbs_base_log,
+        pbs_level,
+        glwe_modular_std_dev,
+        ciphertext_modulus,
+        &mut encryption_generator,
+    );
+
+    let mut fourier_bsk = Fourier128LweBootstrapKey::new(
+        std_bootstrapping_key.input_lwe_dimension(),
+        std_bootstrapping_key.glwe_size(),
+        std_bootstrapping_key.polynomial_size(),
+        std_bootstrapping_key.decomposition_base_log(),
+        std_bootstrapping_key.decomposition_level_count(),
+    );
+
+    let fft = Fft128::new(polynomial_size);
+    let fft = fft.as_view();
+    fourier_bsk.fill_with_forward_fourier(&std_bootstrapping_key, fft);
+
+    let mut lwe_in =
+        LweCiphertext::new(0u128, small_lwe_dimension.to_lwe_size(), ciphertext_modulus);
+    let mut accumulator = GlweCiphertext::new(
+        0u128,
+        glwe_dimension.to_glwe_size(),
+        polynomial_size,
+        ciphertext_modulus,
+    );
+
+    for x in lwe_in.as_mut() {
+        *x = rand::random();
+    }
+    for x in accumulator.as_mut() {
+        *x = rand::random();
+    }
+
+    let mut mem = GlobalPodBuffer::new(
+        fft128::crypto::bootstrap::bootstrap_scratch::<u128>(
+            glwe_dimension.to_glwe_size(),
+            polynomial_size,
+            fft,
+        )
+        .unwrap(),
+    );
+    let mut stack = PodStack::new(&mut mem);
+
+    let mut lwe_out_non_split = LweCiphertext::new(
+        0u128,
+        LweDimension(polynomial_size.0 * glwe_dimension.0).to_lwe_size(),
+        ciphertext_modulus,
+    );
+
+    // Needed as the basic bootstrap function dispatches to the more efficient split version for
+    // u128
+    fn bootstrap_non_split<Scalar: UnsignedTorus + CastInto<usize>>(
+        this: Fourier128LweBootstrapKey<&[f64]>,
+        mut lwe_out: LweCiphertext<&mut [Scalar]>,
+        lwe_in: LweCiphertext<&[Scalar]>,
+        accumulator: GlweCiphertext<&[Scalar]>,
+        fft: Fft128View<'_>,
+        stack: PodStack<'_>,
+    ) {
+        let (mut local_accumulator_data, stack) =
+            stack.collect_aligned(CACHELINE_ALIGN, accumulator.as_ref().iter().copied());
+        let mut local_accumulator = GlweCiphertextMutView::from_container(
+            &mut *local_accumulator_data,
+            accumulator.polynomial_size(),
+            accumulator.ciphertext_modulus(),
+        );
+        this.blind_rotate_assign(&mut local_accumulator.as_mut_view(), &lwe_in, fft, stack);
+        extract_lwe_sample_from_glwe_ciphertext(
+            &local_accumulator,
+            &mut lwe_out,
+            MonomialDegree(0),
+        );
+    }
+
+    bootstrap_non_split(
+        fourier_bsk.as_view(),
+        lwe_out_non_split.as_mut_view(),
+        lwe_in.as_view(),
+        accumulator.as_view(),
+        fft,
+        stack.rb_mut(),
+    );
+
+    let mut lwe_out_split = LweCiphertext::new(
+        0u128,
+        LweDimension(polynomial_size.0 * glwe_dimension.0).to_lwe_size(),
+        ciphertext_modulus,
+    );
+    fourier_bsk.bootstrap_u128(
+        &mut lwe_out_split,
+        &lwe_in,
+        &accumulator,
+        fft,
+        stack.rb_mut(),
+    );
+
+    assert_eq!(lwe_out_split, lwe_out_non_split);
+}

tfhe/src/core_crypto/fft_impl/fft128_u128/math/fft/mod.rs

@@ -206,10 +206,11 @@ pub fn wrapping_add_avx2(
 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
 #[inline(always)]
 pub fn wrapping_neg_avx2(simd: V3, (lo, hi): (u64x4, u64x4)) -> (u64x4, u64x4) {
-    let diff_lo = pulp::cast(simd.wrapping_sub_u64x4(simd.splat_u64x4(0), lo));
-    let overflow = pulp::cast(simd.cmp_lt_u64x4(simd.splat_u64x4(0), lo));
-    let diff_hi = simd.wrapping_sub_u64x4(overflow, hi);
-    (diff_lo, diff_hi)
+    wrapping_add_avx2(
+        simd,
+        (simd.splat_u64x4(1), simd.splat_u64x4(0)),
+        (simd.not_u64x4(lo), simd.not_u64x4(hi)),
+    )
 }
 
 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
@@ -246,10 +247,11 @@ pub fn wrapping_add_avx512(
 #[cfg(feature = "nightly-avx512")]
 #[inline(always)]
 pub fn wrapping_neg_avx512(simd: V4, (lo, hi): (u64x8, u64x8)) -> (u64x8, u64x8) {
-    let diff_lo = simd.wrapping_sub_u64x8(simd.splat_u64x8(0), lo);
-    let overflow = simd.convert_mask_b8_to_u64x8(simd.cmp_lt_u64x8(simd.splat_u64x8(0), lo));
-    let diff_hi = simd.wrapping_sub_u64x8(overflow, hi);
-    (diff_lo, diff_hi)
+    wrapping_add_avx512(
+        simd,
+        (simd.splat_u64x8(1), simd.splat_u64x8(0)),
+        (simd.not_u64x8(lo), simd.not_u64x8(hi)),
+    )
 }
 
 #[inline(always)]
@@ -385,9 +387,10 @@ fn f64_to_i128_avx2(simd: V3, f: f64x4) -> (u64x4, u64x4) {
             simd.shr_dyn_u64x4(hi, shift),
         );
         let neg = wrapping_neg_avx2(simd, abs);
+        let mask = simd.cmp_eq_u64x4(simd.and_u64x4(sign_bit, f), sign_bit);
         (
-            simd.select_u64x4(simd.cmp_eq_u64x4(f, sign_bit), neg.0, abs.0),
-            simd.select_u64x4(simd.cmp_eq_u64x4(f, sign_bit), neg.1, abs.1),
+            simd.select_u64x4(mask, neg.0, abs.0),
+            simd.select_u64x4(mask, neg.1, abs.1),
         )
     };
     (
@@ -554,7 +557,7 @@ fn f128_floor(x: f128) -> f128 {
     let f128(x0, x1) = x;
     let x0_floor = x0.floor();
     if x0_floor == x0 {
-        f128(x0_floor, x1.floor())
+        f128::add_f64_f64(x0_floor, x1.floor())
     } else {
         f128(x0_floor, 0.0)
     }
@@ -566,7 +569,8 @@ fn f128_floor_avx2(simd: V3, (x0, x1): (f64x4, f64x4)) -> (f64x4, f64x4) {
     let x0_floor = simd.floor_f64x4(x0);
     let x1_floor = simd.floor_f64x4(x1);
 
-    (
+    two_sum_f64x4(
+        simd,
         x0_floor,
         simd.select_f64x4(
             simd.cmp_eq_f64x4(x0_floor, x0),
@@ -583,7 +587,8 @@ fn f128_floor_avx512(simd: V4, (x0, x1): (f64x8, f64x8)) -> (f64x8, f64x8) {
     let x0_floor = simd.floor_f64x8(x0);
     let x1_floor = simd.floor_f64x8(x1);
 
-    (
+    two_sum_f64x8(
+        simd,
         x0_floor,
         simd.select_f64x8(
             simd.cmp_eq_f64x8(x0_floor, x0),
@@ -1338,3 +1343,43 @@ impl<'a> Fft128View<'a> {
         );
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    // copied from the standard library
+    fn next_up(this: f64) -> f64 {
+        // We must use strictly integer arithmetic to prevent denormals from
+        // flushing to zero after an arithmetic operation on some platforms.
+        const TINY_BITS: u64 = 0x1; // Smallest positive f64.
+        const CLEAR_SIGN_MASK: u64 = 0x7fff_ffff_ffff_ffff;
+
+        let bits = this.to_bits();
+        if this.is_nan() || bits == f64::INFINITY.to_bits() {
+            return this;
+        }
+
+        let abs = bits & CLEAR_SIGN_MASK;
+        let next_bits = if abs == 0 {
+            TINY_BITS
+        } else if bits == abs {
+            bits + 1
+        } else {
+            bits - 1
+        };
+        f64::from_bits(next_bits)
+    }
+
+    fn ulp(x: f64) -> f64 {
+        next_up(x.abs()) - x.abs()
+    }
+
+    #[test]
+    fn test_f128_floor() {
+        let a = f128(-11984547.0, -1.0316078675142442e-10);
+        let b = f128_floor(a);
+
+        assert!(b.1.abs() <= 0.5 * ulp(b.0))
+    }
+}

tfhe/src/integer/ciphertext/mod.rs

@@ -7,7 +7,7 @@ use crate::shortint::{
 use serde::{Deserialize, Serialize};
 
 /// Structure containing a ciphertext in radix decomposition.
-#[derive(Serialize, Clone, Deserialize)]
+#[derive(Serialize, Clone, Deserialize, PartialEq, Eq, Debug)]
 pub struct BaseRadixCiphertext<Block> {
     /// The blocks are stored from LSB to MSB
     pub(crate) blocks: Vec<Block>,

tfhe/src/integer/server_key/radix_parallel/add.rs

@@ -1,5 +1,3 @@
-use std::sync::Mutex;
-
 use crate::integer::ciphertext::RadixCiphertext;
 use crate::integer::ServerKey;
 use crate::shortint::PBSOrderMarker;
@@ -196,24 +194,25 @@ impl ServerKey {
             } else {
                 // we repeatedly divide the number of terms by two by iteratively reducing
                 // consecutive terms in the array
+                let num_blocks = ct_seq[0].as_mut().blocks.len();
                 while ct_seq.len() > 1 {
-                    let results = Mutex::new(Vec::<RadixCiphertext<PBSOrder>>::with_capacity(
-                        ct_seq.len() / 2,
-                    ));
+                    let mut results =
+                        vec![sks.create_trivial_radix(0u64, num_blocks); ct_seq.len() / 2];
 
                     // if the number of elements is odd, we skip the first element
                     let untouched_prefix = ct_seq.len() % 2;
                     let ct_seq_slice = &mut ct_seq[untouched_prefix..];
 
-                    ct_seq_slice.par_chunks_mut(2).for_each(|chunk| {
-                        let (first, second) = chunk.split_at_mut(1);
-                        let first = &mut first[0];
-                        let second = &mut second[0];
-                        let result = op(sks, first.as_mut(), second.as_mut());
-                        results.lock().unwrap().push(result);
-                    });
+                    results
+                        .par_iter_mut()
+                        .zip(ct_seq_slice.par_chunks_exact_mut(2))
+                        .for_each(|(ct_res, chunk)| {
+                            let (first, second) = chunk.split_at_mut(1);
+                            let first = first[0].as_mut();
+                            let second = second[0].as_mut();
+                            *ct_res = op(sks, first, second);
+                        });
 
-                    let results = results.into_inner().unwrap();
                     ct_seq.truncate(untouched_prefix);
                     ct_seq.extend(results.into_iter().map(CiphertextCow::Owned));
                 }
@@ -281,24 +280,23 @@ impl ServerKey {
             } else {
                 // we repeatedly divide the number of terms by two by iteratively reducing
                 // consecutive terms in the array
+                let num_blocks = ct_seq[0].as_ref().blocks.len();
                 while ct_seq.len() > 1 {
-                    let results = Mutex::new(Vec::<RadixCiphertext<PBSOrder>>::with_capacity(
-                        ct_seq.len() / 2,
-                    ));
-
+                    let mut results =
+                        vec![sks.create_trivial_radix(0u64, num_blocks); ct_seq.len() / 2];
                     // if the number of elements is odd, we skip the first element
                     let untouched_prefix = ct_seq.len() % 2;
                     let ct_seq_slice = &mut ct_seq[untouched_prefix..];
 
-                    ct_seq_slice.par_chunks(2).for_each(|chunk| {
-                        let (first, second) = chunk.split_at(1);
-                        let first = &first[0];
-                        let second = &second[0];
-                        let result = op(sks, first.as_ref(), second.as_ref());
-                        results.lock().unwrap().push(result);
-                    });
+                    results
+                        .par_iter_mut()
+                        .zip(ct_seq_slice.par_chunks_exact(2))
+                        .for_each(|(ct_res, chunk)| {
+                            let first = chunk[0].as_ref();
+                            let second = chunk[1].as_ref();
+                            *ct_res = op(sks, first, second);
+                        });
 
-                    let results = results.into_inner().unwrap();
                     ct_seq.truncate(untouched_prefix);
                     ct_seq.extend(results.into_iter().map(CiphertextCow::Owned));
                 }

tfhe/src/integer/server_key/radix_parallel/mul.rs

@@ -1,5 +1,3 @@
-use std::sync::Mutex;
-
 use crate::integer::ciphertext::RadixCiphertext;
 use crate::integer::ServerKey;
 use crate::shortint::PBSOrderMarker;
@@ -335,14 +333,15 @@ impl ServerKey {
     ) -> RadixCiphertext<PBSOrder> {
         let mut result = self.create_trivial_zero_radix(ct1.blocks.len());
 
-        let terms = Mutex::new(Vec::new());
+        let num_blocks = ct1.blocks.len();
+        let mut terms = vec![self.create_trivial_zero_radix(num_blocks); num_blocks];
 
-        ct2.blocks.par_iter().enumerate().for_each(|(i, ct2_i)| {
-            let term = self.unchecked_block_mul_parallelized(ct1, ct2_i, i);
-            terms.lock().unwrap().push(term);
-        });
-
-        let mut terms = terms.into_inner().unwrap();
+        terms
+            .par_iter_mut()
+            .zip(ct2.blocks.par_iter().enumerate())
+            .for_each(|(term, (i, ct2_i))| {
+                *term = self.unchecked_block_mul_parallelized(ct1, ct2_i, i);
+            });
 
         for term in terms.iter_mut() {
             self.smart_add_assign(&mut result, term);
@@ -407,12 +406,15 @@ impl ServerKey {
             || self.full_propagate_parallelized(ct2),
         );
 
-        let terms = Mutex::new(Vec::new());
-        ct2.blocks.par_iter().enumerate().for_each(|(i, ct2_i)| {
-            let term = self.unchecked_block_mul_parallelized(ct1, ct2_i, i);
-            terms.lock().unwrap().push(term);
-        });
-        let mut terms = terms.into_inner().unwrap();
+        let num_blocks = ct1.blocks.len();
+        let mut terms = vec![self.create_trivial_zero_radix(num_blocks); num_blocks];
+
+        terms
+            .par_iter_mut()
+            .zip(ct2.blocks.par_iter().enumerate())
+            .for_each(|(term, (i, ct2_i))| {
+                *term = self.unchecked_block_mul_parallelized(ct1, ct2_i, i);
+            });
 
         self.smart_binary_op_seq_parallelized(&mut terms, ServerKey::smart_add_parallelized)
             .unwrap_or_else(|| self.create_trivial_zero_radix(ct1.blocks.len()))

tfhe/src/integer/server_key/radix_parallel/scalar_mul.rs

@@ -4,8 +4,6 @@ use crate::integer::server_key::CheckError::CarryFull;
 use crate::integer::ServerKey;
 use crate::shortint::PBSOrderMarker;
 use rayon::prelude::*;
-use std::collections::HashMap;
-use std::sync::Mutex;
 
 impl ServerKey {
     /// Computes homomorphically a multiplication between a scalar and a ciphertext.
@@ -363,53 +361,68 @@ impl ServerKey {
             return zero;
         }
 
+        let num_tasks = self.key.message_modulus.0;
         let b = self.key.message_modulus.0 as u64;
-        let n = ct.blocks.len();
+        let num_blocks = ct.blocks.len();
 
         //Propagate the carries before doing the multiplications
         self.full_propagate_parallelized(ct);
         let ct = &*ct;
 
-        // key is the small scalar we multiply by
-        // value is the vector of blockshifts
-        let mut task_map = HashMap::<u64, Vec<usize>>::new();
+        // index is the small scalar we multiply by, value is the vector of blockshifts
+        let mut task_vec: Vec<Vec<usize>> =
+            vec![Vec::with_capacity((u64::BITS / b.ilog2()) as usize); num_tasks];
 
         // Divide scalar progressively towards zero
         let mut scalar_i = scalar;
-        for i in 0..n {
+        for i in 0..num_blocks {
             let u_i = scalar_i % b;
-            task_map.entry(u_i).or_insert_with(Vec::new).push(i);
+            task_vec[u_i as usize].push(i);
             scalar_i /= b;
             if scalar_i == 0 {
                 break;
             }
         }
 
-        let terms = Mutex::new(Vec::<RadixCiphertext<PBSOrder>>::new());
-        task_map.par_iter().for_each(|(&u_i, blockshifts)| {
-            if u_i == 0 {
-                return;
-            }
+        let task_vec: Vec<_> = task_vec
+            .into_iter()
+            .enumerate()
+            .skip(1) // skip u_i == 0, multiplying by 0 yielding 0
+            .filter(|(_u_i, blockshifts)| !blockshifts.is_empty())
+            .collect();
 
-            let blockshifts = &**blockshifts;
-            let min_blockshift = *blockshifts.iter().min().unwrap();
+        let mut terms: Vec<_> = task_vec
+            .iter()
+            .map(|(_, blockshifts)| {
+                vec![self.create_trivial_zero_radix(num_blocks); blockshifts.len()]
+            })
+            .collect();
+        terms
+            .par_iter_mut()
+            .zip(task_vec.par_iter())
+            .for_each(|(term_vec, (u_i, blockshifts))| {
+                let min_blockshift = blockshifts.iter().min().unwrap();
 
-            let mut tmp = ct.clone();
-            if u_i != 1 {
-                tmp.blocks[0..n - min_blockshift]
+                let u_i = *u_i;
+                let mut tmp = ct.clone();
+                if u_i != 1 {
+                    tmp.blocks[0..num_blocks - *min_blockshift]
+                        .par_iter_mut()
+                        .for_each(|ct_i| self.key.unchecked_scalar_mul_assign(ct_i, u_i as u8));
+                }
+
+                term_vec
                     .par_iter_mut()
-                    .for_each(|ct_i| self.key.unchecked_scalar_mul_assign(ct_i, u_i as u8));
-            }
-
-            let tmp = &tmp;
-            blockshifts.par_iter().for_each(|&shift| {
-                let term = self.blockshift(tmp, shift);
-                terms.lock().unwrap().push(term);
+                    .zip(blockshifts.par_iter())
+                    .for_each(|(term, &shift)| {
+                        *term = self.blockshift(&tmp, shift);
+                    });
             });
-        });
-        let mut terms = terms.into_inner().unwrap();
-        self.smart_binary_op_seq_parallelized(&mut terms, ServerKey::smart_add_parallelized)
-            .unwrap_or(zero)
+        self.smart_binary_op_seq_parallelized(
+            terms.iter_mut().flatten(),
+            ServerKey::smart_add_parallelized,
+        )
+        .unwrap_or(zero)
     }
 
     pub fn smart_scalar_mul_assign_parallelized<PBSOrder: PBSOrderMarker>(
@@ -475,51 +488,67 @@ impl ServerKey {
             return;
         }
 
+        let num_tasks = self.key.message_modulus.0;
         let b = self.key.message_modulus.0 as u64;
-        let n = ct.blocks.len();
+        let num_blocks = ct.blocks.len();
 
         //Propagate the carries before doing the multiplications
         self.full_propagate_parallelized(ct);
 
-        // key is the small scalar we multiply by
-        // value is the vector of blockshifts
-        let mut task_map = HashMap::<u64, Vec<usize>>::new();
+        // index is the small scalar we multiply by, value is the vector of blockshifts
+        let mut task_vec: Vec<Vec<usize>> =
+            vec![Vec::with_capacity((u64::BITS / b.ilog2()) as usize); num_tasks];
 
+        // Divide scalar progressively towards zero
         let mut scalar_i = scalar;
-        for i in 0..n {
+        for i in 0..num_blocks {
             let u_i = scalar_i % b;
-            task_map.entry(u_i).or_insert_with(Vec::new).push(i);
+            task_vec[u_i as usize].push(i);
             scalar_i /= b;
             if scalar_i == 0 {
                 break;
             }
         }
 
-        let terms = Mutex::new(Vec::<RadixCiphertext<PBSOrder>>::new());
-        task_map.par_iter().for_each(|(&u_i, blockshifts)| {
-            if u_i == 0 {
-                return;
-            }
+        let task_vec: Vec<_> = task_vec
+            .into_iter()
+            .enumerate()
+            .skip(1) // skip u_i == 0, multiplying by 0 yielding 0
+            .filter(|(_u_i, blockshifts)| !blockshifts.is_empty())
+            .collect();
 
-            let blockshifts = &**blockshifts;
-            let min_blockshift = *blockshifts.iter().min().unwrap();
+        let mut terms: Vec<_> = task_vec
+            .iter()
+            .map(|(_, blockshifts)| {
+                vec![self.create_trivial_zero_radix(num_blocks); blockshifts.len()]
+            })
+            .collect();
+        terms
+            .par_iter_mut()
+            .zip(task_vec.par_iter())
+            .for_each(|(term_vec, (u_i, blockshifts))| {
+                let min_blockshift = blockshifts.iter().min().unwrap();
 
-            let mut tmp = ct.clone();
-            if u_i != 1 {
-                tmp.blocks[0..n - min_blockshift]
+                let u_i = *u_i;
+                let mut tmp = ct.clone();
+                if u_i != 1 {
+                    tmp.blocks[0..num_blocks - *min_blockshift]
+                        .par_iter_mut()
+                        .for_each(|ct_i| self.key.unchecked_scalar_mul_assign(ct_i, u_i as u8));
+                }
+
+                term_vec
                     .par_iter_mut()
-                    .for_each(|ct_i| self.key.unchecked_scalar_mul_assign(ct_i, u_i as u8));
-            }
-
-            let tmp = &tmp;
-            blockshifts.par_iter().for_each(|&shift| {
-                let term = self.blockshift(tmp, shift);
-                terms.lock().unwrap().push(term);
+                    .zip(blockshifts.par_iter())
+                    .for_each(|(term, &shift)| {
+                        *term = self.blockshift(&tmp, shift);
+                    });
             });
-        });
-        let terms = terms.into_inner().unwrap();
         *ct = self
-            .default_binary_op_seq_parallelized(&terms, ServerKey::add_parallelized)
+            .smart_binary_op_seq_parallelized(
+                terms.iter_mut().flatten(),
+                ServerKey::smart_add_parallelized,
+            )
             .unwrap_or(zero);
         self.full_propagate_parallelized(ct);
     }

tfhe/src/integer/server_key/radix_parallel/tests.rs

@@ -97,7 +97,6 @@ fn integer_smart_add(param: Parameters) {
         // encryption of an integer
         let mut ctxt_1 = cks.encrypt(clear_1);
 
-        // add the two ciphertexts
         let mut ct_res = sks.smart_add_parallelized(&mut ctxt_0, &mut ctxt_1);
 
         clear = (clear_0 + clear_1) % modulus;
@@ -141,7 +140,6 @@ fn integer_smart_add_sequence_multi_thread(param: Parameters) {
                 .map(|clear| cks.encrypt(clear))
                 .collect::<Vec<_>>();
 
-            // add the ciphertexts
             let ct_res = sks
                 .smart_binary_op_seq_parallelized(&mut ctxts, ServerKey::smart_add_parallelized)
                 .unwrap();
@@ -176,7 +174,6 @@ fn integer_smart_add_sequence_single_thread(param: Parameters) {
                 .map(|clear| cks.encrypt(clear))
                 .collect::<Vec<_>>();
 
-            // add the ciphertexts
             let threadpool = rayon::ThreadPoolBuilder::new()
                 .num_threads(1)
                 .build()
@@ -217,9 +214,10 @@ fn integer_default_add(param: Parameters) {
         // encryption of an integer
         let ctxt_1 = cks.encrypt(clear_1);
 
-        // add the two ciphertexts
         let mut ct_res = sks.add_parallelized(&ctxt_0, &ctxt_1);
+        let tmp_ct = sks.add_parallelized(&ctxt_0, &ctxt_1);
         assert!(ct_res.block_carries_are_empty());
+        assert_eq!(ct_res, tmp_ct);
 
         clear = (clear_0 + clear_1) % modulus;
 
@@ -263,11 +261,14 @@ fn integer_default_add_sequence_multi_thread(param: Parameters) {
                 .map(|clear| cks.encrypt(clear))
                 .collect::<Vec<_>>();
 
-            // add the ciphertexts
             let ct_res = sks
                 .default_binary_op_seq_parallelized(&ctxts, ServerKey::add_parallelized)
                 .unwrap();
+            let tmp_ct = sks
+                .default_binary_op_seq_parallelized(&ctxts, ServerKey::add_parallelized)
+                .unwrap();
             assert!(ct_res.block_carries_are_empty());
+            assert_eq!(ct_res, tmp_ct);
             let ct_res: u64 = cks.decrypt(&ct_res);
             let clear = clears.iter().sum::<u64>() % modulus;
 
@@ -299,7 +300,6 @@ fn integer_default_add_sequence_single_thread(param: Parameters) {
                 .map(|clear| cks.encrypt(clear))
                 .collect::<Vec<_>>();
 
-            // add the ciphertexts
             let threadpool = rayon::ThreadPoolBuilder::new()
                 .num_threads(1)
                 .build()
@@ -341,7 +341,6 @@ fn integer_smart_bitand(param: Parameters) {
         // encryption of an integer
         let mut ctxt_1 = cks.encrypt(clear_1);
 
-        // add the two ciphertexts
         let mut ct_res = sks.smart_bitand_parallelized(&mut ctxt_0, &mut ctxt_1);
 
         clear = clear_0 & clear_1;
@@ -387,7 +386,6 @@ fn integer_smart_bitor(param: Parameters) {
         // encryption of an integer
         let mut ctxt_1 = cks.encrypt(clear_1);
 
-        // add the two ciphertexts
         let mut ct_res = sks.smart_bitor_parallelized(&mut ctxt_0, &mut ctxt_1);
 
         clear = (clear_0 | clear_1) % modulus;
@@ -433,7 +431,6 @@ fn integer_smart_bitxor(param: Parameters) {
         // encryption of an integer
         let mut ctxt_1 = cks.encrypt(clear_1);
 
-        // add the two ciphertexts
         let mut ct_res = sks.smart_bitxor_parallelized(&mut ctxt_0, &mut ctxt_1);
 
         clear = (clear_0 ^ clear_1) % modulus;
@@ -479,7 +476,6 @@ fn integer_default_bitand(param: Parameters) {
         // encryption of an integer
         let ctxt_1 = cks.encrypt(clear_1);
 
-        // add the two ciphertexts
         let mut ct_res = sks.bitand_parallelized(&ctxt_0, &ctxt_1);
         assert!(ct_res.block_carries_are_empty());
 
@@ -491,8 +487,10 @@ fn integer_default_bitand(param: Parameters) {
             // encryption of an integer
             let ctxt_2 = cks.encrypt(clear_2);
 
+            let tmp = sks.bitand_parallelized(&ct_res, &ctxt_2);
             ct_res = sks.bitand_parallelized(&ct_res, &ctxt_2);
             assert!(ct_res.block_carries_are_empty());
+            assert_eq!(ct_res, tmp);
             clear &= clear_2;
 
             // decryption of ct_res
@@ -527,7 +525,6 @@ fn integer_default_bitor(param: Parameters) {
         // encryption of an integer
         let ctxt_1 = cks.encrypt(clear_1);
 
-        // add the two ciphertexts
         let mut ct_res = sks.bitor_parallelized(&ctxt_0, &ctxt_1);
         assert!(ct_res.block_carries_are_empty());
 
@@ -539,8 +536,10 @@ fn integer_default_bitor(param: Parameters) {
             // encryption of an integer
             let ctxt_2 = cks.encrypt(clear_2);
 
+            let tmp = sks.bitor_parallelized(&ct_res, &ctxt_2);
             ct_res = sks.bitor_parallelized(&ct_res, &ctxt_2);
             assert!(ct_res.block_carries_are_empty());
+            assert_eq!(ct_res, tmp);
             clear = (clear | clear_2) % modulus;
 
             // decryption of ct_res
@@ -575,7 +574,6 @@ fn integer_default_bitxor(param: Parameters) {
         // encryption of an integer
         let ctxt_1 = cks.encrypt(clear_1);
 
-        // add the two ciphertexts
         let mut ct_res = sks.bitxor_parallelized(&ctxt_0, &ctxt_1);
         assert!(ct_res.block_carries_are_empty());
 
@@ -587,8 +585,10 @@ fn integer_default_bitxor(param: Parameters) {
             // encryption of an integer
             let ctxt_2 = cks.encrypt(clear_2);
 
+            let tmp = sks.bitxor_parallelized(&ct_res, &ctxt_2);
             ct_res = sks.bitxor_parallelized(&ct_res, &ctxt_2);
             assert!(ct_res.block_carries_are_empty());
+            assert_eq!(ct_res, tmp);
             clear = (clear ^ clear_2) % modulus;
 
             // decryption of ct_res
@@ -620,7 +620,6 @@ fn integer_unchecked_small_scalar_mul(param: Parameters) {
         // encryption of an integer
         let ct = cks.encrypt(clear);
 
-        // add the two ciphertexts
         let ct_res = sks.unchecked_small_scalar_mul_parallelized(&ct, scalar);
 
         // decryption of ct_res
@@ -696,8 +695,10 @@ fn integer_default_small_scalar_mul(param: Parameters) {
         clear_res = clear * scalar;
         for _ in 0..NB_TEST_SMALLER {
             // scalar multiplication
+            let tmp = sks.small_scalar_mul_parallelized(&ct_res, scalar);
             ct_res = sks.small_scalar_mul_parallelized(&ct_res, scalar);
             assert!(ct_res.block_carries_are_empty());
+            assert_eq!(tmp, ct_res);
             clear_res *= scalar;
         }
 
@@ -758,7 +759,9 @@ fn integer_default_scalar_mul(param: Parameters) {
 
         // scalar mul
         let ct_res = sks.scalar_mul_parallelized(&ct, scalar);
+        let tmp = sks.scalar_mul_parallelized(&ct, scalar);
         assert!(ct_res.block_carries_are_empty());
+        assert_eq!(ct_res, tmp);
 
         // decryption of ct_res
         let dec_res: u64 = cks.decrypt(&ct_res);
@@ -790,7 +793,6 @@ fn integer_unchecked_scalar_left_shift(param: Parameters) {
         // encryption of an integer
         let ct = cks.encrypt(clear);
 
-        // add the two ciphertexts
         let ct_res = sks.unchecked_scalar_left_shift_parallelized(&ct, scalar);
 
         // decryption of ct_res
@@ -823,9 +825,10 @@ fn integer_default_scalar_left_shift(param: Parameters) {
         // encryption of an integer
         let ct = cks.encrypt(clear);
 
-        // add the two ciphertexts
         let ct_res = sks.scalar_left_shift_parallelized(&ct, scalar);
+        let tmp = sks.scalar_left_shift_parallelized(&ct, scalar);
         assert!(ct_res.block_carries_are_empty());
+        assert_eq!(ct_res, tmp);
 
         // decryption of ct_res
         let dec_res: u64 = cks.decrypt(&ct_res);
@@ -857,7 +860,6 @@ fn integer_unchecked_scalar_right_shift(param: Parameters) {
         // encryption of an integer
         let ct = cks.encrypt(clear);
 
-        // add the two ciphertexts
         let ct_res = sks.unchecked_scalar_right_shift_parallelized(&ct, scalar);
 
         // decryption of ct_res
@@ -890,9 +892,10 @@ fn integer_default_scalar_right_shift(param: Parameters) {
         // encryption of an integer
         let ct = cks.encrypt(clear);
 
-        // add the two ciphertexts
         let ct_res = sks.scalar_right_shift_parallelized(&ct, scalar);
+        let tmp = sks.scalar_right_shift_parallelized(&ct, scalar);
         assert!(ct_res.block_carries_are_empty());
+        assert_eq!(ct_res, tmp);
 
         // decryption of ct_res
         let dec_res: u64 = cks.decrypt(&ct_res);
@@ -950,11 +953,13 @@ fn integer_default_neg(param: Parameters) {
         let ctxt = cks.encrypt(clear);
 
         // Negates the ctxt
-        let ct_tmp = sks.neg_parallelized(&ctxt);
-        assert!(ct_tmp.block_carries_are_empty());
+        let ct_res = sks.neg_parallelized(&ctxt);
+        let tmp = sks.neg_parallelized(&ctxt);
+        assert!(ct_res.block_carries_are_empty());
+        assert_eq!(ct_res, tmp);
 
         // Decrypt the result
-        let dec: u64 = cks.decrypt(&ct_tmp);
+        let dec: u64 = cks.decrypt(&ct_res);
 
         // Check the correctness
         let clear_result = clear.wrapping_neg() % modulus;
@@ -1022,8 +1027,10 @@ fn integer_default_sub(param: Parameters) {
 
         //subtract multiple times to raise the degree
         for _ in 0..NB_TEST_SMALLER {
+            let tmp = sks.sub_parallelized(&res, &ctxt_2);
             res = sks.sub_parallelized(&res, &ctxt_2);
             assert!(res.block_carries_are_empty());
+            assert_eq!(res, tmp);
             clear = (clear.wrapping_sub(clear2)) % modulus;
             // println!("clear = {}, clear2 = {}", clear, cks.decrypt(&res));
         }
@@ -1057,7 +1064,6 @@ fn integer_unchecked_block_mul(param: Parameters) {
         // encryption of an integer
         let ct_one = cks.encrypt_one_block(clear_1);
 
-        // add the two ciphertexts
         let ct_res = sks.unchecked_block_mul_parallelized(&ct_zero, &ct_one, 0);
 
         // decryption of ct_res
@@ -1133,8 +1139,10 @@ fn integer_default_block_mul(param: Parameters) {
         res = sks.block_mul_parallelized(&res, &ctxt_2, 0);
         assert!(res.block_carries_are_empty());
         for _ in 0..5 {
+            let tmp = sks.block_mul_parallelized(&res, &ctxt_2, 0);
             res = sks.block_mul_parallelized(&res, &ctxt_2, 0);
             assert!(res.block_carries_are_empty());
+            assert_eq!(res, tmp);
             clear = (clear * clear2) % modulus;
         }
         let dec: u64 = cks.decrypt(&res);
@@ -1211,8 +1219,10 @@ fn integer_default_mul(param: Parameters) {
         res = sks.mul_parallelized(&res, &ctxt_2);
         assert!(res.block_carries_are_empty());
         for _ in 0..5 {
+            let tmp = sks.mul_parallelized(&res, &ctxt_2);
             res = sks.mul_parallelized(&res, &ctxt_2);
             assert!(res.block_carries_are_empty());
+            assert_eq!(res, tmp);
             clear = (clear * clear2) % modulus;
         }
         let dec: u64 = cks.decrypt(&res);
@@ -1245,7 +1255,6 @@ fn integer_smart_scalar_add(param: Parameters) {
         // encryption of an integer
         let mut ctxt_0 = cks.encrypt(clear_0);
 
-        // add the two ciphertexts
         let mut ct_res = sks.smart_scalar_add_parallelized(&mut ctxt_0, clear_1);
 
         clear = (clear_0 + clear_1) % modulus;
@@ -1287,7 +1296,6 @@ fn integer_default_scalar_add(param: Parameters) {
         // encryption of an integer
         let ctxt_0 = cks.encrypt(clear_0);
 
-        // add the two ciphertexts
         let mut ct_res = sks.scalar_add_parallelized(&ctxt_0, clear_1);
         assert!(ct_res.block_carries_are_empty());
 
@@ -1296,8 +1304,10 @@ fn integer_default_scalar_add(param: Parameters) {
         // println!("clear_0 = {}, clear_1 = {}", clear_0, clear_1);
         //add multiple times to raise the degree
         for _ in 0..NB_TEST_SMALLER {
+            let tmp = sks.scalar_add_parallelized(&ct_res, clear_1);
             ct_res = sks.scalar_add_parallelized(&ct_res, clear_1);
             assert!(ct_res.block_carries_are_empty());
+            assert_eq!(ct_res, tmp);
             clear = (clear + clear_1) % modulus;
 
             // decryption of ct_res
@@ -1331,7 +1341,6 @@ fn integer_smart_scalar_sub(param: Parameters) {
         // encryption of an integer
         let mut ctxt_0 = cks.encrypt(clear_0);
 
-        // add the two ciphertexts
         let mut ct_res = sks.smart_scalar_sub_parallelized(&mut ctxt_0, clear_1);
 
         clear = (clear_0 - clear_1) % modulus;
@@ -1373,7 +1382,6 @@ fn integer_default_scalar_sub(param: Parameters) {
         // encryption of an integer
         let ctxt_0 = cks.encrypt(clear_0);
 
-        // add the two ciphertexts
         let mut ct_res = sks.scalar_sub_parallelized(&ctxt_0, clear_1);
         assert!(ct_res.block_carries_are_empty());
 
@@ -1382,8 +1390,10 @@ fn integer_default_scalar_sub(param: Parameters) {
         // println!("clear_0 = {}, clear_1 = {}", clear_0, clear_1);
         //add multiple times to raise the degree
         for _ in 0..NB_TEST_SMALLER {
+            let tmp = sks.scalar_sub_parallelized(&ct_res, clear_1);
             ct_res = sks.scalar_sub_parallelized(&ct_res, clear_1);
             assert!(ct_res.block_carries_are_empty());
+            assert_eq!(ct_res, tmp);
             clear = (clear.wrapping_sub(clear_1)) % modulus;
 
             // decryption of ct_res

tfhe/src/lib.rs

@@ -1,4 +1,4 @@
-//! Welcome to the TFHR-rs API documentation!
+//! Welcome to the TFHE-rs API documentation!
 //!
 //! TFHE-rs is a fully homomorphic encryption (FHE) library that implements Zama's variant of TFHE.
 

tfhe/src/shortint/ciphertext/mod.rs

@@ -35,7 +35,7 @@ pub trait PBSOrderMarker: seal::Sealed + Debug + Clone + Copy + Send + Sync {
     fn pbs_order() -> PBSOrder;
 }
 
-#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
 pub struct KeyswitchBootstrap;
 
 impl PBSOrderMarker for KeyswitchBootstrap {
@@ -118,7 +118,7 @@ impl Degree {
     }
 }
 
-#[derive(Clone)]
+#[derive(Clone, Debug, PartialEq, Eq)]
 #[must_use]
 pub struct CiphertextBase<OpOrder: PBSOrderMarker> {
     pub ct: LweCiphertextOwned<u64>,