chore(csprng): add dieharder test suite

.github/workflows/csprng_randomness_testing.yml

@@ -0,0 +1,74 @@
+name: CSPRNG randomness testing Workflow
+
+env:
+  CARGO_TERM_COLOR: always
+  ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+  RUSTFLAGS: "-C target-cpu=native"
+
+on:
+  # Allows you to run this workflow manually from the Actions tab as an alternative.
+  workflow_dispatch:
+    # All the inputs are provided by Slab
+    inputs:
+      instance_id:
+        description: "AWS instance ID"
+        type: string
+      instance_image_id:
+        description: "AWS instance AMI ID"
+        type: string
+      instance_type:
+        description: "AWS instance product type"
+        type: string
+      runner_name:
+        description: "Action runner name"
+        type: string
+      request_id:
+        description: 'Slab request ID'
+        type: string
+      fork_repo:
+        description: 'Name of forked repo as user/repo'
+        type: string
+      fork_git_sha:
+        description: 'Git SHA to checkout from fork'
+        type: string
+
+jobs:
+  csprng-randomness-teting:
+    name: CSPRNG randomness testing
+    concurrency:
+      group: ${{ github.workflow }}_${{ github.ref }}_${{ inputs.instance_image_id }}_${{ inputs.instance_type }}
+      cancel-in-progress: true
+    runs-on: ${{ inputs.runner_name }}
+
+    steps:
+      - name: Checkout tfhe-rs
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744
+        with:
+          repository: ${{ inputs.fork_repo }}
+          ref: ${{ inputs.fork_git_sha }}
+
+      - name: Set up home
+        run: |
+          echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}"
+
+      - name: Install latest stable
+        uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af
+        with:
+          toolchain: stable
+          default: true
+
+      - name: Dieharder randomness test suite
+        run: |
+          make dieharder_csprng
+
+      - name: Slack Notification
+        if: ${{ failure() }}
+        continue-on-error: true
+        uses: rtCamp/action-slack-notify@b24d75fe0e728a4bf9fc42ee217caa686d141ee8
+        env:
+          SLACK_COLOR: ${{ job.status }}
+          SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
+          SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
+          SLACK_MESSAGE: "concrete-csprng randomness check finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
+          SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}

.github/workflows/trigger_aws_tests_on_pr.yml

@@ -32,3 +32,4 @@ jobs:
             @slab-ci cpu_integer_test
             @slab-ci cpu_multi_bit_test
             @slab-ci cpu_wasm_test
+            @slab-ci csprng_randomness_testing

.gitignore

@@ -13,3 +13,6 @@ target/
 # Some of our bench outputs
 /tfhe/benchmarks_parameters
 **/*.csv
+
+# dieharder run log
+dieharder_run.log

Makefile

@@ -79,6 +79,15 @@ install_node:
 	$(SHELL) -i -c 'nvm install node' || \
 	( echo "Unable to install node, unknown error." && exit 1 )
 
+.PHONY: install_dieharder # Install dieharder for apt distributions or macOS
+install_dieharder:
+	@dieharder -h > /dev/null 2>&1 || \
+	if [[ "$(OS)" == "Linux" ]]; then \
+		sudo apt update && sudo apt install -y dieharder; \
+	elif [[ "$(OS)" == "Darwin" ]]; then\
+		brew install dieharder; \
+	fi || ( echo "Unable to install dieharder, unknown error." && exit 1 )
+
 .PHONY: fmt # Format rust code
 fmt: install_rs_check_toolchain
 	cargo "$(CARGO_RS_CHECK_TOOLCHAIN)" fmt
@@ -434,6 +443,10 @@ no_tfhe_typo:
 no_dbg_log:
 	@./scripts/no_dbg_calls.sh
 
+.PHONY: dieharder_csprng # Run the dieharder test suite on our CSPRNG implementation
+dieharder_csprng: install_dieharder build_concrete_csprng
+	./scripts/dieharder_test.sh
+
 #
 # Benchmarks
 #

ci/slab.toml

@@ -77,3 +77,8 @@ check_run_name = "PBS CPU AWS Benchmarks"
 workflow = "wasm_client_benchmark.yml"
 profile = "cpu-small"
 check_run_name = "WASM Client AWS Benchmarks"
+
+[command.csprng_randomness_testing]
+workflow = "csprng_randomness_testing.yml"
+profile = "cpu-small"
+check_run_name = "CSPRNG randomness testing"

concrete-csprng/Cargo.toml

@@ -20,6 +20,7 @@ libc = "0.2.133"
 [dev-dependencies]
 rand = "0.8.3"
 criterion = "0.3"
+clap = "=4.2.7"
 
 [features]
 parallel = ["rayon"]
@@ -45,7 +46,7 @@ path = "benches/benchmark.rs"
 harness = false
 required-features = ["seeder_x86_64_rdseed", "generator_x86_64_aesni"]
 
-[[bin]]
+[[example]]
 name = "generate"
-path = "src/main.rs"
+path = "examples/generate.rs"
 required-features = ["seeder_unix", "generator_fallback"]

concrete-csprng/examples/generate.rs

@@ -0,0 +1,113 @@
+//! This program uses the concrete csprng to generate an infinite stream of random bytes on
+//! the program stdout. It can also generate a fixed number of bytes by passing a value along the
+//! optional argument `--bytes_total`. For testing purpose.
+use clap::{value_parser, Arg, Command};
+#[cfg(feature = "generator_x86_64_aesni")]
+use concrete_csprng::generators::AesniRandomGenerator as ActivatedRandomGenerator;
+#[cfg(feature = "generator_aarch64_aes")]
+use concrete_csprng::generators::NeonAesRandomGenerator as ActivatedRandomGenerator;
+#[cfg(all(
+    not(feature = "generator_x86_64_aesni"),
+    not(feature = "generator_aarch64_aes"),
+    feature = "generator_fallback"
+))]
+use concrete_csprng::generators::SoftwareRandomGenerator as ActivatedRandomGenerator;
+
+use concrete_csprng::generators::RandomGenerator;
+
+#[cfg(target_os = "macos")]
+use concrete_csprng::seeders::AppleSecureEnclaveSeeder as ActivatedSeeder;
+#[cfg(all(not(target_os = "macos"), feature = "seeder_x86_64_rdseed"))]
+use concrete_csprng::seeders::RdseedSeeder as ActivatedSeeder;
+#[cfg(all(
+    not(target_os = "macos"),
+    not(feature = "seeder_x86_64_rdseed"),
+    feature = "seeder_unix"
+))]
+use concrete_csprng::seeders::UnixSeeder as ActivatedSeeder;
+
+use concrete_csprng::seeders::Seeder;
+
+use std::io::prelude::*;
+use std::io::{stdout, StdoutLock};
+
+fn write_bytes(
+    buffer: &mut [u8],
+    generator: &mut ActivatedRandomGenerator,
+    stdout: &mut StdoutLock<'_>,
+) -> std::io::Result<()> {
+    buffer.iter_mut().zip(generator).for_each(|(b, g)| *b = g);
+    stdout.write_all(buffer)
+}
+
+fn infinite_bytes_generation(
+    buffer: &mut [u8],
+    generator: &mut ActivatedRandomGenerator,
+    stdout: &mut StdoutLock<'_>,
+) {
+    while write_bytes(buffer, generator, stdout).is_ok() {}
+}
+
+fn bytes_generation(
+    bytes_total: usize,
+    buffer: &mut [u8],
+    generator: &mut ActivatedRandomGenerator,
+    stdout: &mut StdoutLock<'_>,
+) {
+    let quotient = bytes_total / buffer.len();
+    let remaining = bytes_total % buffer.len();
+
+    for _ in 0..quotient {
+        write_bytes(buffer, generator, stdout).unwrap();
+    }
+
+    write_bytes(&mut buffer[0..remaining], generator, stdout).unwrap()
+}
+
+pub fn main() {
+    let matches = Command::new(
+        "Generate a stream of random numbers, specify no flags for infinite generation",
+    )
+    .arg(
+        Arg::new("bytes_total")
+            .short('b')
+            .long("bytes_total")
+            .value_parser(value_parser!(usize))
+            .help("Total number of bytes that has to be generated"),
+    )
+    .get_matches();
+
+    // Ugly hack to be able to use UnixSeeder
+    #[cfg(all(
+        not(target_os = "macos"),
+        not(feature = "seeder_x86_64_rdseed"),
+        feature = "seeder_unix"
+    ))]
+    let new_seeder = || ActivatedSeeder::new(0);
+    #[cfg(not(all(
+        not(target_os = "macos"),
+        not(feature = "seeder_x86_64_rdseed"),
+        feature = "seeder_unix"
+    )))]
+    let new_seeder = || ActivatedSeeder;
+
+    let mut seeder = new_seeder();
+    let seed = seeder.seed();
+    // Don't print on std out
+    eprintln!("seed={seed:?}");
+    let mut generator = ActivatedRandomGenerator::new(seed);
+    let stdout = stdout();
+    let mut buffer = [0u8; 16];
+
+    // lock stdout as there is a single thread running
+    let mut stdout = stdout.lock();
+
+    match matches.get_one::<usize>("bytes_total") {
+        Some(&total) => {
+            bytes_generation(total, &mut buffer, &mut generator, &mut stdout);
+        }
+        None => {
+            infinite_bytes_generation(&mut buffer, &mut generator, &mut stdout);
+        }
+    };
+}

concrete-csprng/src/main.rs

@@ -1,58 +0,0 @@
-//! This program uses the concrete csprng to generate an infinite stream of random bytes on
-//! the program stdout. For testing purpose.
-#[cfg(feature = "generator_x86_64_aesni")]
-use concrete_csprng::generators::AesniRandomGenerator as ActivatedRandomGenerator;
-#[cfg(feature = "generator_aarch64_aes")]
-use concrete_csprng::generators::NeonAesRandomGenerator as ActivatedRandomGenerator;
-#[cfg(all(
-    not(feature = "generator_x86_64_aesni"),
-    not(feature = "generator_aarch64_aes"),
-    feature = "generator_fallback"
-))]
-use concrete_csprng::generators::SoftwareRandomGenerator as ActivatedRandomGenerator;
-
-use concrete_csprng::generators::RandomGenerator;
-
-#[cfg(target_os = "macos")]
-use concrete_csprng::seeders::AppleSecureEnclaveSeeder as ActivatedSeeder;
-#[cfg(all(not(target_os = "macos"), feature = "seeder_x86_64_rdseed"))]
-use concrete_csprng::seeders::RdseedSeeder as ActivatedSeeder;
-#[cfg(all(
-    not(target_os = "macos"),
-    not(feature = "seeder_x86_64_rdseed"),
-    feature = "seeder_unix"
-))]
-use concrete_csprng::seeders::UnixSeeder as ActivatedSeeder;
-
-use concrete_csprng::seeders::Seeder;
-
-use std::io::prelude::*;
-use std::io::stdout;
-
-pub fn main() {
-    // Ugly hack to be able to use UnixSeeder
-    #[cfg(all(
-        not(target_os = "macos"),
-        not(feature = "seeder_x86_64_rdseed"),
-        feature = "seeder_unix"
-    ))]
-    let new_seeder = || ActivatedSeeder::new(0);
-    #[cfg(not(all(
-        not(target_os = "macos"),
-        not(feature = "seeder_x86_64_rdseed"),
-        feature = "seeder_unix"
-    )))]
-    let new_seeder = || ActivatedSeeder;
-
-    let mut seeder = new_seeder();
-    let mut generator = ActivatedRandomGenerator::new(seeder.seed());
-    let mut stdout = stdout();
-    let mut buffer = [0u8; 16];
-    loop {
-        buffer
-            .iter_mut()
-            .zip(&mut generator)
-            .for_each(|(b, g)| *b = g);
-        stdout.write_all(&buffer).unwrap();
-    }
-}

scripts/dieharder_test.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+
+# dieharder does not support running a subset of its tests, so we'll check which ones are not good
+# and ignore the output from those tests in the final log
+
+set -e
+
+DIEHARDER_RUN_LOG_FILE="dieharder_run.log"
+
+bad_tests="$(dieharder -l | \
+# select lines with the -d
+grep -w '\-d' | \
+# forget about the good tests
+grep -v -i 'good' | \
+# get the test id
+cut -d ' ' -f 4 | \
+# nice formatting
+xargs)"
+
+
+bad_test_filter=""
+for bad_test in ${bad_tests}; do
+    bad_test_filter="${bad_test_filter:+${bad_test_filter}|}$(dieharder -d "${bad_test}" -t 1 -p 1 -D test_name | xargs)"
+done
+
+echo "The following tests will be ignored as they are marked as either 'suspect' or 'do not use': "
+echo ""
+echo "${bad_test_filter}"
+echo ""
+
+# by default we may have no pv just forward the input
+pv="cat"
+if which pv > /dev/null; then
+    pv="pv -t -a -b"
+fi
+
+rm -f "${DIEHARDER_RUN_LOG_FILE}"
+
+# ignore potential errors and parse the log afterwards
+set +e
+
+# We are writing in both cases
+# shellcheck disable=SC2094
+./target/release/examples/generate 2>"${DIEHARDER_RUN_LOG_FILE}" | \
+$pv | \
+# -a: all tests
+# -g 200: get random bytes from input
+# -Y 1: disambiguate results, i.e. if a weak result appear check if it's a random failure/weakness
+# -k 2: better maths formulas to determine some test statistics
+dieharder -a -g 200 -Y 1 -k 2 | \
+tee -a "${DIEHARDER_RUN_LOG_FILE}"
+set -e
+
+printf "\n\n"
+
+cat "${DIEHARDER_RUN_LOG_FILE}"
+
+if ! grep -q -i "failed" < "${DIEHARDER_RUN_LOG_FILE}"; then
+    echo "All tests passed!"
+    exit 0
+fi
+
+printf "\n\n"
+
+failed_tests="$(grep -i "failed" < "${DIEHARDER_RUN_LOG_FILE}")"
+true_failed_test="$(grep -i "failed" < "${DIEHARDER_RUN_LOG_FILE}" | { grep -v -E "${bad_test_filter}" || true; } | sed -z '$ s/\n$//')"
+
+if [[ "${true_failed_test}" == "" ]]; then
+    echo "There were test failures, but the tests were either marked as 'suspect' or 'do not use'"
+    echo "${failed_tests}"
+    exit 0
+fi
+
+echo "The following tests failed:"
+echo "${true_failed_test}"
+
+exit 1