diff --git a/.github/workflows/csprng_randomness_testing.yml b/.github/workflows/csprng_randomness_testing.yml new file mode 100644 index 000000000..84b9ccb1e --- /dev/null +++ b/.github/workflows/csprng_randomness_testing.yml @@ -0,0 +1,74 @@ +name: CSPRNG randomness testing Workflow + +env: + CARGO_TERM_COLOR: always + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + RUSTFLAGS: "-C target-cpu=native" + +on: + # Allows you to run this workflow manually from the Actions tab as an alternative. + workflow_dispatch: + # All the inputs are provided by Slab + inputs: + instance_id: + description: "AWS instance ID" + type: string + instance_image_id: + description: "AWS instance AMI ID" + type: string + instance_type: + description: "AWS instance product type" + type: string + runner_name: + description: "Action runner name" + type: string + request_id: + description: 'Slab request ID' + type: string + fork_repo: + description: 'Name of forked repo as user/repo' + type: string + fork_git_sha: + description: 'Git SHA to checkout from fork' + type: string + +jobs: + csprng-randomness-teting: + name: CSPRNG randomness testing + concurrency: + group: ${{ github.workflow }}_${{ github.ref }}_${{ inputs.instance_image_id }}_${{ inputs.instance_type }} + cancel-in-progress: true + runs-on: ${{ inputs.runner_name }} + + steps: + - name: Checkout tfhe-rs + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 + with: + repository: ${{ inputs.fork_repo }} + ref: ${{ inputs.fork_git_sha }} + + - name: Set up home + run: | + echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" + + - name: Install latest stable + uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af + with: + toolchain: stable + default: true + + - name: Dieharder randomness test suite + run: | + make dieharder_csprng + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@b24d75fe0e728a4bf9fc42ee217caa686d141ee8 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png + SLACK_MESSAGE: "concrete-csprng randomness check finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} diff --git a/.github/workflows/trigger_aws_tests_on_pr.yml b/.github/workflows/trigger_aws_tests_on_pr.yml index ae2612282..1fa736f98 100644 --- a/.github/workflows/trigger_aws_tests_on_pr.yml +++ b/.github/workflows/trigger_aws_tests_on_pr.yml @@ -32,3 +32,4 @@ jobs: @slab-ci cpu_integer_test @slab-ci cpu_multi_bit_test @slab-ci cpu_wasm_test + @slab-ci csprng_randomness_testing diff --git a/.gitignore b/.gitignore index 043239f0b..fbb3c3ab9 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ target/ # Some of our bench outputs /tfhe/benchmarks_parameters **/*.csv + +# dieharder run log +dieharder_run.log diff --git a/Makefile b/Makefile index 34b40a543..03012ac38 100644 --- a/Makefile +++ b/Makefile @@ -79,6 +79,15 @@ install_node: $(SHELL) -i -c 'nvm install node' || \ ( echo "Unable to install node, unknown error." && exit 1 ) +.PHONY: install_dieharder # Install dieharder for apt distributions or macOS +install_dieharder: + @dieharder -h > /dev/null 2>&1 || \ + if [[ "$(OS)" == "Linux" ]]; then \ + sudo apt update && sudo apt install -y dieharder; \ + elif [[ "$(OS)" == "Darwin" ]]; then\ + brew install dieharder; \ + fi || ( echo "Unable to install dieharder, unknown error." && exit 1 ) + .PHONY: fmt # Format rust code fmt: install_rs_check_toolchain cargo "$(CARGO_RS_CHECK_TOOLCHAIN)" fmt @@ -434,6 +443,10 @@ no_tfhe_typo: no_dbg_log: @./scripts/no_dbg_calls.sh +.PHONY: dieharder_csprng # Run the dieharder test suite on our CSPRNG implementation +dieharder_csprng: install_dieharder build_concrete_csprng + ./scripts/dieharder_test.sh + # # Benchmarks # diff --git a/ci/slab.toml b/ci/slab.toml index 95ec2f239..53a07320c 100644 --- a/ci/slab.toml +++ b/ci/slab.toml @@ -77,3 +77,8 @@ check_run_name = "PBS CPU AWS Benchmarks" workflow = "wasm_client_benchmark.yml" profile = "cpu-small" check_run_name = "WASM Client AWS Benchmarks" + +[command.csprng_randomness_testing] +workflow = "csprng_randomness_testing.yml" +profile = "cpu-small" +check_run_name = "CSPRNG randomness testing" diff --git a/concrete-csprng/Cargo.toml b/concrete-csprng/Cargo.toml index c40f323d3..427281362 100644 --- a/concrete-csprng/Cargo.toml +++ b/concrete-csprng/Cargo.toml @@ -20,6 +20,7 @@ libc = "0.2.133" [dev-dependencies] rand = "0.8.3" criterion = "0.3" +clap = "=4.2.7" [features] parallel = ["rayon"] @@ -45,7 +46,7 @@ path = "benches/benchmark.rs" harness = false required-features = ["seeder_x86_64_rdseed", "generator_x86_64_aesni"] -[[bin]] +[[example]] name = "generate" -path = "src/main.rs" +path = "examples/generate.rs" required-features = ["seeder_unix", "generator_fallback"] diff --git a/concrete-csprng/examples/generate.rs b/concrete-csprng/examples/generate.rs new file mode 100644 index 000000000..6e82076c8 --- /dev/null +++ b/concrete-csprng/examples/generate.rs @@ -0,0 +1,113 @@ +//! This program uses the concrete csprng to generate an infinite stream of random bytes on +//! the program stdout. It can also generate a fixed number of bytes by passing a value along the +//! optional argument `--bytes_total`. For testing purpose. +use clap::{value_parser, Arg, Command}; +#[cfg(feature = "generator_x86_64_aesni")] +use concrete_csprng::generators::AesniRandomGenerator as ActivatedRandomGenerator; +#[cfg(feature = "generator_aarch64_aes")] +use concrete_csprng::generators::NeonAesRandomGenerator as ActivatedRandomGenerator; +#[cfg(all( + not(feature = "generator_x86_64_aesni"), + not(feature = "generator_aarch64_aes"), + feature = "generator_fallback" +))] +use concrete_csprng::generators::SoftwareRandomGenerator as ActivatedRandomGenerator; + +use concrete_csprng::generators::RandomGenerator; + +#[cfg(target_os = "macos")] +use concrete_csprng::seeders::AppleSecureEnclaveSeeder as ActivatedSeeder; +#[cfg(all(not(target_os = "macos"), feature = "seeder_x86_64_rdseed"))] +use concrete_csprng::seeders::RdseedSeeder as ActivatedSeeder; +#[cfg(all( + not(target_os = "macos"), + not(feature = "seeder_x86_64_rdseed"), + feature = "seeder_unix" +))] +use concrete_csprng::seeders::UnixSeeder as ActivatedSeeder; + +use concrete_csprng::seeders::Seeder; + +use std::io::prelude::*; +use std::io::{stdout, StdoutLock}; + +fn write_bytes( + buffer: &mut [u8], + generator: &mut ActivatedRandomGenerator, + stdout: &mut StdoutLock<'_>, +) -> std::io::Result<()> { + buffer.iter_mut().zip(generator).for_each(|(b, g)| *b = g); + stdout.write_all(buffer) +} + +fn infinite_bytes_generation( + buffer: &mut [u8], + generator: &mut ActivatedRandomGenerator, + stdout: &mut StdoutLock<'_>, +) { + while write_bytes(buffer, generator, stdout).is_ok() {} +} + +fn bytes_generation( + bytes_total: usize, + buffer: &mut [u8], + generator: &mut ActivatedRandomGenerator, + stdout: &mut StdoutLock<'_>, +) { + let quotient = bytes_total / buffer.len(); + let remaining = bytes_total % buffer.len(); + + for _ in 0..quotient { + write_bytes(buffer, generator, stdout).unwrap(); + } + + write_bytes(&mut buffer[0..remaining], generator, stdout).unwrap() +} + +pub fn main() { + let matches = Command::new( + "Generate a stream of random numbers, specify no flags for infinite generation", + ) + .arg( + Arg::new("bytes_total") + .short('b') + .long("bytes_total") + .value_parser(value_parser!(usize)) + .help("Total number of bytes that has to be generated"), + ) + .get_matches(); + + // Ugly hack to be able to use UnixSeeder + #[cfg(all( + not(target_os = "macos"), + not(feature = "seeder_x86_64_rdseed"), + feature = "seeder_unix" + ))] + let new_seeder = || ActivatedSeeder::new(0); + #[cfg(not(all( + not(target_os = "macos"), + not(feature = "seeder_x86_64_rdseed"), + feature = "seeder_unix" + )))] + let new_seeder = || ActivatedSeeder; + + let mut seeder = new_seeder(); + let seed = seeder.seed(); + // Don't print on std out + eprintln!("seed={seed:?}"); + let mut generator = ActivatedRandomGenerator::new(seed); + let stdout = stdout(); + let mut buffer = [0u8; 16]; + + // lock stdout as there is a single thread running + let mut stdout = stdout.lock(); + + match matches.get_one::("bytes_total") { + Some(&total) => { + bytes_generation(total, &mut buffer, &mut generator, &mut stdout); + } + None => { + infinite_bytes_generation(&mut buffer, &mut generator, &mut stdout); + } + }; +} diff --git a/concrete-csprng/src/main.rs b/concrete-csprng/src/main.rs deleted file mode 100644 index 343d1bc39..000000000 --- a/concrete-csprng/src/main.rs +++ /dev/null @@ -1,58 +0,0 @@ -//! This program uses the concrete csprng to generate an infinite stream of random bytes on -//! the program stdout. For testing purpose. -#[cfg(feature = "generator_x86_64_aesni")] -use concrete_csprng::generators::AesniRandomGenerator as ActivatedRandomGenerator; -#[cfg(feature = "generator_aarch64_aes")] -use concrete_csprng::generators::NeonAesRandomGenerator as ActivatedRandomGenerator; -#[cfg(all( - not(feature = "generator_x86_64_aesni"), - not(feature = "generator_aarch64_aes"), - feature = "generator_fallback" -))] -use concrete_csprng::generators::SoftwareRandomGenerator as ActivatedRandomGenerator; - -use concrete_csprng::generators::RandomGenerator; - -#[cfg(target_os = "macos")] -use concrete_csprng::seeders::AppleSecureEnclaveSeeder as ActivatedSeeder; -#[cfg(all(not(target_os = "macos"), feature = "seeder_x86_64_rdseed"))] -use concrete_csprng::seeders::RdseedSeeder as ActivatedSeeder; -#[cfg(all( - not(target_os = "macos"), - not(feature = "seeder_x86_64_rdseed"), - feature = "seeder_unix" -))] -use concrete_csprng::seeders::UnixSeeder as ActivatedSeeder; - -use concrete_csprng::seeders::Seeder; - -use std::io::prelude::*; -use std::io::stdout; - -pub fn main() { - // Ugly hack to be able to use UnixSeeder - #[cfg(all( - not(target_os = "macos"), - not(feature = "seeder_x86_64_rdseed"), - feature = "seeder_unix" - ))] - let new_seeder = || ActivatedSeeder::new(0); - #[cfg(not(all( - not(target_os = "macos"), - not(feature = "seeder_x86_64_rdseed"), - feature = "seeder_unix" - )))] - let new_seeder = || ActivatedSeeder; - - let mut seeder = new_seeder(); - let mut generator = ActivatedRandomGenerator::new(seeder.seed()); - let mut stdout = stdout(); - let mut buffer = [0u8; 16]; - loop { - buffer - .iter_mut() - .zip(&mut generator) - .for_each(|(b, g)| *b = g); - stdout.write_all(&buffer).unwrap(); - } -} diff --git a/scripts/dieharder_test.sh b/scripts/dieharder_test.sh new file mode 100755 index 000000000..c8bbb28fa --- /dev/null +++ b/scripts/dieharder_test.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +# dieharder does not support running a subset of its tests, so we'll check which ones are not good +# and ignore the output from those tests in the final log + +set -e + +DIEHARDER_RUN_LOG_FILE="dieharder_run.log" + +bad_tests="$(dieharder -l | \ +# select lines with the -d +grep -w '\-d' | \ +# forget about the good tests +grep -v -i 'good' | \ +# get the test id +cut -d ' ' -f 4 | \ +# nice formatting +xargs)" + + +bad_test_filter="" +for bad_test in ${bad_tests}; do + bad_test_filter="${bad_test_filter:+${bad_test_filter}|}$(dieharder -d "${bad_test}" -t 1 -p 1 -D test_name | xargs)" +done + +echo "The following tests will be ignored as they are marked as either 'suspect' or 'do not use': " +echo "" +echo "${bad_test_filter}" +echo "" + +# by default we may have no pv just forward the input +pv="cat" +if which pv > /dev/null; then + pv="pv -t -a -b" +fi + +rm -f "${DIEHARDER_RUN_LOG_FILE}" + +# ignore potential errors and parse the log afterwards +set +e + +# We are writing in both cases +# shellcheck disable=SC2094 +./target/release/examples/generate 2>"${DIEHARDER_RUN_LOG_FILE}" | \ +$pv | \ +# -a: all tests +# -g 200: get random bytes from input +# -Y 1: disambiguate results, i.e. if a weak result appear check if it's a random failure/weakness +# -k 2: better maths formulas to determine some test statistics +dieharder -a -g 200 -Y 1 -k 2 | \ +tee -a "${DIEHARDER_RUN_LOG_FILE}" +set -e + +printf "\n\n" + +cat "${DIEHARDER_RUN_LOG_FILE}" + +if ! grep -q -i "failed" < "${DIEHARDER_RUN_LOG_FILE}"; then + echo "All tests passed!" + exit 0 +fi + +printf "\n\n" + +failed_tests="$(grep -i "failed" < "${DIEHARDER_RUN_LOG_FILE}")" +true_failed_test="$(grep -i "failed" < "${DIEHARDER_RUN_LOG_FILE}" | { grep -v -E "${bad_test_filter}" || true; } | sed -z '$ s/\n$//')" + +if [[ "${true_failed_test}" == "" ]]; then + echo "There were test failures, but the tests were either marked as 'suspect' or 'do not use'" + echo "${failed_tests}" + exit 0 +fi + +echo "The following tests failed:" +echo "${true_failed_test}" + +exit 1