From bbacc9dcce25eb99dac4e1fa97dcb15390d7097e Mon Sep 17 00:00:00 2001 From: G <28568419+s1fr0@users.noreply.github.com> Date: Wed, 28 Sep 2022 11:33:14 +0200 Subject: [PATCH] Add `utils` module (#53) * refactor(rln/zerokit): move poseidon to separate utils crate * refactor(rln/zerokit): move merkle tree to utils crate * refactor(rln/zerokit): move poseidon to separate utils crate * fix(utils/rln): fmt & conflict resolve * feat(utils): add parallel feature --- Cargo.toml | 1 + rln/Cargo.toml | 7 +- rln/src/lib.rs | 2 - rln/src/poseidon_hash.rs | 154 +----------------- rln/src/poseidon_tree.rs | 2 +- utils/Cargo.toml | 18 ++ utils/src/lib.rs | 5 + .../src/merkle_tree}/merkle_tree.rs | 2 + utils/src/merkle_tree/mod.rs | 2 + utils/src/poseidon/mod.rs | 5 + .../src/poseidon}/poseidon_constants.rs | 31 +++- utils/src/poseidon/poseidon_hash.rs | 142 ++++++++++++++++ 12 files changed, 210 insertions(+), 161 deletions(-) create mode 100644 utils/Cargo.toml create mode 100644 utils/src/lib.rs rename {rln/src => utils/src/merkle_tree}/merkle_tree.rs (99%) create mode 100644 utils/src/merkle_tree/mod.rs create mode 100644 utils/src/poseidon/mod.rs rename {rln/src => utils/src/poseidon}/poseidon_constants.rs (99%) create mode 100644 utils/src/poseidon/poseidon_hash.rs diff --git a/Cargo.toml b/Cargo.toml index 5b96cb0..65abcd0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,4 +4,5 @@ members = [ "semaphore", "rln", "rln-wasm", + "utils", ] diff --git a/rln/Cargo.toml b/rln/Cargo.toml index 632dab5..133875c 100644 --- a/rln/Cargo.toml +++ b/rln/Cargo.toml @@ -34,16 +34,13 @@ num-traits = "0.2.11" once_cell = "1.14.0" rand = "0.8" tiny-keccak = { version = "2.0.2", features = ["keccak"] } +utils = { path = "../utils/", default-features = false } # serialization serde_json = "1.0.48" -[dev-dependencies] - -hex-literal = "0.3.4" - [features] default = ["parallel", "wasmer/sys-default"] fullmerkletree = [] -parallel = ["ark-ec/parallel", "ark-ff/parallel", "ark-std/parallel", "ark-groth16/parallel"] +parallel = ["ark-ec/parallel", "ark-ff/parallel", "ark-std/parallel", "ark-groth16/parallel", "utils/parallel"] wasm = ["wasmer/js", "wasmer/std"] diff --git a/rln/src/lib.rs b/rln/src/lib.rs index 7598b07..d942e41 100644 --- a/rln/src/lib.rs +++ b/rln/src/lib.rs @@ -1,8 +1,6 @@ #![allow(dead_code)] pub mod circuit; -pub mod merkle_tree; -pub mod poseidon_constants; pub mod poseidon_hash; pub mod poseidon_tree; pub mod protocol; diff --git a/rln/src/poseidon_hash.rs b/rln/src/poseidon_hash.rs index 450fde0..ff3febb 100644 --- a/rln/src/poseidon_hash.rs +++ b/rln/src/poseidon_hash.rs @@ -1,14 +1,12 @@ -// This crate implements the Poseidon hash algorithm https://eprint.iacr.org/2019/458.pdf +// This crate instantiate the Poseidon hash algorithm -// Implementation partially taken from https://github.com/arnaucube/poseidon-rs/blob/233027d6075a637c29ad84a8a44f5653b81f0410/src/lib.rs -// and adapted to work over arkworks field traits and custom data structures +use crate::circuit::Fr; +use once_cell::sync::Lazy; +use utils::poseidon::Poseidon; -use crate::poseidon_constants::find_poseidon_ark_and_mds; -use ark_ff::{FpParameters, PrimeField}; - -// These indexed constants hardcodes the round parameters tuple (t, RF, RN) from the paper for the Bn254 scalar field -// SKIP_MATRICES is the index of the randomly generated secure MDS matrix. See security note in the poseidon_constants crate on this. -// TODO: generate in-code such parameters +// These indexed constants hardcodes the supported round parameters tuples (t, RF, RN, SKIP_MATRICES) for the Bn254 scalar field +// SKIP_MATRICES is the index of the randomly generated secure MDS matrix. See security note in the zerokit_utils::poseidon::poseidon_constants crate on this. +// TODO: generate these parameters pub const ROUND_PARAMS: [(usize, usize, usize, usize); 8] = [ (2, 8, 56, 0), (3, 8, 57, 0), @@ -20,145 +18,7 @@ pub const ROUND_PARAMS: [(usize, usize, usize, usize); 8] = [ (9, 8, 63, 0), ]; -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct RoundParamenters { - pub t: usize, - pub n_rounds_f: usize, - pub n_rounds_p: usize, - pub skip_matrices: usize, - pub c: Vec, - pub m: Vec>, -} - -pub struct Poseidon { - round_params: Vec>, -} -impl Poseidon { - // Loads round parameters and generates round constants - // poseidon_params is a vector containing tuples (t, RF, RP, skip_matrices) - // where: t is the rate (input lenght + 1), RF is the number of full rounds, RP is the number of partial rounds - // and skip_matrices is a (temporary) parameter used to generate secure MDS matrices (see comments in the description of find_poseidon_ark_and_mds) - // TODO: implement automatic generation of round parameters - pub fn from(poseidon_params: &[(usize, usize, usize, usize)]) -> Self { - let mut read_params = Vec::>::new(); - - for i in 0..poseidon_params.len() { - let (t, n_rounds_f, n_rounds_p, skip_matrices) = poseidon_params[i]; - let (ark, mds) = find_poseidon_ark_and_mds::( - 1, // is_field = 1 - 0, // is_sbox_inverse = 0 - F::Params::MODULUS_BITS as u64, - t, - n_rounds_f as u64, - n_rounds_p as u64, - skip_matrices, - ); - let rp = RoundParamenters { - t: t, - n_rounds_p: n_rounds_p, - n_rounds_f: n_rounds_f, - skip_matrices: skip_matrices, - c: ark, - m: mds, - }; - read_params.push(rp); - } - - Poseidon { - round_params: read_params, - } - } - - pub fn get_parameters(&self) -> Vec> { - self.round_params.clone() - } - - pub fn ark(&self, state: &mut [F], c: &[F], it: usize) { - for i in 0..state.len() { - state[i] += c[it + i]; - } - } - - pub fn sbox(&self, n_rounds_f: usize, n_rounds_p: usize, state: &mut [F], i: usize) { - if (i < n_rounds_f / 2) || (i >= n_rounds_f / 2 + n_rounds_p) { - for j in 0..state.len() { - let aux = state[j]; - state[j] *= state[j]; - state[j] *= state[j]; - state[j] *= aux; - } - } else { - let aux = state[0]; - state[0] *= state[0]; - state[0] *= state[0]; - state[0] *= aux; - } - } - - pub fn mix(&self, state: &[F], m: &[Vec]) -> Vec { - let mut new_state: Vec = Vec::new(); - for i in 0..state.len() { - new_state.push(F::zero()); - for j in 0..state.len() { - let mut mij = m[i][j]; - mij *= state[j]; - new_state[i] += mij; - } - } - new_state.clone() - } - - pub fn hash(&self, inp: Vec) -> Result { - // Note that the rate t becomes input lenght + 1, hence for lenght N we pick parameters with T = N + 1 - let t = inp.len() + 1; - - // We seek the index (Poseidon's round_params is an ordered vector) for the parameters corresponding to t - let param_index = self.round_params.iter().position(|el| el.t == t); - - if inp.is_empty() || param_index.is_none() { - return Err("No parameters found for inputs length".to_string()); - } - - let param_index = param_index.unwrap(); - - let mut state = vec![F::zero(); t]; - state[1..].clone_from_slice(&inp); - - for i in 0..(self.round_params[param_index].n_rounds_f - + self.round_params[param_index].n_rounds_p) - { - self.ark( - &mut state, - &self.round_params[param_index].c, - (i as usize) * self.round_params[param_index].t, - ); - self.sbox( - self.round_params[param_index].n_rounds_f, - self.round_params[param_index].n_rounds_p, - &mut state, - i, - ); - state = self.mix(&state, &self.round_params[param_index].m); - } - - Ok(state[0]) - } -} - -impl Default for Poseidon -where - F: PrimeField, -{ - // Default instantiation has no round constants set. Will return an error when hashing is attempted. - fn default() -> Self { - Self::from(&[]) - } -} - -use crate::circuit::Fr; -use once_cell::sync::Lazy; // Poseidon Hash wrapper over above implementation. Adapted from semaphore-rs poseidon hash wrapper. - static POSEIDON: Lazy> = Lazy::new(|| Poseidon::::from(&ROUND_PARAMS)); pub fn poseidon_hash(input: &[Fr]) -> Fr { diff --git a/rln/src/poseidon_tree.rs b/rln/src/poseidon_tree.rs index ab6c839..410c1ae 100644 --- a/rln/src/poseidon_tree.rs +++ b/rln/src/poseidon_tree.rs @@ -3,9 +3,9 @@ // Implementation inspired by https://github.com/worldcoin/semaphore-rs/blob/d462a4372f1fd9c27610f2acfe4841fab1d396aa/src/poseidon_tree.rs (no differences) use crate::circuit::Fr; -use crate::merkle_tree::*; use crate::poseidon_hash::poseidon_hash; use cfg_if::cfg_if; +use utils::merkle_tree::*; // The zerokit RLN default Merkle tree implementation is the OptimalMerkleTree. // To switch to FullMerkleTree implementation, it is enough to enable the fullmerkletree feature diff --git a/utils/Cargo.toml b/utils/Cargo.toml new file mode 100644 index 0000000..dadd7bd --- /dev/null +++ b/utils/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "utils" +version = "0.1.0" +edition = "2021" + +[dependencies] +ark-ff = { version = "0.3.0", default-features = false, features = ["asm"] } +num-bigint = { version = "0.4.3", default-features = false, features = ["rand"] } + +[dev-dependencies] +ark-bn254 = { version = "0.3.0" } +num-traits = "0.2.11" +hex-literal = "0.3.4" +tiny-keccak = { version = "2.0.2", features = ["keccak"] } + +[features] +default = ["parallel"] +parallel = ["ark-ff/parallel"] \ No newline at end of file diff --git a/utils/src/lib.rs b/utils/src/lib.rs new file mode 100644 index 0000000..94e8694 --- /dev/null +++ b/utils/src/lib.rs @@ -0,0 +1,5 @@ +pub mod poseidon; +pub use self::poseidon::*; + +pub mod merkle_tree; +pub use self::merkle_tree::*; diff --git a/rln/src/merkle_tree.rs b/utils/src/merkle_tree/merkle_tree.rs similarity index 99% rename from rln/src/merkle_tree.rs rename to utils/src/merkle_tree/merkle_tree.rs index 2ba3c9e..f1e18e5 100644 --- a/rln/src/merkle_tree.rs +++ b/utils/src/merkle_tree/merkle_tree.rs @@ -13,6 +13,8 @@ //! * Disk based storage backend (using mmaped files should be easy) //! * Implement serialization for tree and Merkle proof +#![allow(dead_code)] + use std::collections::HashMap; use std::io; use std::{ diff --git a/utils/src/merkle_tree/mod.rs b/utils/src/merkle_tree/mod.rs new file mode 100644 index 0000000..1b9547c --- /dev/null +++ b/utils/src/merkle_tree/mod.rs @@ -0,0 +1,2 @@ +pub mod merkle_tree; +pub use self::merkle_tree::*; diff --git a/utils/src/poseidon/mod.rs b/utils/src/poseidon/mod.rs new file mode 100644 index 0000000..d37721b --- /dev/null +++ b/utils/src/poseidon/mod.rs @@ -0,0 +1,5 @@ +pub mod poseidon_hash; +pub use self::poseidon_hash::*; + +pub mod poseidon_constants; +pub use self::poseidon_constants::*; diff --git a/rln/src/poseidon_constants.rs b/utils/src/poseidon/poseidon_constants.rs similarity index 99% rename from rln/src/poseidon_constants.rs rename to utils/src/poseidon/poseidon_constants.rs index 0eebed9..866a45b 100644 --- a/rln/src/poseidon_constants.rs +++ b/utils/src/poseidon/poseidon_constants.rs @@ -12,7 +12,6 @@ #![allow(dead_code)] use ark_ff::{FpParameters, PrimeField}; -use ark_std::vec::Vec; use num_bigint::BigUint; pub struct PoseidonGrainLFSR { @@ -274,10 +273,10 @@ pub fn find_poseidon_ark_and_mds( #[cfg(test)] mod test { - use super::*; - use crate::circuit::Fr; use crate::poseidon_hash::Poseidon; - use crate::utils::str_to_fr; + use ark_bn254::Fr; + use num_bigint::BigUint; + use num_traits::Num; const ROUND_PARAMS: [(usize, usize, usize, usize); 8] = [ (2, 8, 56, 0), @@ -290,14 +289,34 @@ mod test { (9, 8, 63, 0), ]; + fn str_to_fr(input: &str, radix: u32) -> Fr { + assert!((radix == 10) || (radix == 16)); + + // We remove any quote present and we trim + let single_quote: char = '\"'; + let mut input_clean = input.replace(single_quote, ""); + input_clean = input_clean.trim().to_string(); + + if radix == 10 { + BigUint::from_str_radix(&input_clean, radix) + .unwrap() + .try_into() + .unwrap() + } else { + input_clean = input_clean.replace("0x", ""); + BigUint::from_str_radix(&input_clean, radix) + .unwrap() + .try_into() + .unwrap() + } + } // The following constants were taken from https://github.com/arnaucube/poseidon-rs/blob/233027d6075a637c29ad84a8a44f5653b81f0410/src/constants.rs // Constants were generated from the Poseidon reference implementation as // generate_parameters_grain.sage 1 0 254 T RF RP 0x30644e72e131a029b85045b68181585d2833e84879b9709143e1f593f0000001 // with T in [2, 3, 4, 5, 6, 7, 8, 9], RF in [8, 8, 8, 8, 8, 8, 8, 8] and RP in [56, 57, 56, 60, 60, 63, 64, 63], respectively. // (in implementation, if we want to hash N elements we use parameters for T = N+1) // The constants were generated and are valid only for Bn254 scalar field characteristic, i.e. 0x30644e72e131a029b85045b68181585d2833e84879b9709143e1f593f0000001 - - // The following constants should correspond to the one obtained by instantiating Poseidon with ROUND_PARAMS above (skip_matrices all set to 0) + // They should correspond to the one obtained by instantiating Poseidon with ROUND_PARAMS above (skip_matrices all set to 0) fn constants() -> (Vec>, Vec>>) { let c_str: Vec> = vec![ vec![ diff --git a/utils/src/poseidon/poseidon_hash.rs b/utils/src/poseidon/poseidon_hash.rs new file mode 100644 index 0000000..4c62a41 --- /dev/null +++ b/utils/src/poseidon/poseidon_hash.rs @@ -0,0 +1,142 @@ +// This crate implements the Poseidon hash algorithm https://eprint.iacr.org/2019/458.pdf + +// Implementation partially taken from https://github.com/arnaucube/poseidon-rs/blob/233027d6075a637c29ad84a8a44f5653b81f0410/src/lib.rs +// and adapted to work over arkworks field traits and custom data structures + +use crate::poseidon_constants::find_poseidon_ark_and_mds; +use ark_ff::{FpParameters, PrimeField}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RoundParamenters { + pub t: usize, + pub n_rounds_f: usize, + pub n_rounds_p: usize, + pub skip_matrices: usize, + pub c: Vec, + pub m: Vec>, +} + +pub struct Poseidon { + round_params: Vec>, +} +impl Poseidon { + // Loads round parameters and generates round constants + // poseidon_params is a vector containing tuples (t, RF, RP, skip_matrices) + // where: t is the rate (input lenght + 1), RF is the number of full rounds, RP is the number of partial rounds + // and skip_matrices is a (temporary) parameter used to generate secure MDS matrices (see comments in the description of find_poseidon_ark_and_mds) + // TODO: implement automatic generation of round parameters + pub fn from(poseidon_params: &[(usize, usize, usize, usize)]) -> Self { + let mut read_params = Vec::>::new(); + + for i in 0..poseidon_params.len() { + let (t, n_rounds_f, n_rounds_p, skip_matrices) = poseidon_params[i]; + let (ark, mds) = find_poseidon_ark_and_mds::( + 1, // is_field = 1 + 0, // is_sbox_inverse = 0 + F::Params::MODULUS_BITS as u64, + t, + n_rounds_f as u64, + n_rounds_p as u64, + skip_matrices, + ); + let rp = RoundParamenters { + t: t, + n_rounds_p: n_rounds_p, + n_rounds_f: n_rounds_f, + skip_matrices: skip_matrices, + c: ark, + m: mds, + }; + read_params.push(rp); + } + + Poseidon { + round_params: read_params, + } + } + + pub fn get_parameters(&self) -> Vec> { + self.round_params.clone() + } + + pub fn ark(&self, state: &mut [F], c: &[F], it: usize) { + for i in 0..state.len() { + state[i] += c[it + i]; + } + } + + pub fn sbox(&self, n_rounds_f: usize, n_rounds_p: usize, state: &mut [F], i: usize) { + if (i < n_rounds_f / 2) || (i >= n_rounds_f / 2 + n_rounds_p) { + for j in 0..state.len() { + let aux = state[j]; + state[j] *= state[j]; + state[j] *= state[j]; + state[j] *= aux; + } + } else { + let aux = state[0]; + state[0] *= state[0]; + state[0] *= state[0]; + state[0] *= aux; + } + } + + pub fn mix(&self, state: &[F], m: &[Vec]) -> Vec { + let mut new_state: Vec = Vec::new(); + for i in 0..state.len() { + new_state.push(F::zero()); + for j in 0..state.len() { + let mut mij = m[i][j]; + mij *= state[j]; + new_state[i] += mij; + } + } + new_state.clone() + } + + pub fn hash(&self, inp: Vec) -> Result { + // Note that the rate t becomes input lenght + 1, hence for lenght N we pick parameters with T = N + 1 + let t = inp.len() + 1; + + // We seek the index (Poseidon's round_params is an ordered vector) for the parameters corresponding to t + let param_index = self.round_params.iter().position(|el| el.t == t); + + if inp.is_empty() || param_index.is_none() { + return Err("No parameters found for inputs length".to_string()); + } + + let param_index = param_index.unwrap(); + + let mut state = vec![F::zero(); t]; + state[1..].clone_from_slice(&inp); + + for i in 0..(self.round_params[param_index].n_rounds_f + + self.round_params[param_index].n_rounds_p) + { + self.ark( + &mut state, + &self.round_params[param_index].c, + (i as usize) * self.round_params[param_index].t, + ); + self.sbox( + self.round_params[param_index].n_rounds_f, + self.round_params[param_index].n_rounds_p, + &mut state, + i, + ); + state = self.mix(&state, &self.round_params[param_index].m); + } + + Ok(state[0]) + } +} + +impl Default for Poseidon +where + F: PrimeField, +{ + // Default instantiation has no round constants set. Will return an error when hashing is attempted. + fn default() -> Self { + Self::from(&[]) + } +}