chore: Release

* Expose generate proof with witness ffi

* Force run CI

* Update ci.yml

* Fix wasm with_witness

* Remove forced CI run

.github/workflows/ci.yml

@@ -134,4 +134,4 @@ jobs:
       - uses: boa-dev/criterion-compare-action@v3
         with:
           branchName: ${{ github.base_ref }}
-          cwd: ${{ matrix.crate }}
+          cwd: ${{ matrix.crate }}

Cargo.lock

@@ -2695,7 +2695,7 @@ dependencies = [
 
 [[package]]
 name = "rln"
-version = "0.3.1"
+version = "0.3.6"
 dependencies = [
  "ark-bn254 0.4.0",
  "ark-circom 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -4260,7 +4260,7 @@ dependencies = [
 
 [[package]]
 name = "zerokit_utils"
-version = "0.3.1"
+version = "0.3.2"
 dependencies = [
  "ark-bn254 0.4.0",
  "ark-ff 0.4.1",

Cross.toml

@@ -29,4 +29,7 @@ image = "ghcr.io/cross-rs/mips64-unknown-linux-gnuabi64:latest"
 image = "ghcr.io/cross-rs/mips64el-unknown-linux-gnuabi64:latest"
 
 [target.mipsel-unknown-linux-gnu]
-image = "ghcr.io/cross-rs/mipsel-unknown-linux-gnu:latest"
+image = "ghcr.io/cross-rs/mipsel-unknown-linux-gnu:latest"
+
+[target.aarch64-linux-android]
+image = "ghcr.io/cross-rs/aarch64-linux-android:edge"

rln-cli/src/commands.rs

@@ -15,6 +15,8 @@ pub(crate) enum Commands {
         /// Sets a custom config file
         #[arg(short, long)]
         config: PathBuf,
+        #[arg(short, long)]
+        tree_config_input: PathBuf,
     },
     SetTree {
         tree_height: usize,

rln-cli/src/main.rs

@@ -1,4 +1,3 @@
-use std::io::Cursor;
 use std::{fs::File, io::Read, path::Path};
 
 use clap::Parser;
@@ -35,6 +34,7 @@ fn main() -> Result<()> {
         Some(Commands::NewWithParams {
             tree_height,
             config,
+            tree_config_input,
         }) => {
             let mut resources: Vec<Vec<u8>> = Vec::new();
             for filename in ["rln.wasm", "rln_final.zkey", "verification_key.json"] {
@@ -45,13 +45,13 @@ fn main() -> Result<()> {
                 file.read_exact(&mut buffer)?;
                 resources.push(buffer);
             }
+            let tree_config_input_file = File::open(&tree_config_input)?;
             state.rln = Some(RLN::new_with_params(
                 *tree_height,
                 resources[0].clone(),
                 resources[1].clone(),
                 resources[2].clone(),
-                // TODO: use appropriate tree_config
-                Cursor::new(""),
+                tree_config_input_file,
             )?);
             Ok(())
         }

rln-cli/src/state.rs

@@ -2,7 +2,7 @@ use color_eyre::Result;
 use rln::public::RLN;
 use std::fs::File;
 
-use crate::config::{InnerConfig, Config};
+use crate::config::{Config, InnerConfig};
 
 #[derive(Default)]
 pub(crate) struct State<'a> {

rln/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rln"
-version = "0.3.1"
+version = "0.3.6"
 edition = "2021"
 license = "MIT OR Apache-2.0"
 description = "APIs to manage, compute and verify zkSNARK proofs and RLN primitives"
@@ -42,7 +42,7 @@ once_cell = "=1.17.1"
 rand = "=0.8.5"
 rand_chacha = "=0.3.1"
 tiny-keccak = { version = "=2.0.2", features = ["keccak"] }
-utils = { package = "zerokit_utils", version = "=0.3.1", path = "../utils/", default-features = false }
+utils = { package = "zerokit_utils", version = "=0.3.2", path = "../utils/", default-features = false }
 
 # serialization
 serde_json = "=1.0.96"

rln/src/ffi.rs

@@ -261,6 +261,12 @@ pub extern "C" fn get_leaf(ctx: *mut RLN, index: usize, output_buffer: *mut Buff
     call_with_output_arg!(ctx, get_leaf, output_buffer, index)
 }
 
+#[allow(clippy::not_unsafe_ptr_arg_deref)]
+#[no_mangle]
+pub extern "C" fn leaves_set(ctx: *mut RLN) -> usize {
+    ctx.process().leaves_set()
+}
+
 #[allow(clippy::not_unsafe_ptr_arg_deref)]
 #[no_mangle]
 pub extern "C" fn set_next_leaf(ctx: *mut RLN, input_buffer: *const Buffer) -> bool {
@@ -294,6 +300,22 @@ pub extern "C" fn atomic_operation(
     call!(ctx, atomic_operation, index, leaves_buffer, indices_buffer)
 }
 
+#[allow(clippy::not_unsafe_ptr_arg_deref)]
+#[no_mangle]
+pub extern "C" fn seq_atomic_operation(
+    ctx: *mut RLN,
+    leaves_buffer: *const Buffer,
+    indices_buffer: *const Buffer,
+) -> bool {
+    call!(
+        ctx,
+        atomic_operation,
+        ctx.process().leaves_set(),
+        leaves_buffer,
+        indices_buffer
+    )
+}
+
 #[allow(clippy::not_unsafe_ptr_arg_deref)]
 #[no_mangle]
 pub extern "C" fn get_root(ctx: *const RLN, output_buffer: *mut Buffer) -> bool {
@@ -339,6 +361,21 @@ pub extern "C" fn generate_rln_proof(
     call_with_output_arg!(ctx, generate_rln_proof, output_buffer, input_buffer)
 }
 
+#[allow(clippy::not_unsafe_ptr_arg_deref)]
+#[no_mangle]
+pub extern "C" fn generate_rln_proof_with_witness(
+    ctx: *mut RLN,
+    input_buffer: *const Buffer,
+    output_buffer: *mut Buffer,
+) -> bool {
+    call_with_output_arg!(
+        ctx,
+        generate_rln_proof_with_witness,
+        output_buffer,
+        input_buffer
+    )
+}
+
 #[allow(clippy::not_unsafe_ptr_arg_deref)]
 #[no_mangle]
 pub extern "C" fn verify_rln_proof(

rln/src/pm_tree_adapter.rs

@@ -197,11 +197,13 @@ impl ZerokitMerkleTree for PmTree {
         let mut indices = indices.into_iter().collect::<Vec<_>>();
         indices.sort();
 
-        match (leaves.is_empty(), indices.is_empty()) {
-            (true, true) => Err(Report::msg("no leaves or indices to be removed")),
-            (false, true) => self.set_range_with_leaves(start, leaves),
-            (true, false) => self.remove_indices(indices),
-            (false, false) => self.remove_indices_and_set_leaves(start, leaves, indices),
+        match (leaves.len(), indices.len()) {
+            (0, 0) => Err(Report::msg("no leaves or indices to be removed")),
+            (1, 0) => self.set(start, leaves[0]),
+            (0, 1) => self.delete(indices[0]),
+            (_, 0) => self.set_range_with_leaves(start, leaves),
+            (0, _) => self.remove_indices(&indices),
+            (_, _) => self.remove_indices_and_set_leaves(start, leaves, &indices),
         }
     }
 
@@ -244,7 +246,8 @@ impl ZerokitMerkleTree for PmTree {
         let data = self.tree.db.get(METADATA_KEY)?;
 
         if data.is_none() {
-            return Err(Report::msg("metadata does not exist"));
+            // send empty Metadata
+            return Ok(Vec::new());
         }
         Ok(data.unwrap())
     }
@@ -264,18 +267,11 @@ impl PmTree {
             .map_err(|e| Report::msg(e.to_string()))
     }
 
-    fn remove_indices(&mut self, indices: Vec<usize>) -> Result<()> {
+    fn remove_indices(&mut self, indices: &[usize]) -> Result<()> {
         let start = indices[0];
         let end = indices.last().unwrap() + 1;
 
-        let mut new_leaves: Vec<_> = (start..end)
-            .map(|i| self.tree.get(i))
-            .collect::<Result<_, _>>()?;
-
-        new_leaves
-            .iter_mut()
-            .take(indices.len())
-            .for_each(|leaf| *leaf = PmTreeHasher::default_leaf());
+        let new_leaves = (start..end).map(|_| PmTreeHasher::default_leaf());
 
         self.tree
             .set_range(start, new_leaves)
@@ -286,16 +282,13 @@ impl PmTree {
         &mut self,
         start: usize,
         leaves: Vec<FrOfPmTreeHasher>,
-        indices: Vec<usize>,
+        indices: &[usize],
     ) -> Result<()> {
         let min_index = *indices.first().unwrap();
         let max_index = start + leaves.len();
 
-        // Generated a placeholder with the exact size needed,
-        // Initiated with default values to be overridden throughout the method
         let mut set_values = vec![PmTreeHasher::default_leaf(); max_index - min_index];
 
-        // If the index is not in indices list, keep the original value
         for i in min_index..start {
             if !indices.contains(&i) {
                 let value = self.tree.get(i)?;
@@ -303,7 +296,6 @@ impl PmTree {
             }
         }
 
-        // Insert new leaves after 'start' position
         for (i, &leaf) in leaves.iter().enumerate() {
             set_values[start - min_index + i] = leaf;
         }

rln/src/public.rs

@@ -406,6 +406,10 @@ impl RLN<'_> {
         Ok(())
     }
 
+    pub fn leaves_set(&mut self) -> usize {
+        self.tree.leaves_set()
+    }
+
     /// Sets a leaf value at the next available never-set leaf index.
     ///
     /// This function updates the internal Merkle tree `next_index` value indicating the next available index corresponding to a never-set leaf as `next_index = next_index + 1`.
@@ -716,9 +720,9 @@ impl RLN<'_> {
         mut output_data: W,
     ) -> Result<()> {
         // We read input RLN witness and we serialize_compressed it
-        let mut witness_byte: Vec<u8> = Vec::new();
-        input_data.read_to_end(&mut witness_byte)?;
-        let (rln_witness, _) = proof_inputs_to_rln_witness(&mut self.tree, &witness_byte)?;
+        let mut input_byte: Vec<u8> = Vec::new();
+        input_data.read_to_end(&mut input_byte)?;
+        let (rln_witness, _) = proof_inputs_to_rln_witness(&mut self.tree, &input_byte)?;
         let proof_values = proof_values_from_witness(&rln_witness);
 
         let proof = generate_proof(self.witness_calculator, &self.proving_key, &rln_witness)?;
@@ -731,12 +735,33 @@ impl RLN<'_> {
         Ok(())
     }
 
-    // TODO: this function seems to use redundant witness (as bigint and serialized) and should be refactored
     // Generate RLN Proof using a witness calculated from outside zerokit
     //
     // output_data is  [ proof<128> | root<32> | epoch<32> | share_x<32> | share_y<32> | nullifier<32> | rln_identifier<32> ]
     // we skip it from documentation for now
-    #[doc(hidden)]
+    #[cfg(not(target_arch = "wasm32"))]
+    pub fn generate_rln_proof_with_witness<R: Read, W: Write>(
+        &mut self,
+        mut input_data: R,
+        mut output_data: W,
+    ) -> Result<()> {
+        let mut witness_byte: Vec<u8> = Vec::new();
+        input_data.read_to_end(&mut witness_byte)?;
+        let (rln_witness, _) = deserialize_witness(&witness_byte)?;
+        let proof_values = proof_values_from_witness(&rln_witness);
+
+        let proof = generate_proof(self.witness_calculator, &self.proving_key, &rln_witness)?;
+
+        // Note: we export a serialization of ark-groth16::Proof not semaphore::Proof
+        // This proof is compressed, i.e. 128 bytes long
+        proof.serialize_compressed(&mut output_data)?;
+        output_data.write_all(&serialize_proof_values(&proof_values))?;
+        Ok(())
+    }
+
+    // Different in wasm since witness_calculator is not available
+    // See: https://github.com/vacp2p/zerokit/blob/b903d8d740e0b8b82057bcc5377ddce05ae5676b/rln/src/public.rs#L47-L49
+    #[cfg(target_arch = "wasm32")]
     pub fn generate_rln_proof_with_witness<W: Write>(
         &mut self,
         calculated_witness: Vec<BigInt>,
@@ -1796,13 +1821,10 @@ mod test {
             .collect();
 
         // Generating the proof
+        let mut input_buffer = Cursor::new(serialized_witness);
         let mut output_buffer = Cursor::new(Vec::<u8>::new());
-        rln.generate_rln_proof_with_witness(
-            calculated_witness_vec,
-            serialized_witness,
-            &mut output_buffer,
-        )
-        .unwrap();
+        rln.generate_rln_proof_with_witness(&mut input_buffer, &mut output_buffer)
+            .unwrap();
 
         // output_data is [ proof<128> | share_y<32> | nullifier<32> | root<32> | epoch<32> | share_x<32> | rln_identifier<32> ]
         let mut proof_data = output_buffer.into_inner();
@@ -2086,4 +2108,19 @@ mod test {
 
         assert_eq!(arbitrary_metadata, received_metadata);
     }
+
+    #[test]
+    fn test_empty_metadata() {
+        let tree_height = TEST_TREE_HEIGHT;
+
+        let input_buffer =
+            Cursor::new(json!({ "resources_folder": TEST_RESOURCES_FOLDER }).to_string());
+        let rln = RLN::new(tree_height, input_buffer).unwrap();
+
+        let mut buffer = Cursor::new(Vec::<u8>::new());
+        rln.get_metadata(&mut buffer).unwrap();
+        let received_metadata = buffer.into_inner();
+
+        assert_eq!(received_metadata.len(), 0);
+    }
 }

rln/tests/ffi.rs

@@ -143,6 +143,8 @@ mod test {
         assert!(success, "RLN object creation failed");
         let rln_pointer = unsafe { &mut *rln_pointer.assume_init() };
 
+        assert_eq!(rln_pointer.leaves_set(), 0);
+
         // We generate a vector of random leaves
         let mut leaves: Vec<Fr> = Vec::new();
         let mut rng = thread_rng();
@@ -159,6 +161,7 @@ mod test {
         let input_buffer = &Buffer::from(leaves_ser.as_ref());
         let success = init_tree_with_leaves(rln_pointer, input_buffer);
         assert!(success, "init tree with leaves call failed");
+        assert_eq!(rln_pointer.leaves_set(), no_of_leaves);
 
         // We get the root of the tree obtained adding leaves in batch
         let mut output_buffer = MaybeUninit::<Buffer>::uninit();
@@ -1206,7 +1209,7 @@ mod test {
     }
 
     #[test]
-    fn test_metadata() {
+    fn test_valid_metadata() {
         // We create a RLN instance
         let tree_height = TEST_TREE_HEIGHT;
 
@@ -1232,4 +1235,25 @@ mod test {
 
         assert_eq!(result_data, seed_bytes.to_vec());
     }
+
+    #[test]
+    fn test_empty_metadata() {
+        // We create a RLN instance
+        let tree_height = TEST_TREE_HEIGHT;
+
+        let mut rln_pointer = MaybeUninit::<*mut RLN>::uninit();
+        let input_config = json!({ "resources_folder": TEST_RESOURCES_FOLDER }).to_string();
+        let input_buffer = &Buffer::from(input_config.as_bytes());
+        let success = new(tree_height, input_buffer, rln_pointer.as_mut_ptr());
+        assert!(success, "RLN object creation failed");
+        let rln_pointer = unsafe { &mut *rln_pointer.assume_init() };
+
+        let mut output_buffer = MaybeUninit::<Buffer>::uninit();
+        let success = get_metadata(rln_pointer, output_buffer.as_mut_ptr());
+        assert!(success, "get_metadata call failed");
+
+        let output_buffer = unsafe { output_buffer.assume_init() };
+
+        assert_eq!(output_buffer.len, 0);
+    }
 }

utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "zerokit_utils"
-version = "0.3.1"
+version = "0.3.2"
 edition = "2021"
 license = "MIT OR Apache-2.0"
 description = "Various utilities for Zerokit"

utils/src/pm_tree/sled_adapter.rs

@@ -2,29 +2,51 @@ use pmtree::*;
 
 use sled::Db as Sled;
 use std::collections::HashMap;
+use std::thread;
+use std::time::Duration;
 
 pub struct SledDB(Sled);
 
+impl SledDB {
+    fn new_with_tries(config: <SledDB as Database>::Config, tries: u32) -> PmtreeResult<Self> {
+        // If we've tried more than 10 times, we give up and return an error.
+        if tries >= 10 {
+            return Err(PmtreeErrorKind::DatabaseError(
+                DatabaseErrorKind::CustomError(format!(
+                    "Cannot create database: exceeded maximum retry attempts. {config:#?}"
+                )),
+            ));
+        }
+        match config.open() {
+            Ok(db) => Ok(SledDB(db)),
+            Err(e) if e.to_string().contains("WouldBlock") => {
+                // try till the fd is freed
+                // sleep for 10^tries milliseconds, then recursively try again
+                thread::sleep(Duration::from_millis(10u64.pow(tries)));
+                Self::new_with_tries(config, tries + 1)
+            }
+            Err(e) => {
+                // On any other error, we return immediately.
+                Err(PmtreeErrorKind::DatabaseError(
+                    DatabaseErrorKind::CustomError(format!(
+                        "Cannot create database: {e} {config:#?}"
+                    )),
+                ))
+            }
+        }
+    }
+}
+
 impl Database for SledDB {
     type Config = sled::Config;
 
     fn new(config: Self::Config) -> PmtreeResult<Self> {
-        let db: Sled = match config.open() {
-            Ok(db) => db,
-            Err(e) => {
-                return Err(PmtreeErrorKind::DatabaseError(
-                    DatabaseErrorKind::CustomError(format!(
-                        "Cannot create database: {e} {config:#?}",
-                    )),
-                ))
-            }
-        };
-
-        Ok(SledDB(db))
+        let db = Self::new_with_tries(config, 0)?;
+        Ok(db)
     }
 
     fn load(config: Self::Config) -> PmtreeResult<Self> {
-        let db: Sled = match sled::open(&config.path) {
+        let db = match config.open() {
             Ok(db) => db,
             Err(e) => {
                 return Err(PmtreeErrorKind::DatabaseError(