Initial code to use rocksdb for pmtree

Clean feature naming with env vars

- Use arrays for feature sets in matrix.
- Add job-level env (FEATURES_CARGO, FEATURES_TAG, TARGET).
- Use FEATURES_TAG for artifact/file names → no more dots/commas.

Example:
`x86_64-unknown-linux-gnu-fullmerkletree.parallel-rln.tar.gz` →
`x86_64-unknown-linux-gnu-fullmerkletree-parallel-rln.tar.gz`

.github/workflows/nightly-release.yml

@@ -10,39 +10,41 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        feature:
-          - "stateless"
-          - "stateless,parallel"
-          - "pmtree-ft"
-          - "pmtree-ft,parallel"
-          - "fullmerkletree"
-          - "fullmerkletree,parallel"
-          - "optimalmerkletree"
-          - "optimalmerkletree,parallel"
-        target:
-          - x86_64-unknown-linux-gnu
-          - aarch64-unknown-linux-gnu
+        features:
+          - ["stateless"]
+          - ["stateless", "parallel"]
+          - ["pmtree-ft"]
+          - ["pmtree-ft", "parallel"]
+          - ["fullmerkletree"]
+          - ["fullmerkletree", "parallel"]
+          - ["optimalmerkletree"]
+          - ["optimalmerkletree", "parallel"]
+        target: [x86_64-unknown-linux-gnu, aarch64-unknown-linux-gnu]
+    env:
+      FEATURES_CARGO: ${{ join(matrix.features, ',') }}
+      FEATURES_TAG: ${{ join(matrix.features, '-') }}
+      TARGET: ${{ matrix.target }}
     steps:
       - name: Checkout sources
         uses: actions/checkout@v4
       - name: Install stable toolchain
         uses: dtolnay/rust-toolchain@stable
         with:
-          target: ${{ matrix.target }}
+          target: ${{ env.TARGET }}
       - uses: Swatinem/rust-cache@v2
       - name: Install dependencies
         run: make installdeps
       - name: Cross build
         run: |
-          cross build --release --target ${{ matrix.target }} --no-default-features --features ${{ matrix.feature }} --workspace --exclude rln-cli
+          cross build --release --target $TARGET --no-default-features --features "$FEATURES_CARGO" --workspace
           mkdir release
-          cp target/${{ matrix.target }}/release/librln* release/
-          tar -czvf ${{ matrix.target }}-${{ matrix.feature }}-rln.tar.gz release/
+          cp target/$TARGET/release/librln* release/
+          tar -czvf $TARGET-$FEATURES_TAG-rln.tar.gz release/
       - name: Upload archive artifact
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.target }}-${{ matrix.feature }}-archive
-          path: ${{ matrix.target }}-${{ matrix.feature }}-rln.tar.gz
+          name: ${{ env.TARGET }}-${{ env.FEATURES_TAG }}-archive
+          path: ${{ env.TARGET }}-${{ env.FEATURES_TAG }}-rln.tar.gz
           retention-days: 2
 
   macos:
@@ -50,39 +52,41 @@ jobs:
     runs-on: macos-latest
     strategy:
       matrix:
-        feature:
-          - "stateless"
-          - "stateless,parallel"
-          - "pmtree-ft"
-          - "pmtree-ft,parallel"
-          - "fullmerkletree"
-          - "fullmerkletree,parallel"
-          - "optimalmerkletree"
-          - "optimalmerkletree,parallel"
-        target:
-          - x86_64-apple-darwin
-          - aarch64-apple-darwin
+        features:
+          - ["stateless"]
+          - ["stateless", "parallel"]
+          - ["pmtree-ft"]
+          - ["pmtree-ft", "parallel"]
+          - ["fullmerkletree"]
+          - ["fullmerkletree", "parallel"]
+          - ["optimalmerkletree"]
+          - ["optimalmerkletree", "parallel"]
+        target: [x86_64-apple-darwin, aarch64-apple-darwin]
+    env:
+      FEATURES_CARGO: ${{ join(matrix.features, ',') }}
+      FEATURES_TAG: ${{ join(matrix.features, '-') }}
+      TARGET: ${{ matrix.target }}
     steps:
       - name: Checkout sources
         uses: actions/checkout@v4
       - name: Install stable toolchain
         uses: dtolnay/rust-toolchain@stable
         with:
-          target: ${{ matrix.target }}
+          target: ${{ env.TARGET }}
       - uses: Swatinem/rust-cache@v2
       - name: Install dependencies
         run: make installdeps
       - name: Cross build
         run: |
-          cross build --release --target ${{ matrix.target }} --no-default-features --features ${{ matrix.feature }} --workspace --exclude rln-cli
+          cross build --release --target $TARGET --no-default-features --features "$FEATURES_CARGO" --workspace
           mkdir release
-          cp target/${{ matrix.target }}/release/librln* release/
-          tar -czvf ${{ matrix.target }}-${{ matrix.feature }}-rln.tar.gz release/
+          cp target/$TARGET/release/librln* release/
+          tar -czvf $TARGET-$FEATURES_TAG-rln.tar.gz release/
       - name: Upload archive artifact
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.target }}-${{ matrix.feature }}-archive
-          path: ${{ matrix.target }}-${{ matrix.feature }}-rln.tar.gz
+          name: ${{ env.TARGET }}-${{ env.FEATURES_TAG }}-archive
+          path: ${{ env.TARGET }}-${{ env.FEATURES_TAG }}-rln.tar.gz
           retention-days: 2
 
   rln-wasm:
@@ -103,19 +107,11 @@ jobs:
       - name: Install nightly toolchain
         uses: dtolnay/rust-toolchain@nightly
         with:
-          targets: wasm32-unknown-unknown
           components: rust-src
+          targets: wasm32-unknown-unknown
       - uses: Swatinem/rust-cache@v2
-        with:
-          key: rln-wasm-${{ matrix.feature }}
       - name: Install dependencies
         run: make installdeps
-      - name: Install wasm-pack
-        run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
-      - name: Install binaryen
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y binaryen
       - name: Build rln-wasm package
         run: |
           if [[ ${{ matrix.feature }} == *parallel* ]]; then
@@ -152,22 +148,9 @@ jobs:
         uses: dtolnay/rust-toolchain@stable
         with:
           targets: wasm32-unknown-unknown
-      - name: Install nightly toolchain
-        uses: dtolnay/rust-toolchain@nightly
-        with:
-          targets: wasm32-unknown-unknown
-          components: rust-src
       - uses: Swatinem/rust-cache@v2
-        with:
-          key: rln-wasm-utils
       - name: Install dependencies
         run: make installdeps
-      - name: Install wasm-pack
-        run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
-      - name: Install binaryen
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y binaryen
       - name: Build rln-wasm-utils package
         run: |
           wasm-pack build --release --target web --scope waku

CONTRIBUTING.md

@@ -105,8 +105,9 @@ docs: update RLN protocol flow documentation
 Use scopes to improve the Changelog:
 
 - `rln` - Core RLN implementation
-- `rln-wasm` - WebAssembly bindings
 - `rln-cli` - Command-line interface
+- `rln-wasm` - WebAssembly bindings
+- `rln-wasm-utils` - WebAssembly utilities
 - `utils` - Cryptographic utilities (Merkle trees, Poseidon hash)
 - `ci` - Continuous integration
 
@@ -139,8 +140,8 @@ Ensure the following commands pass before submitting:
 # Format code
 cargo fmt --all
 
-# Check for common mistakes  
-cargo clippy --all-targets --all-features
+# Check for common mistakes
+cargo clippy --all-targets
 
 # Run all tests
 make test

Cargo.toml

@@ -1,6 +1,6 @@
 [workspace]
-members = ["rln", "rln-cli", "utils"]
-exclude = ["rln-wasm", "rln-wasm-utils"]
+members = ["rln", "utils"]
+exclude = ["rln-cli", "rln-wasm", "rln-wasm-utils"]
 resolver = "2"
 
 # Compilation profile for any non-workspace member.

rln-cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rln-cli"
-version = "0.4.0"
+version = "0.5.0"
 edition = "2021"
 
 [[example]]
@@ -13,8 +13,8 @@ path = "src/examples/stateless.rs"
 required-features = ["stateless"]
 
 [dependencies]
-rln = { path = "../rln", version = "0.8.0", default-features = false }
-zerokit_utils = { path = "../utils", version = "0.6.0", default-features = false }
+rln = { path = "../rln", version = "0.9.0", default-features = false }
+zerokit_utils = { path = "../utils", version = "0.7.0", default-features = false }
 clap = { version = "4.5.41", features = ["cargo", "derive", "env"] }
 color-eyre = "0.6.5"
 serde_json = "1.0.141"

rln-wasm-utils/Cargo.toml

@@ -9,7 +9,7 @@ crate-type = ["cdylib", "rlib"]
 
 [dependencies]
 # TODO: remove this once we have a proper release
-rln = { path = "../rln", default-features = false, features = ["stateless"] }
+rln = { path = "../rln", version = "0.9.0", default-features = false, features = ["stateless"] }
 js-sys = "0.3.77"
 wasm-bindgen = "0.2.100"
 rand = "0.8.5"
@@ -24,7 +24,7 @@ console_error_panic_hook = { version = "0.1.7", optional = true }
 getrandom = { version = "0.2.16", features = ["js"] }
 
 [dev-dependencies]
-wasm-bindgen-test = "0.3.37"
+wasm-bindgen-test = "0.3.50"
 web-sys = { version = "0.3.77", features = ["console"] }
 ark-std = { version = "0.5.0", default-features = false }
 

rln-wasm/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rln-wasm"
-version = "0.2.0"
+version = "0.3.0"
 edition = "2021"
 license = "MIT or Apache2"
 
@@ -8,11 +8,11 @@ license = "MIT or Apache2"
 crate-type = ["cdylib", "rlib"]
 
 [dependencies]
-rln = { path = "../rln", version = "0.8.0", default-features = false, features = [
+rln = { path = "../rln", version = "0.9.0", default-features = false, features = [
     "stateless",
 ] }
 rln-wasm-utils = { path = "../rln-wasm-utils", version = "0.1.0", default-features = false }
-zerokit_utils = { path = "../utils", version = "0.6.0", default-features = false }
+zerokit_utils = { path = "../utils", version = "0.7.0", default-features = false }
 num-bigint = { version = "0.4.6", default-features = false }
 js-sys = "0.3.77"
 wasm-bindgen = "0.2.100"

rln/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rln"
-version = "0.8.0"
+version = "0.9.0"
 edition = "2021"
 license = "MIT OR Apache-2.0"
 description = "APIs to manage, compute and verify zkSNARK proofs and RLN primitives"
@@ -43,7 +43,7 @@ ruint = { version = "1.15.0", features = ["rand", "serde", "ark-ff-04"] }
 tiny-keccak = { version = "2.0.2", features = ["keccak"] }
 zeroize = "1.8"
 tempfile = "3.21.0"
-utils = { package = "zerokit_utils", version = "0.6.0", path = "../utils", default-features = false }
+utils = { package = "zerokit_utils", version = "0.7.0", path = "../utils", default-features = false }
 
 # serialization
 prost = "0.14.1"

rln/src/public.rs

@@ -73,7 +73,7 @@ impl<R: Read> TreeConfigInput for R {
     }
 }
 
-#[cfg(not(feature = "stateless"))]
+#[cfg(feature = "pmtree-ft")]
 impl TreeConfigInput for <PoseidonTree as ZerokitMerkleTree>::Config {
     fn into_tree_config(self) -> Result<<PoseidonTree as ZerokitMerkleTree>::Config, RLNError> {
         Ok(self)

utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "zerokit_utils"
-version = "0.6.0"
+version = "0.7.0"
 edition = "2021"
 license = "MIT OR Apache-2.0"
 description = "Various utilities for Zerokit"
@@ -15,6 +15,7 @@ bench = false
 ark-ff = { version = "0.5.0", default-features = false }
 num-bigint = { version = "0.4.6", default-features = false }
 pmtree = { package = "vacp2p_pmtree", version = "2.0.3", optional = true }
+rocksdb = { git = "https://github.com/tillrohrmann/rust-rocksdb", branch = "issues/836", optional = true }
 sled = "0.34.7"
 serde_json = "1.0.141"
 lazy_static = "1.5.0"
@@ -32,7 +33,8 @@ criterion = { version = "0.7.0", features = ["html_reports"] }
 [features]
 default = []
 parallel = ["ark-ff/parallel"]
-pmtree-ft = ["pmtree"]
+pmtree-ft = ["pmtree", "rocksdb"]
+# pmtree-ft-rocksdb = ["pmtree", "rocksdb"]
 
 [[bench]]
 name = "merkle_tree_benchmark"

utils/src/pm_tree/mod.rs

@@ -1,4 +1,8 @@
 pub mod sled_adapter;
 pub use self::sled_adapter::SledDB;
+
+pub mod rocksdb_adapter;
+pub use self::rocksdb_adapter::RocksDbWrapper;
+
 pub use pmtree;
 pub use sled::{Config, Mode};

utils/src/pm_tree/rocksdb_adapter.rs

@@ -0,0 +1,116 @@
+use std::{
+    collections::HashMap,
+    sync::Arc,
+};
+use std::path::PathBuf;
+use pmtree::{DBKey, Database, DatabaseErrorKind, PmtreeErrorKind, PmtreeResult, TreeErrorKind, Value};
+use rocksdb::{
+    ColumnFamily, ColumnFamilyDescriptor, DB, Options, ReadOptions, WriteBatch, WriteBatchWithIndex,
+};
+
+#[derive(Default)]
+pub struct RocksDbWrapperConfig {
+    options: Options,
+    db_path: PathBuf,
+    cfs: Vec<ColumnFamilyDescriptor>,
+    // TODO: ColumnFamily for type? but no default?
+    cf_tree: String,
+}
+
+pub struct RocksDbWrapper {
+    db: Arc<DB>,
+    config: RocksDbWrapperConfig
+}
+
+impl RocksDbWrapper {
+    fn new_with_db(db: Arc<DB>, config: RocksDbWrapperConfig) -> PmtreeResult<Self> {
+        Ok(RocksDbWrapper {
+            db,
+            config,
+        })
+    }
+}
+
+impl Database for RocksDbWrapper {
+
+    type Config = RocksDbWrapperConfig;
+
+    fn new(mut config: Self::Config) -> PmtreeResult<Self>
+    where
+        Self: Sized
+    {
+        let cfs = std::mem::take(&mut config.cfs);
+
+        let db = DB::open_cf_descriptors(
+            &config.options,
+            &config.db_path,
+            cfs
+        ).map_err(|e|
+            PmtreeErrorKind::DatabaseError(DatabaseErrorKind::CustomError(e.to_string()))
+        )?;
+
+        Ok(Self {
+            db: Arc::new(db),
+            config
+        })
+    }
+
+    fn load(config: Self::Config) -> PmtreeResult<Self>
+    where
+        Self: Sized
+    {
+        // FIXME
+        // Self::new(config)
+        unimplemented!()
+    }
+
+    fn get(&self, key: DBKey) -> PmtreeResult<Option<Value>> {
+        // Unwrap safe - assume the db is created with this column family
+        let cf = self.db.cf_handle(self.config.cf_tree.as_str()).unwrap();
+        match self.db.get_cf(cf, key.as_slice()) {
+            Ok(value) => Ok(value),
+            Err(_e) => Err(PmtreeErrorKind::TreeError(TreeErrorKind::InvalidKey))
+        }
+    }
+
+    fn put(&mut self, key: DBKey, value: Value) -> PmtreeResult<()> {
+        // Unwrap safe - assume the db is created with this column family
+        let cf = self.db.cf_handle(self.config.cf_tree.as_str()).unwrap();
+        match self.db.put_cf(cf, key.as_slice(), value.as_slice()) {
+            Ok(_) => Ok(()),
+            Err(_e) => Err(PmtreeErrorKind::TreeError(TreeErrorKind::InvalidKey))
+        }
+    }
+
+    fn put_batch(&mut self, subtree: HashMap<DBKey, Value>) -> PmtreeResult<()> {
+        // Unwrap safe - assume the db is created with this column family
+        let cf = self.db.cf_handle(self.config.cf_tree.as_str()).unwrap();
+        let mut batch = WriteBatch::new();
+
+        for (key, value) in subtree {
+            batch.put_cf(cf, key.as_slice(), value.as_slice());
+        }
+
+        self
+            .db
+            .write(batch)
+            .map_err(|_| PmtreeErrorKind::TreeError(TreeErrorKind::InvalidKey))?
+            ;
+
+        Ok(())
+    }
+
+    fn close(&mut self) -> PmtreeResult<()> {
+
+        // Unwrap safe - assume the db is created with this column family
+        let cf = self.db.cf_handle(self.config.cf_tree.as_str()).unwrap();
+        self
+            .db
+            .flush_cf(cf)
+            .map_err(|e| PmtreeErrorKind::DatabaseError(DatabaseErrorKind::CustomError(
+                e.to_string()
+            )))? ;
+        Ok(())
+    }
+
+}