Initial code to use rocksdb for pmtree

Cargo.lock

@@ -425,6 +425,26 @@ version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
 
+[[package]]
+name = "bindgen"
+version = "0.69.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088"
+dependencies = [
+ "bitflags 2.9.4",
+ "cexpr",
+ "clang-sys",
+ "itertools 0.10.5",
+ "lazy_static",
+ "lazycell",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "rustc-hash",
+ "shlex",
+ "syn 2.0.106",
+]
+
 [[package]]
 name = "bitflags"
 version = "1.3.2"
@@ -491,12 +511,43 @@ version = "1.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a"
 
+[[package]]
+name = "bzip2-sys"
+version = "0.1.13+1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14"
+dependencies = [
+ "cc",
+ "pkg-config",
+]
+
 [[package]]
 name = "cast"
 version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
 
+[[package]]
+name = "cc"
+version = "1.2.41"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac9fe6cdbb24b6ade63616c0a0688e45bb56732262c158df3c0c4bea4ca47cb7"
+dependencies = [
+ "find-msvc-tools",
+ "jobserver",
+ "libc",
+ "shlex",
+]
+
+[[package]]
+name = "cexpr"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
+dependencies = [
+ "nom",
+]
+
 [[package]]
 name = "cfg-if"
 version = "1.0.3"
@@ -530,6 +581,17 @@ dependencies = [
  "half",
 ]
 
+[[package]]
+name = "clang-sys"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
+dependencies = [
+ "glob",
+ "libc",
+ "libloading",
+]
+
 [[package]]
 name = "clap"
 version = "4.5.47"
@@ -789,6 +851,12 @@ dependencies = [
  "bytes",
 ]
 
+[[package]]
+name = "find-msvc-tools"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127"
+
 [[package]]
 name = "fixed-hash"
 version = "0.8.0"
@@ -865,6 +933,12 @@ dependencies = [
  "wasi 0.14.7+wasi-0.2.4",
 ]
 
+[[package]]
+name = "glob"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
+
 [[package]]
 name = "half"
 version = "2.6.0"
@@ -968,6 +1042,16 @@ version = "1.0.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
 
+[[package]]
+name = "jobserver"
+version = "0.1.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33"
+dependencies = [
+ "getrandom 0.3.3",
+ "libc",
+]
+
 [[package]]
 name = "js-sys"
 version = "0.3.80"
@@ -993,18 +1077,59 @@ version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
 
+[[package]]
+name = "lazycell"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
+
 [[package]]
 name = "libc"
 version = "0.2.175"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543"
 
+[[package]]
+name = "libloading"
+version = "0.8.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"
+dependencies = [
+ "cfg-if",
+ "windows-link",
+]
+
 [[package]]
 name = "libm"
 version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de"
 
+[[package]]
+name = "librocksdb-sys"
+version = "0.17.2+9.10.0"
+source = "git+https://github.com/tillrohrmann/rust-rocksdb?branch=issues%2F836#9c2482629ac4a3f73e622fc6be1b6458b6bae8cc"
+dependencies = [
+ "bindgen",
+ "bzip2-sys",
+ "cc",
+ "libc",
+ "libz-sys",
+ "lz4-sys",
+ "zstd-sys",
+]
+
+[[package]]
+name = "libz-sys"
+version = "1.1.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d"
+dependencies = [
+ "cc",
+ "pkg-config",
+ "vcpkg",
+]
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.11.0"
@@ -1033,6 +1158,16 @@ version = "0.4.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432"
 
+[[package]]
+name = "lz4-sys"
+version = "1.11.1+lz4-1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6"
+dependencies = [
+ "cc",
+ "libc",
+]
+
 [[package]]
 name = "memchr"
 version = "2.7.5"
@@ -1051,6 +1186,22 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
 [[package]]
 name = "num-bigint"
 version = "0.4.6"
@@ -1168,6 +1319,12 @@ version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b"
 
+[[package]]
+name = "pkg-config"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
+
 [[package]]
 name = "plotters"
 version = "0.3.7"
@@ -1464,6 +1621,15 @@ dependencies = [
  "rustc-hex",
 ]
 
+[[package]]
+name = "rocksdb"
+version = "0.23.0"
+source = "git+https://github.com/tillrohrmann/rust-rocksdb?branch=issues%2F836#9c2482629ac4a3f73e622fc6be1b6458b6bae8cc"
+dependencies = [
+ "libc",
+ "librocksdb-sys",
+]
+
 [[package]]
 name = "ruint"
 version = "1.16.0"
@@ -1497,6 +1663,12 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "48fd7bd8a6377e15ad9d42a8ec25371b94ddc67abe7c8b9127bec79bebaaae18"
 
+[[package]]
+name = "rustc-hash"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
+
 [[package]]
 name = "rustc-hex"
 version = "2.1.0"
@@ -1639,6 +1811,12 @@ dependencies = [
  "digest 0.10.7",
 ]
 
+[[package]]
+name = "shlex"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
+
 [[package]]
 name = "sled"
 version = "0.34.7"
@@ -1878,6 +2056,12 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
 
+[[package]]
+name = "vcpkg"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+
 [[package]]
 name = "version_check"
 version = "0.9.5"
@@ -2110,9 +2294,20 @@ dependencies = [
  "num-bigint",
  "num-traits",
  "rayon",
+ "rocksdb",
  "serde_json",
  "sled",
  "thiserror",
  "tiny-keccak",
  "vacp2p_pmtree",
 ]
+
+[[package]]
+name = "zstd-sys"
+version = "2.0.16+zstd.1.5.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748"
+dependencies = [
+ "cc",
+ "pkg-config",
+]

utils/Cargo.toml

@@ -15,6 +15,7 @@ bench = false
 ark-ff = { version = "0.5.0", default-features = false }
 num-bigint = { version = "0.4.6", default-features = false }
 pmtree = { package = "vacp2p_pmtree", version = "2.0.3", optional = true }
+rocksdb = { git = "https://github.com/tillrohrmann/rust-rocksdb", branch = "issues/836", optional = true }
 sled = "0.34.7"
 serde_json = "1.0.141"
 lazy_static = "1.5.0"
@@ -32,7 +33,8 @@ criterion = { version = "0.7.0", features = ["html_reports"] }
 [features]
 default = []
 parallel = ["ark-ff/parallel"]
-pmtree-ft = ["pmtree"]
+pmtree-ft = ["pmtree", "rocksdb"]
+# pmtree-ft-rocksdb = ["pmtree", "rocksdb"]
 
 [[bench]]
 name = "merkle_tree_benchmark"

utils/src/pm_tree/mod.rs

@@ -1,4 +1,8 @@
 pub mod sled_adapter;
 pub use self::sled_adapter::SledDB;
+
+pub mod rocksdb_adapter;
+pub use self::rocksdb_adapter::RocksDbWrapper;
+
 pub use pmtree;
 pub use sled::{Config, Mode};

utils/src/pm_tree/rocksdb_adapter.rs

@@ -0,0 +1,116 @@
+use std::{
+    collections::HashMap,
+    sync::Arc,
+};
+use std::path::PathBuf;
+use pmtree::{DBKey, Database, DatabaseErrorKind, PmtreeErrorKind, PmtreeResult, TreeErrorKind, Value};
+use rocksdb::{
+    ColumnFamily, ColumnFamilyDescriptor, DB, Options, ReadOptions, WriteBatch, WriteBatchWithIndex,
+};
+
+#[derive(Default)]
+pub struct RocksDbWrapperConfig {
+    options: Options,
+    db_path: PathBuf,
+    cfs: Vec<ColumnFamilyDescriptor>,
+    // TODO: ColumnFamily for type? but no default?
+    cf_tree: String,
+}
+
+pub struct RocksDbWrapper {
+    db: Arc<DB>,
+    config: RocksDbWrapperConfig
+}
+
+impl RocksDbWrapper {
+    fn new_with_db(db: Arc<DB>, config: RocksDbWrapperConfig) -> PmtreeResult<Self> {
+        Ok(RocksDbWrapper {
+            db,
+            config,
+        })
+    }
+}
+
+impl Database for RocksDbWrapper {
+
+    type Config = RocksDbWrapperConfig;
+
+    fn new(mut config: Self::Config) -> PmtreeResult<Self>
+    where
+        Self: Sized
+    {
+        let cfs = std::mem::take(&mut config.cfs);
+
+        let db = DB::open_cf_descriptors(
+            &config.options,
+            &config.db_path,
+            cfs
+        ).map_err(|e|
+            PmtreeErrorKind::DatabaseError(DatabaseErrorKind::CustomError(e.to_string()))
+        )?;
+
+        Ok(Self {
+            db: Arc::new(db),
+            config
+        })
+    }
+
+    fn load(config: Self::Config) -> PmtreeResult<Self>
+    where
+        Self: Sized
+    {
+        // FIXME
+        // Self::new(config)
+        unimplemented!()
+    }
+
+    fn get(&self, key: DBKey) -> PmtreeResult<Option<Value>> {
+        // Unwrap safe - assume the db is created with this column family
+        let cf = self.db.cf_handle(self.config.cf_tree.as_str()).unwrap();
+        match self.db.get_cf(cf, key.as_slice()) {
+            Ok(value) => Ok(value),
+            Err(_e) => Err(PmtreeErrorKind::TreeError(TreeErrorKind::InvalidKey))
+        }
+    }
+
+    fn put(&mut self, key: DBKey, value: Value) -> PmtreeResult<()> {
+        // Unwrap safe - assume the db is created with this column family
+        let cf = self.db.cf_handle(self.config.cf_tree.as_str()).unwrap();
+        match self.db.put_cf(cf, key.as_slice(), value.as_slice()) {
+            Ok(_) => Ok(()),
+            Err(_e) => Err(PmtreeErrorKind::TreeError(TreeErrorKind::InvalidKey))
+        }
+    }
+
+    fn put_batch(&mut self, subtree: HashMap<DBKey, Value>) -> PmtreeResult<()> {
+        // Unwrap safe - assume the db is created with this column family
+        let cf = self.db.cf_handle(self.config.cf_tree.as_str()).unwrap();
+        let mut batch = WriteBatch::new();
+
+        for (key, value) in subtree {
+            batch.put_cf(cf, key.as_slice(), value.as_slice());
+        }
+
+        self
+            .db
+            .write(batch)
+            .map_err(|_| PmtreeErrorKind::TreeError(TreeErrorKind::InvalidKey))?
+            ;
+
+        Ok(())
+    }
+
+    fn close(&mut self) -> PmtreeResult<()> {
+
+        // Unwrap safe - assume the db is created with this column family
+        let cf = self.db.cf_handle(self.config.cf_tree.as_str()).unwrap();
+        self
+            .db
+            .flush_cf(cf)
+            .map_err(|e| PmtreeErrorKind::DatabaseError(DatabaseErrorKind::CustomError(
+                e.to_string()
+            )))? ;
+        Ok(())
+    }
+
+}