From 56dcc25f63b96317495e0dd1a128b82e2e013936 Mon Sep 17 00:00:00 2001 From: sydhds Date: Thu, 16 Oct 2025 11:56:09 +0200 Subject: [PATCH] Initial code to use rocksdb for pmtree --- Cargo.lock | 195 +++++++++++++++++++++++++++ utils/Cargo.toml | 4 +- utils/src/pm_tree/mod.rs | 4 + utils/src/pm_tree/rocksdb_adapter.rs | 116 ++++++++++++++++ 4 files changed, 318 insertions(+), 1 deletion(-) create mode 100644 utils/src/pm_tree/rocksdb_adapter.rs diff --git a/Cargo.lock b/Cargo.lock index 0ead010..de084a6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -425,6 +425,26 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags 2.9.4", + "cexpr", + "clang-sys", + "itertools 0.10.5", + "lazy_static", + "lazycell", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.106", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -491,12 +511,43 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "cast" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "cc" +version = "1.2.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac9fe6cdbb24b6ade63616c0a0688e45bb56732262c158df3c0c4bea4ca47cb7" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.3" @@ -530,6 +581,17 @@ dependencies = [ "half", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "4.5.47" @@ -789,6 +851,12 @@ dependencies = [ "bytes", ] +[[package]] +name = "find-msvc-tools" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" + [[package]] name = "fixed-hash" version = "0.8.0" @@ -865,6 +933,12 @@ dependencies = [ "wasi 0.14.7+wasi-0.2.4", ] +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + [[package]] name = "half" version = "2.6.0" @@ -968,6 +1042,16 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.3", + "libc", +] + [[package]] name = "js-sys" version = "0.3.80" @@ -993,18 +1077,59 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.175" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libm" version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +[[package]] +name = "librocksdb-sys" +version = "0.17.2+9.10.0" +source = "git+https://github.com/tillrohrmann/rust-rocksdb?branch=issues%2F836#9c2482629ac4a3f73e622fc6be1b6458b6bae8cc" +dependencies = [ + "bindgen", + "bzip2-sys", + "cc", + "libc", + "libz-sys", + "lz4-sys", + "zstd-sys", +] + +[[package]] +name = "libz-sys" +version = "1.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.11.0" @@ -1033,6 +1158,16 @@ version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "memchr" version = "2.7.5" @@ -1051,6 +1186,22 @@ dependencies = [ "zeroize", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -1168,6 +1319,12 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + [[package]] name = "plotters" version = "0.3.7" @@ -1464,6 +1621,15 @@ dependencies = [ "rustc-hex", ] +[[package]] +name = "rocksdb" +version = "0.23.0" +source = "git+https://github.com/tillrohrmann/rust-rocksdb?branch=issues%2F836#9c2482629ac4a3f73e622fc6be1b6458b6bae8cc" +dependencies = [ + "libc", + "librocksdb-sys", +] + [[package]] name = "ruint" version = "1.16.0" @@ -1497,6 +1663,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48fd7bd8a6377e15ad9d42a8ec25371b94ddc67abe7c8b9127bec79bebaaae18" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hex" version = "2.1.0" @@ -1639,6 +1811,12 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "sled" version = "0.34.7" @@ -1878,6 +2056,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -2110,9 +2294,20 @@ dependencies = [ "num-bigint", "num-traits", "rayon", + "rocksdb", "serde_json", "sled", "thiserror", "tiny-keccak", "vacp2p_pmtree", ] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/utils/Cargo.toml b/utils/Cargo.toml index cd51fb2..7a5085e 100644 --- a/utils/Cargo.toml +++ b/utils/Cargo.toml @@ -15,6 +15,7 @@ bench = false ark-ff = { version = "0.5.0", default-features = false } num-bigint = { version = "0.4.6", default-features = false } pmtree = { package = "vacp2p_pmtree", version = "2.0.3", optional = true } +rocksdb = { git = "https://github.com/tillrohrmann/rust-rocksdb", branch = "issues/836", optional = true } sled = "0.34.7" serde_json = "1.0.141" lazy_static = "1.5.0" @@ -32,7 +33,8 @@ criterion = { version = "0.7.0", features = ["html_reports"] } [features] default = [] parallel = ["ark-ff/parallel"] -pmtree-ft = ["pmtree"] +pmtree-ft = ["pmtree", "rocksdb"] +# pmtree-ft-rocksdb = ["pmtree", "rocksdb"] [[bench]] name = "merkle_tree_benchmark" diff --git a/utils/src/pm_tree/mod.rs b/utils/src/pm_tree/mod.rs index 4cd8be0..503c9cb 100644 --- a/utils/src/pm_tree/mod.rs +++ b/utils/src/pm_tree/mod.rs @@ -1,4 +1,8 @@ pub mod sled_adapter; pub use self::sled_adapter::SledDB; + +pub mod rocksdb_adapter; +pub use self::rocksdb_adapter::RocksDbWrapper; + pub use pmtree; pub use sled::{Config, Mode}; diff --git a/utils/src/pm_tree/rocksdb_adapter.rs b/utils/src/pm_tree/rocksdb_adapter.rs new file mode 100644 index 0000000..2f90856 --- /dev/null +++ b/utils/src/pm_tree/rocksdb_adapter.rs @@ -0,0 +1,116 @@ +use std::{ + collections::HashMap, + sync::Arc, +}; +use std::path::PathBuf; +use pmtree::{DBKey, Database, DatabaseErrorKind, PmtreeErrorKind, PmtreeResult, TreeErrorKind, Value}; +use rocksdb::{ + ColumnFamily, ColumnFamilyDescriptor, DB, Options, ReadOptions, WriteBatch, WriteBatchWithIndex, +}; + +#[derive(Default)] +pub struct RocksDbWrapperConfig { + options: Options, + db_path: PathBuf, + cfs: Vec, + // TODO: ColumnFamily for type? but no default? + cf_tree: String, +} + +pub struct RocksDbWrapper { + db: Arc, + config: RocksDbWrapperConfig +} + +impl RocksDbWrapper { + fn new_with_db(db: Arc, config: RocksDbWrapperConfig) -> PmtreeResult { + Ok(RocksDbWrapper { + db, + config, + }) + } +} + +impl Database for RocksDbWrapper { + + type Config = RocksDbWrapperConfig; + + fn new(mut config: Self::Config) -> PmtreeResult + where + Self: Sized + { + let cfs = std::mem::take(&mut config.cfs); + + let db = DB::open_cf_descriptors( + &config.options, + &config.db_path, + cfs + ).map_err(|e| + PmtreeErrorKind::DatabaseError(DatabaseErrorKind::CustomError(e.to_string())) + )?; + + Ok(Self { + db: Arc::new(db), + config + }) + } + + fn load(config: Self::Config) -> PmtreeResult + where + Self: Sized + { + // FIXME + // Self::new(config) + unimplemented!() + } + + fn get(&self, key: DBKey) -> PmtreeResult> { + // Unwrap safe - assume the db is created with this column family + let cf = self.db.cf_handle(self.config.cf_tree.as_str()).unwrap(); + match self.db.get_cf(cf, key.as_slice()) { + Ok(value) => Ok(value), + Err(_e) => Err(PmtreeErrorKind::TreeError(TreeErrorKind::InvalidKey)) + } + } + + fn put(&mut self, key: DBKey, value: Value) -> PmtreeResult<()> { + // Unwrap safe - assume the db is created with this column family + let cf = self.db.cf_handle(self.config.cf_tree.as_str()).unwrap(); + match self.db.put_cf(cf, key.as_slice(), value.as_slice()) { + Ok(_) => Ok(()), + Err(_e) => Err(PmtreeErrorKind::TreeError(TreeErrorKind::InvalidKey)) + } + } + + fn put_batch(&mut self, subtree: HashMap) -> PmtreeResult<()> { + // Unwrap safe - assume the db is created with this column family + let cf = self.db.cf_handle(self.config.cf_tree.as_str()).unwrap(); + let mut batch = WriteBatch::new(); + + for (key, value) in subtree { + batch.put_cf(cf, key.as_slice(), value.as_slice()); + } + + self + .db + .write(batch) + .map_err(|_| PmtreeErrorKind::TreeError(TreeErrorKind::InvalidKey))? + ; + + Ok(()) + } + + fn close(&mut self) -> PmtreeResult<()> { + + // Unwrap safe - assume the db is created with this column family + let cf = self.db.cf_handle(self.config.cf_tree.as_str()).unwrap(); + self + .db + .flush_cf(cf) + .map_err(|e| PmtreeErrorKind::DatabaseError(DatabaseErrorKind::CustomError( + e.to_string() + )))? ; + Ok(()) + } + +} \ No newline at end of file