From 662c0486a1e232a5e04d5badfa38cd99cfa8fb22 Mon Sep 17 00:00:00 2001 From: ligt Date: Mon, 15 Dec 2025 17:15:57 +0700 Subject: [PATCH] feat(storage): add rocksdb provider into database provider (#20253) --- crates/cli/commands/src/common.rs | 13 +- .../cli/commands/src/stage/dump/execution.rs | 3 +- .../src/stage/dump/hashing_account.rs | 3 +- .../src/stage/dump/hashing_storage.rs | 3 +- crates/cli/commands/src/stage/dump/merkle.rs | 3 +- crates/e2e-test-utils/src/setup_import.rs | 13 ++ crates/ethereum/node/src/node.rs | 3 +- crates/ethereum/node/tests/it/testing.rs | 1 + crates/exex/test-utils/src/lib.rs | 8 +- crates/node/builder/src/launch/common.rs | 18 +- crates/node/core/src/args/datadir_args.rs | 4 + crates/node/core/src/dirs.rs | 12 + crates/optimism/node/src/node.rs | 3 +- crates/stages/api/src/stage.rs | 9 +- .../stages/stages/src/test_utils/test_db.rs | 13 +- crates/storage/db-common/src/init.rs | 4 +- crates/storage/db/src/lib.rs | 8 + .../src/providers/blockchain_provider.rs | 15 +- .../src/providers/database/builder.rs | 77 +++++-- .../provider/src/providers/database/mod.rs | 42 +++- .../src/providers/database/provider.rs | 22 +- crates/storage/provider/src/providers/mod.rs | 5 +- .../src/providers/rocksdb/provider.rs | 6 +- .../provider/src/providers/rocksdb_stub.rs | 210 ++++++++++++++++++ crates/storage/provider/src/test_utils/mod.rs | 8 +- .../storage/provider/src/test_utils/noop.rs | 10 +- crates/storage/provider/src/traits/full.rs | 7 +- crates/storage/provider/src/traits/mod.rs | 3 + .../provider/src/traits/rocksdb_provider.rs | 9 + docs/vocs/docs/pages/cli/op-reth/db.mdx | 3 + .../vocs/docs/pages/cli/op-reth/import-op.mdx | 3 + .../pages/cli/op-reth/import-receipts-op.mdx | 3 + .../docs/pages/cli/op-reth/init-state.mdx | 3 + docs/vocs/docs/pages/cli/op-reth/init.mdx | 3 + docs/vocs/docs/pages/cli/op-reth/node.mdx | 3 + docs/vocs/docs/pages/cli/op-reth/p2p/body.mdx | 3 + .../docs/pages/cli/op-reth/p2p/header.mdx | 3 + docs/vocs/docs/pages/cli/op-reth/prune.mdx | 3 + .../docs/pages/cli/op-reth/re-execute.mdx | 3 + .../docs/pages/cli/op-reth/stage/drop.mdx | 3 + .../docs/pages/cli/op-reth/stage/dump.mdx | 3 + .../vocs/docs/pages/cli/op-reth/stage/run.mdx | 3 + .../docs/pages/cli/op-reth/stage/unwind.mdx | 3 + docs/vocs/docs/pages/cli/reth/db.mdx | 3 + docs/vocs/docs/pages/cli/reth/download.mdx | 3 + docs/vocs/docs/pages/cli/reth/export-era.mdx | 3 + docs/vocs/docs/pages/cli/reth/import-era.mdx | 3 + docs/vocs/docs/pages/cli/reth/import.mdx | 3 + docs/vocs/docs/pages/cli/reth/init-state.mdx | 3 + docs/vocs/docs/pages/cli/reth/init.mdx | 3 + docs/vocs/docs/pages/cli/reth/node.mdx | 3 + docs/vocs/docs/pages/cli/reth/p2p/body.mdx | 3 + docs/vocs/docs/pages/cli/reth/p2p/header.mdx | 3 + docs/vocs/docs/pages/cli/reth/prune.mdx | 3 + docs/vocs/docs/pages/cli/reth/re-execute.mdx | 3 + docs/vocs/docs/pages/cli/reth/stage/drop.mdx | 3 + docs/vocs/docs/pages/cli/reth/stage/dump.mdx | 3 + docs/vocs/docs/pages/cli/reth/stage/run.mdx | 3 + .../vocs/docs/pages/cli/reth/stage/unwind.mdx | 3 + examples/rpc-db/src/main.rs | 3 +- 60 files changed, 564 insertions(+), 64 deletions(-) create mode 100644 crates/storage/provider/src/providers/rocksdb_stub.rs create mode 100644 crates/storage/provider/src/traits/rocksdb_provider.rs diff --git a/crates/cli/commands/src/common.rs b/crates/cli/commands/src/common.rs index 62fb365f38..500439d9a7 100644 --- a/crates/cli/commands/src/common.rs +++ b/crates/cli/commands/src/common.rs @@ -23,7 +23,7 @@ use reth_node_core::{ dirs::{ChainPath, DataDirPath}, }; use reth_provider::{ - providers::{BlockchainProvider, NodeTypesForProvider, StaticFileProvider}, + providers::{BlockchainProvider, NodeTypesForProvider, RocksDBProvider, StaticFileProvider}, ProviderFactory, StaticFileProviderFactory, }; use reth_stages::{sets::DefaultStages, Pipeline, PipelineTarget}; @@ -75,10 +75,12 @@ impl EnvironmentArgs { let data_dir = self.datadir.clone().resolve_datadir(self.chain.chain()); let db_path = data_dir.db(); let sf_path = data_dir.static_files(); + let rocksdb_path = data_dir.rocksdb(); if access.is_read_write() { reth_fs_util::create_dir_all(&db_path)?; reth_fs_util::create_dir_all(&sf_path)?; + reth_fs_util::create_dir_all(&rocksdb_path)?; } let config_path = self.config.clone().unwrap_or_else(|| data_dir.config()); @@ -108,8 +110,13 @@ impl EnvironmentArgs { StaticFileProvider::read_only(sf_path, false)?, ), }; + // TransactionDB only support read-write mode + let rocksdb_provider = RocksDBProvider::builder(data_dir.rocksdb()) + .with_database_log_level(self.db.log_level) + .build()?; - let provider_factory = self.create_provider_factory(&config, db, sfp, access)?; + let provider_factory = + self.create_provider_factory(&config, db, sfp, rocksdb_provider, access)?; if access.is_read_write() { debug!(target: "reth::cli", chain=%self.chain.chain(), genesis=?self.chain.genesis_hash(), "Initializing genesis"); init_genesis_with_settings(&provider_factory, self.static_files.to_settings())?; @@ -128,6 +135,7 @@ impl EnvironmentArgs { config: &Config, db: Arc, static_file_provider: StaticFileProvider, + rocksdb_provider: RocksDBProvider, access: AccessRights, ) -> eyre::Result>>> where @@ -138,6 +146,7 @@ impl EnvironmentArgs { db, self.chain.clone(), static_file_provider, + rocksdb_provider, )? .with_prune_modes(prune_modes.clone()); diff --git a/crates/cli/commands/src/stage/dump/execution.rs b/crates/cli/commands/src/stage/dump/execution.rs index 887f97dddd..dafbf7c31a 100644 --- a/crates/cli/commands/src/stage/dump/execution.rs +++ b/crates/cli/commands/src/stage/dump/execution.rs @@ -9,7 +9,7 @@ use reth_evm::ConfigureEvm; use reth_node_builder::NodeTypesWithDB; use reth_node_core::dirs::{ChainPath, DataDirPath}; use reth_provider::{ - providers::{ProviderNodeTypes, StaticFileProvider}, + providers::{ProviderNodeTypes, RocksDBProvider, StaticFileProvider}, DatabaseProviderFactory, ProviderFactory, }; use reth_stages::{stages::ExecutionStage, Stage, StageCheckpoint, UnwindInput}; @@ -42,6 +42,7 @@ where Arc::new(output_db), db_tool.chain(), StaticFileProvider::read_write(output_datadir.static_files())?, + RocksDBProvider::builder(output_datadir.rocksdb()).build()?, )?, to, from, diff --git a/crates/cli/commands/src/stage/dump/hashing_account.rs b/crates/cli/commands/src/stage/dump/hashing_account.rs index 0e976d4235..ecd138ece3 100644 --- a/crates/cli/commands/src/stage/dump/hashing_account.rs +++ b/crates/cli/commands/src/stage/dump/hashing_account.rs @@ -6,7 +6,7 @@ use reth_db_api::{database::Database, table::TableImporter, tables}; use reth_db_common::DbTool; use reth_node_core::dirs::{ChainPath, DataDirPath}; use reth_provider::{ - providers::{ProviderNodeTypes, StaticFileProvider}, + providers::{ProviderNodeTypes, RocksDBProvider, StaticFileProvider}, DatabaseProviderFactory, ProviderFactory, }; use reth_stages::{stages::AccountHashingStage, Stage, StageCheckpoint, UnwindInput}; @@ -39,6 +39,7 @@ pub(crate) async fn dump_hashing_account_stage = ProviderFactory::new( db.clone(), chain_spec.clone(), reth_provider::providers::StaticFileProvider::read_write(static_files_path).unwrap(), + reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path).build().unwrap(), ) .expect("failed to create provider factory"); diff --git a/crates/ethereum/node/src/node.rs b/crates/ethereum/node/src/node.rs index bdf03404aa..cf409cce9c 100644 --- a/crates/ethereum/node/src/node.rs +++ b/crates/ethereum/node/src/node.rs @@ -118,13 +118,14 @@ impl EthereumNode { /// use reth_chainspec::ChainSpecBuilder; /// use reth_db::open_db_read_only; /// use reth_node_ethereum::EthereumNode; - /// use reth_provider::providers::StaticFileProvider; + /// use reth_provider::providers::{RocksDBProvider, StaticFileProvider}; /// use std::sync::Arc; /// /// let factory = EthereumNode::provider_factory_builder() /// .db(Arc::new(open_db_read_only("db", Default::default()).unwrap())) /// .chainspec(ChainSpecBuilder::mainnet().build().into()) /// .static_file(StaticFileProvider::read_only("db/static_files", false).unwrap()) + /// .rocksdb_provider(RocksDBProvider::builder("db/rocksdb").build().unwrap()) /// .build_provider_factory(); /// ``` pub fn provider_factory_builder() -> ProviderFactoryBuilder { diff --git a/crates/ethereum/node/tests/it/testing.rs b/crates/ethereum/node/tests/it/testing.rs index 16f26a8c6a..6bd21a0879 100644 --- a/crates/ethereum/node/tests/it/testing.rs +++ b/crates/ethereum/node/tests/it/testing.rs @@ -28,6 +28,7 @@ async fn testing_rpc_build_block_works() -> eyre::Result<()> { datadir: MaybePlatformPath::::from_str(tempdir.path().to_str().unwrap()) .expect("valid datadir"), static_files_path: Some(tempdir.path().join("static")), + rocksdb_path: Some(tempdir.path().join("rocksdb")), }; let config = NodeConfig::test().with_datadir_args(datadir_args).with_rpc(rpc_args); let db = create_test_rw_db(); diff --git a/crates/exex/test-utils/src/lib.rs b/crates/exex/test-utils/src/lib.rs index 4b29dec99c..8430ea5d91 100644 --- a/crates/exex/test-utils/src/lib.rs +++ b/crates/exex/test-utils/src/lib.rs @@ -20,7 +20,9 @@ use futures_util::FutureExt; use reth_chainspec::{ChainSpec, MAINNET}; use reth_consensus::test_utils::TestConsensus; use reth_db::{ - test_utils::{create_test_rw_db, create_test_static_files_dir, TempDatabase}, + test_utils::{ + create_test_rocksdb_dir, create_test_rw_db, create_test_static_files_dir, TempDatabase, + }, DatabaseEnv, }; use reth_db_common::init::init_genesis; @@ -50,7 +52,7 @@ use reth_node_ethereum::{ use reth_payload_builder::noop::NoopPayloadBuilderService; use reth_primitives_traits::{Block as _, RecoveredBlock}; use reth_provider::{ - providers::{BlockchainProvider, StaticFileProvider}, + providers::{BlockchainProvider, RocksDBProvider, StaticFileProvider}, BlockReader, EthStorage, ProviderFactory, }; use reth_tasks::TaskManager; @@ -239,11 +241,13 @@ pub async fn test_exex_context_with_chain_spec( let consensus = Arc::new(TestConsensus::default()); let (static_dir, _) = create_test_static_files_dir(); + let (rocksdb_dir, _) = create_test_rocksdb_dir(); let db = create_test_rw_db(); let provider_factory = ProviderFactory::>::new( db, chain_spec.clone(), StaticFileProvider::read_write(static_dir.keep()).expect("static file provider"), + RocksDBProvider::builder(rocksdb_dir.keep()).build().unwrap(), )?; let genesis_hash = init_genesis(&provider_factory)?; diff --git a/crates/node/builder/src/launch/common.rs b/crates/node/builder/src/launch/common.rs index 7c6f5bfa7b..9d40526135 100644 --- a/crates/node/builder/src/launch/common.rs +++ b/crates/node/builder/src/launch/common.rs @@ -65,7 +65,7 @@ use reth_node_metrics::{ version::VersionInfo, }; use reth_provider::{ - providers::{NodeTypesForProvider, ProviderNodeTypes, StaticFileProvider}, + providers::{NodeTypesForProvider, ProviderNodeTypes, RocksDBProvider, StaticFileProvider}, BlockHashReader, BlockNumReader, ProviderError, ProviderFactory, ProviderResult, StageCheckpointReader, StaticFileProviderBuilder, StaticFileProviderFactory, }; @@ -485,9 +485,19 @@ where .with_blocks_per_file_for_segments(static_files_config.as_blocks_per_file_map()) .build()?; - let factory = - ProviderFactory::new(self.right().clone(), self.chain_spec(), static_file_provider)? - .with_prune_modes(self.prune_modes()); + // Initialize RocksDB provider with metrics and statistics enabled + let rocksdb_provider = RocksDBProvider::builder(self.data_dir().rocksdb()) + .with_metrics() + .with_statistics() + .build()?; + + let factory = ProviderFactory::new( + self.right().clone(), + self.chain_spec(), + static_file_provider, + rocksdb_provider, + )? + .with_prune_modes(self.prune_modes()); // Check for consistency between database and static files. If it fails, it unwinds to // the first block that's consistent between database and static files. diff --git a/crates/node/core/src/args/datadir_args.rs b/crates/node/core/src/args/datadir_args.rs index cb0590f177..a968334ae3 100644 --- a/crates/node/core/src/args/datadir_args.rs +++ b/crates/node/core/src/args/datadir_args.rs @@ -27,6 +27,10 @@ pub struct DatadirArgs { verbatim_doc_comment )] pub static_files_path: Option, + + /// The absolute path to store `RocksDB` database in. + #[arg(long = "datadir.rocksdb", value_name = "PATH", verbatim_doc_comment)] + pub rocksdb_path: Option, } impl DatadirArgs { diff --git a/crates/node/core/src/dirs.rs b/crates/node/core/src/dirs.rs index 4f8507c4e6..a21cde2280 100644 --- a/crates/node/core/src/dirs.rs +++ b/crates/node/core/src/dirs.rs @@ -301,6 +301,18 @@ impl ChainPath { } } + /// Returns the path to the `RocksDB` database directory for this chain. + /// + /// `//rocksdb` + pub fn rocksdb(&self) -> PathBuf { + let datadir_args = &self.2; + if let Some(rocksdb_path) = &datadir_args.rocksdb_path { + rocksdb_path.clone() + } else { + self.data_dir().join("rocksdb") + } + } + /// Returns the path to the reth p2p secret key for this chain. /// /// `//discovery-secret` diff --git a/crates/optimism/node/src/node.rs b/crates/optimism/node/src/node.rs index c177051677..dd68ab8a8e 100644 --- a/crates/optimism/node/src/node.rs +++ b/crates/optimism/node/src/node.rs @@ -218,13 +218,14 @@ impl OpNode { /// use reth_db::open_db_read_only; /// use reth_optimism_chainspec::OpChainSpecBuilder; /// use reth_optimism_node::OpNode; - /// use reth_provider::providers::StaticFileProvider; + /// use reth_provider::providers::{RocksDBProvider, StaticFileProvider}; /// use std::sync::Arc; /// /// let factory = OpNode::provider_factory_builder() /// .db(Arc::new(open_db_read_only("db", Default::default()).unwrap())) /// .chainspec(OpChainSpecBuilder::base_mainnet().build().into()) /// .static_file(StaticFileProvider::read_only("db/static_files", false).unwrap()) + /// .rocksdb_provider(RocksDBProvider::builder("db/rocksdb").build().unwrap()) /// .build_provider_factory(); /// ``` pub fn provider_factory_builder() -> ProviderFactoryBuilder { diff --git a/crates/stages/api/src/stage.rs b/crates/stages/api/src/stage.rs index f9a5c1e8b3..b56f59cc91 100644 --- a/crates/stages/api/src/stage.rs +++ b/crates/stages/api/src/stage.rs @@ -324,11 +324,13 @@ impl + ?Sized> StageExt for S {} #[cfg(test)] mod tests { use reth_chainspec::MAINNET; - use reth_db::test_utils::{create_test_rw_db, create_test_static_files_dir}; + use reth_db::test_utils::{ + create_test_rocksdb_dir, create_test_rw_db, create_test_static_files_dir, + }; use reth_db_api::{models::StoredBlockBodyIndices, tables, transaction::DbTxMut}; use reth_provider::{ - test_utils::MockNodeTypesWithDB, ProviderFactory, StaticFileProviderBuilder, - StaticFileProviderFactory, StaticFileSegment, + providers::RocksDBProvider, test_utils::MockNodeTypesWithDB, ProviderFactory, + StaticFileProviderBuilder, StaticFileProviderFactory, StaticFileSegment, }; use reth_stages_types::StageCheckpoint; use reth_testing_utils::generators::{self, random_signed_tx}; @@ -346,6 +348,7 @@ mod tests { .with_blocks_per_file(1) .build() .unwrap(), + RocksDBProvider::builder(create_test_rocksdb_dir().0.keep()).build().unwrap(), ) .unwrap(); diff --git a/crates/stages/stages/src/test_utils/test_db.rs b/crates/stages/stages/src/test_utils/test_db.rs index 5df7707df0..e404450379 100644 --- a/crates/stages/stages/src/test_utils/test_db.rs +++ b/crates/stages/stages/src/test_utils/test_db.rs @@ -1,7 +1,10 @@ use alloy_primitives::{keccak256, Address, BlockNumber, TxHash, TxNumber, B256}; use reth_chainspec::MAINNET; use reth_db::{ - test_utils::{create_test_rw_db, create_test_rw_db_with_path, create_test_static_files_dir}, + test_utils::{ + create_test_rocksdb_dir, create_test_rw_db, create_test_rw_db_with_path, + create_test_static_files_dir, + }, DatabaseEnv, }; use reth_db_api::{ @@ -17,7 +20,9 @@ use reth_db_api::{ use reth_ethereum_primitives::{Block, EthPrimitives, Receipt}; use reth_primitives_traits::{Account, SealedBlock, SealedHeader, StorageEntry}; use reth_provider::{ - providers::{StaticFileProvider, StaticFileProviderRWRefMut, StaticFileWriter}, + providers::{ + RocksDBProvider, StaticFileProvider, StaticFileProviderRWRefMut, StaticFileWriter, + }, test_utils::MockNodeTypesWithDB, HistoryWriter, ProviderError, ProviderFactory, StaticFileProviderFactory, StatsReader, }; @@ -38,12 +43,14 @@ impl Default for TestStageDB { /// Create a new instance of [`TestStageDB`] fn default() -> Self { let (static_dir, static_dir_path) = create_test_static_files_dir(); + let (_, rocksdb_dir_path) = create_test_rocksdb_dir(); Self { temp_static_files_dir: static_dir, factory: ProviderFactory::new( create_test_rw_db(), MAINNET.clone(), StaticFileProvider::read_write(static_dir_path).unwrap(), + RocksDBProvider::builder(rocksdb_dir_path).build().unwrap(), ) .expect("failed to create test provider factory"), } @@ -53,6 +60,7 @@ impl Default for TestStageDB { impl TestStageDB { pub fn new(path: &Path) -> Self { let (static_dir, static_dir_path) = create_test_static_files_dir(); + let (_, rocksdb_dir_path) = create_test_rocksdb_dir(); Self { temp_static_files_dir: static_dir, @@ -60,6 +68,7 @@ impl TestStageDB { create_test_rw_db_with_path(path), MAINNET.clone(), StaticFileProvider::read_write(static_dir_path).unwrap(), + RocksDBProvider::builder(rocksdb_dir_path).build().unwrap(), ) .expect("failed to create test provider factory"), } diff --git a/crates/storage/db-common/src/init.rs b/crates/storage/db-common/src/init.rs index f2790dfe23..811966c781 100644 --- a/crates/storage/db-common/src/init.rs +++ b/crates/storage/db-common/src/init.rs @@ -711,7 +711,7 @@ mod tests { }; use reth_provider::{ test_utils::{create_test_provider_factory_with_chain_spec, MockNodeTypesWithDB}, - ProviderFactory, + ProviderFactory, RocksDBProviderFactory, }; use std::{collections::BTreeMap, sync::Arc}; @@ -756,6 +756,7 @@ mod tests { fn fail_init_inconsistent_db() { let factory = create_test_provider_factory_with_chain_spec(SEPOLIA.clone()); let static_file_provider = factory.static_file_provider(); + let rocksdb_provider = factory.rocksdb_provider(); init_genesis(&factory).unwrap(); // Try to init db with a different genesis block @@ -764,6 +765,7 @@ mod tests { factory.into_db(), MAINNET.clone(), static_file_provider, + rocksdb_provider, ) .unwrap(), ); diff --git a/crates/storage/db/src/lib.rs b/crates/storage/db/src/lib.rs index d3851d04ad..cd37f50d61 100644 --- a/crates/storage/db/src/lib.rs +++ b/crates/storage/db/src/lib.rs @@ -159,6 +159,14 @@ pub mod test_utils { (temp_dir, path) } + /// Create `rocksdb` path for testing + #[track_caller] + pub fn create_test_rocksdb_dir() -> (TempDir, PathBuf) { + let temp_dir = TempDir::with_prefix("reth-test-rocksdb-").expect(ERROR_TEMPDIR); + let path = temp_dir.path().to_path_buf(); + (temp_dir, path) + } + /// Get a temporary directory path to use for the database pub fn tempdir_path() -> PathBuf { let builder = tempfile::Builder::new().prefix("reth-test-").rand_bytes(8).tempdir(); diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index ec93cc0210..9a6286cc8b 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -1,14 +1,15 @@ use crate::{ providers::{ - ConsistentProvider, ProviderNodeTypes, StaticFileProvider, StaticFileProviderRWRefMut, + ConsistentProvider, ProviderNodeTypes, RocksDBProvider, StaticFileProvider, + StaticFileProviderRWRefMut, }, AccountReader, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, BlockSource, CanonChainTracker, CanonStateNotifications, CanonStateSubscriptions, ChainSpecProvider, ChainStateBlockReader, ChangeSetReader, DatabaseProviderFactory, HashedPostStateProvider, HeaderProvider, ProviderError, ProviderFactory, PruneCheckpointReader, - ReceiptProvider, ReceiptProviderIdExt, StageCheckpointReader, StateProviderBox, - StateProviderFactory, StateReader, StaticFileProviderFactory, TransactionVariant, - TransactionsProvider, TrieReader, + ReceiptProvider, ReceiptProviderIdExt, RocksDBProviderFactory, StageCheckpointReader, + StateProviderBox, StateProviderFactory, StateReader, StaticFileProviderFactory, + TransactionVariant, TransactionsProvider, TrieReader, }; use alloy_consensus::transaction::TransactionMeta; use alloy_eips::{BlockHashOrNumber, BlockId, BlockNumHash, BlockNumberOrTag}; @@ -176,6 +177,12 @@ impl StaticFileProviderFactory for BlockchainProvider { } } +impl RocksDBProviderFactory for BlockchainProvider { + fn rocksdb_provider(&self) -> RocksDBProvider { + self.database.rocksdb_provider() + } +} + impl HeaderProvider for BlockchainProvider { type Header = HeaderTy; diff --git a/crates/storage/provider/src/providers/database/builder.rs b/crates/storage/provider/src/providers/database/builder.rs index bcd61f188f..4b1b55b4d4 100644 --- a/crates/storage/provider/src/providers/database/builder.rs +++ b/crates/storage/provider/src/providers/database/builder.rs @@ -4,7 +4,7 @@ //! up to the intended build target. use crate::{ - providers::{NodeTypesForProvider, StaticFileProvider}, + providers::{NodeTypesForProvider, RocksDBProvider, StaticFileProvider}, ProviderFactory, }; use reth_db::{ @@ -104,11 +104,12 @@ impl ProviderFactoryBuilder { where N: NodeTypesForProvider, { - let ReadOnlyConfig { db_dir, db_args, static_files_dir, watch_static_files } = + let ReadOnlyConfig { db_dir, db_args, static_files_dir, rocksdb_dir, watch_static_files } = config.into(); self.db(Arc::new(open_db_read_only(db_dir, db_args)?)) .chainspec(chainspec) .static_file(StaticFileProvider::read_only(static_files_dir, watch_static_files)?) + .rocksdb_provider(RocksDBProvider::builder(&rocksdb_dir).build()?) .build_provider_factory() .map_err(Into::into) } @@ -120,7 +121,7 @@ impl Default for ProviderFactoryBuilder { } } -/// Settings for how to open the database and static files. +/// Settings for how to open the database, static files, and `RocksDB`. /// /// The default derivation from a path assumes the path is the datadir: /// [`ReadOnlyConfig::from_datadir`] @@ -132,6 +133,8 @@ pub struct ReadOnlyConfig { pub db_args: DatabaseArguments, /// The path to the static file dir pub static_files_dir: PathBuf, + /// The path to the `RocksDB` directory + pub rocksdb_dir: PathBuf, /// Whether the static files should be watched for changes. pub watch_static_files: bool, } @@ -144,6 +147,7 @@ impl ReadOnlyConfig { /// ```text /// -`datadir` /// |__db + /// |__rocksdb /// |__static_files /// ``` /// @@ -151,7 +155,13 @@ impl ReadOnlyConfig { /// [`StaticFileProvider::read_only`] pub fn from_datadir(datadir: impl AsRef) -> Self { let datadir = datadir.as_ref(); - Self::from_dirs(datadir.join("db"), datadir.join("static_files")) + Self { + db_dir: datadir.join("db"), + db_args: Default::default(), + static_files_dir: datadir.join("static_files"), + rocksdb_dir: datadir.join("rocksdb"), + watch_static_files: true, + } } /// Disables long-lived read transaction safety guarantees. @@ -169,7 +179,8 @@ impl ReadOnlyConfig { /// /// ```text /// - db - /// -static_files + /// - rocksdb + /// - static_files /// ``` /// /// By default this watches the static file directory for changes, see also @@ -180,13 +191,10 @@ impl ReadOnlyConfig { /// If the path does not exist pub fn from_db_dir(db_dir: impl AsRef) -> Self { let db_dir = db_dir.as_ref(); - let static_files_dir = std::fs::canonicalize(db_dir) - .unwrap() - .parent() - .unwrap() - .to_path_buf() - .join("static_files"); - Self::from_dirs(db_dir, static_files_dir) + let datadir = std::fs::canonicalize(db_dir).unwrap().parent().unwrap().to_path_buf(); + let static_files_dir = datadir.join("static_files"); + let rocksdb_dir = datadir.join("rocksdb"); + Self::from_dirs(db_dir, static_files_dir, rocksdb_dir) } /// Creates the config for the given paths. @@ -194,11 +202,16 @@ impl ReadOnlyConfig { /// /// By default this watches the static file directory for changes, see also /// [`StaticFileProvider::read_only`] - pub fn from_dirs(db_dir: impl AsRef, static_files_dir: impl AsRef) -> Self { + pub fn from_dirs( + db_dir: impl AsRef, + static_files_dir: impl AsRef, + rocksdb_dir: impl AsRef, + ) -> Self { Self { - static_files_dir: static_files_dir.as_ref().into(), db_dir: db_dir.as_ref().into(), db_args: Default::default(), + static_files_dir: static_files_dir.as_ref().into(), + rocksdb_dir: rocksdb_dir.as_ref().into(), watch_static_files: true, } } @@ -317,7 +330,37 @@ impl TypesAnd3 { } } -impl TypesAnd3, StaticFileProvider> +impl TypesAnd3, StaticFileProvider> +where + N: NodeTypes, +{ + /// Configures the `RocksDB` provider. + pub fn rocksdb_provider( + self, + rocksdb_provider: RocksDBProvider, + ) -> TypesAnd4, StaticFileProvider, RocksDBProvider> { + TypesAnd4::new(self.val_1, self.val_2, self.val_3, rocksdb_provider) + } +} + +/// This is staging type that contains the configured types and _four_ values. +#[derive(Debug)] +pub struct TypesAnd4 { + _types: PhantomData, + val_1: Val1, + val_2: Val2, + val_3: Val3, + val_4: Val4, +} + +impl TypesAnd4 { + /// Creates a new instance with the given types and four values. + pub fn new(val_1: Val1, val_2: Val2, val_3: Val3, val_4: Val4) -> Self { + Self { _types: Default::default(), val_1, val_2, val_3, val_4 } + } +} + +impl TypesAnd4, StaticFileProvider, RocksDBProvider> where N: NodeTypesForProvider, DB: Database + DatabaseMetrics + Clone + Unpin + 'static, @@ -326,7 +369,7 @@ where pub fn build_provider_factory( self, ) -> ProviderResult>> { - let Self { _types, val_1, val_2, val_3 } = self; - ProviderFactory::new(val_1, val_2, val_3) + let Self { _types, val_1, val_2, val_3, val_4 } = self; + ProviderFactory::new(val_1, val_2, val_3, val_4) } } diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index 9f3d54f351..0270f24644 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -1,15 +1,15 @@ use crate::{ providers::{ - state::latest::LatestStateProvider, NodeTypesForProvider, StaticFileProvider, - StaticFileProviderRWRefMut, + state::latest::LatestStateProvider, NodeTypesForProvider, RocksDBProvider, + StaticFileProvider, StaticFileProviderRWRefMut, }, to_range, traits::{BlockSource, ReceiptProvider}, BlockHashReader, BlockNumReader, BlockReader, ChainSpecProvider, DatabaseProviderFactory, EitherWriterDestination, HashedPostStateProvider, HeaderProvider, HeaderSyncGapProvider, - MetadataProvider, ProviderError, PruneCheckpointReader, StageCheckpointReader, - StateProviderBox, StaticFileProviderFactory, StaticFileWriter, TransactionVariant, - TransactionsProvider, + MetadataProvider, ProviderError, PruneCheckpointReader, RocksDBProviderFactory, + StageCheckpointReader, StateProviderBox, StaticFileProviderFactory, StaticFileWriter, + TransactionVariant, TransactionsProvider, }; use alloy_consensus::transaction::TransactionMeta; use alloy_eips::BlockHashOrNumber; @@ -72,6 +72,8 @@ pub struct ProviderFactory { storage: Arc, /// Storage configuration settings for this node storage_settings: Arc>, + /// `RocksDB` provider + rocksdb_provider: RocksDBProvider, } impl ProviderFactory>> { @@ -87,6 +89,7 @@ impl ProviderFactory { db: N::DB, chain_spec: Arc, static_file_provider: StaticFileProvider, + rocksdb_provider: RocksDBProvider, ) -> ProviderResult { // Load storage settings from database at init time. Creates a temporary provider // to read persisted settings, falling back to legacy defaults if none exist. @@ -100,6 +103,7 @@ impl ProviderFactory { Default::default(), Default::default(), Arc::new(RwLock::new(legacy_settings)), + rocksdb_provider.clone(), ) .storage_settings()? .unwrap_or(legacy_settings); @@ -111,6 +115,7 @@ impl ProviderFactory { prune_modes: PruneModes::default(), storage: Default::default(), storage_settings: Arc::new(RwLock::new(storage_settings)), + rocksdb_provider, }) } } @@ -144,6 +149,12 @@ impl StorageSettingsCache for ProviderFactory { } } +impl RocksDBProviderFactory for ProviderFactory { + fn rocksdb_provider(&self) -> RocksDBProvider { + self.rocksdb_provider.clone() + } +} + impl>> ProviderFactory { /// Create new database provider by passing a path. [`ProviderFactory`] will own the database /// instance. @@ -152,11 +163,13 @@ impl>> ProviderFactory { chain_spec: Arc, args: DatabaseArguments, static_file_provider: StaticFileProvider, + rocksdb_provider: RocksDBProvider, ) -> RethResult { Self::new( Arc::new(init_db(path, args).map_err(RethError::msg)?), chain_spec, static_file_provider, + rocksdb_provider, ) .map_err(RethError::Provider) } @@ -178,6 +191,7 @@ impl ProviderFactory { self.prune_modes.clone(), self.storage.clone(), self.storage_settings.clone(), + self.rocksdb_provider.clone(), )) } @@ -194,6 +208,7 @@ impl ProviderFactory { self.prune_modes.clone(), self.storage.clone(), self.storage_settings.clone(), + self.rocksdb_provider.clone(), ))) } @@ -595,8 +610,15 @@ where N: NodeTypesWithDB, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let Self { db, chain_spec, static_file_provider, prune_modes, storage, storage_settings } = - self; + let Self { + db, + chain_spec, + static_file_provider, + prune_modes, + storage, + storage_settings, + rocksdb_provider, + } = self; f.debug_struct("ProviderFactory") .field("db", &db) .field("chain_spec", &chain_spec) @@ -604,6 +626,7 @@ where .field("prune_modes", &prune_modes) .field("storage", &storage) .field("storage_settings", &*storage_settings.read()) + .field("rocksdb_provider", &rocksdb_provider) .finish() } } @@ -617,6 +640,7 @@ impl Clone for ProviderFactory { prune_modes: self.prune_modes.clone(), storage: self.storage.clone(), storage_settings: self.storage_settings.clone(), + rocksdb_provider: self.rocksdb_provider.clone(), } } } @@ -635,7 +659,7 @@ mod tests { use reth_chainspec::ChainSpecBuilder; use reth_db::{ mdbx::DatabaseArguments, - test_utils::{create_test_static_files_dir, ERROR_TEMPDIR}, + test_utils::{create_test_rocksdb_dir, create_test_static_files_dir, ERROR_TEMPDIR}, }; use reth_db_api::tables; use reth_primitives_traits::SignerRecoverable; @@ -674,11 +698,13 @@ mod tests { fn provider_factory_with_database_path() { let chain_spec = ChainSpecBuilder::mainnet().build(); let (_static_dir, static_dir_path) = create_test_static_files_dir(); + let (_, rocksdb_path) = create_test_rocksdb_dir(); let factory = ProviderFactory::>::new_with_database_path( tempfile::TempDir::new().expect(ERROR_TEMPDIR).keep(), Arc::new(chain_spec), DatabaseArguments::new(Default::default()), StaticFileProvider::read_write(static_dir_path).unwrap(), + RocksDBProvider::builder(&rocksdb_path).build().unwrap(), ) .unwrap(); let provider = factory.provider().unwrap(); diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index f36b0c8bb6..a5ae621d27 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -4,6 +4,7 @@ use crate::{ }, providers::{ database::{chain::ChainStorage, metrics}, + rocksdb::RocksDBProvider, static_file::StaticFileWriter, NodeTypesForProvider, StaticFileProvider, }, @@ -16,10 +17,10 @@ use crate::{ DBProvider, EitherReader, EitherWriter, EitherWriterDestination, HashingWriter, HeaderProvider, HeaderSyncGapProvider, HistoricalStateProvider, HistoricalStateProviderRef, HistoryWriter, LatestStateProvider, LatestStateProviderRef, OriginalValuesKnown, ProviderError, - PruneCheckpointReader, PruneCheckpointWriter, RevertsInit, StageCheckpointReader, - StateProviderBox, StateWriter, StaticFileProviderFactory, StatsReader, StorageReader, - StorageTrieWriter, TransactionVariant, TransactionsProvider, TransactionsProviderExt, - TrieReader, TrieWriter, + PruneCheckpointReader, PruneCheckpointWriter, RevertsInit, RocksDBProviderFactory, + StageCheckpointReader, StateProviderBox, StateWriter, StaticFileProviderFactory, StatsReader, + StorageReader, StorageTrieWriter, TransactionVariant, TransactionsProvider, + TransactionsProviderExt, TrieReader, TrieWriter, }; use alloy_consensus::{ transaction::{SignerRecoverable, TransactionMeta, TxHashRef}, @@ -164,6 +165,8 @@ pub struct DatabaseProvider { storage: Arc, /// Storage configuration settings for this node storage_settings: Arc>, + /// `RocksDB` provider + rocksdb_provider: RocksDBProvider, /// Minimum distance from tip required for pruning minimum_pruning_distance: u64, } @@ -251,6 +254,13 @@ impl StaticFileProviderFactory for DatabaseProvider { } } +impl RocksDBProviderFactory for DatabaseProvider { + /// Returns the `RocksDB` provider. + fn rocksdb_provider(&self) -> RocksDBProvider { + self.rocksdb_provider.clone() + } +} + impl> ChainSpecProvider for DatabaseProvider { @@ -270,6 +280,7 @@ impl DatabaseProvider { prune_modes: PruneModes, storage: Arc, storage_settings: Arc>, + rocksdb_provider: RocksDBProvider, ) -> Self { Self { tx, @@ -278,6 +289,7 @@ impl DatabaseProvider { prune_modes, storage, storage_settings, + rocksdb_provider, minimum_pruning_distance: MINIMUM_PRUNING_DISTANCE, } } @@ -523,6 +535,7 @@ impl DatabaseProvider { prune_modes: PruneModes, storage: Arc, storage_settings: Arc>, + rocksdb_provider: RocksDBProvider, ) -> Self { Self { tx, @@ -531,6 +544,7 @@ impl DatabaseProvider { prune_modes, storage, storage_settings, + rocksdb_provider, minimum_pruning_distance: MINIMUM_PRUNING_DISTANCE, } } diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index 01f2473f77..25acfccd19 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -31,9 +31,10 @@ pub use consistent::ConsistentProvider; // RocksDB currently only supported on Unix platforms // Windows support is planned for future releases -#[cfg(all(unix, feature = "rocksdb"))] +#[cfg_attr(all(unix, feature = "rocksdb"), path = "rocksdb/mod.rs")] +#[cfg_attr(not(all(unix, feature = "rocksdb")), path = "rocksdb_stub.rs")] pub(crate) mod rocksdb; -#[cfg(all(unix, feature = "rocksdb"))] + pub use rocksdb::{RocksDBBuilder, RocksDBProvider, RocksTx}; /// Helper trait to bound [`NodeTypes`] so that combined with database they satisfy diff --git a/crates/storage/provider/src/providers/rocksdb/provider.rs b/crates/storage/provider/src/providers/rocksdb/provider.rs index f70981e58e..057ea91b2c 100644 --- a/crates/storage/provider/src/providers/rocksdb/provider.rs +++ b/crates/storage/provider/src/providers/rocksdb/provider.rs @@ -151,8 +151,10 @@ impl RocksDBBuilder { } /// Sets the log level from `DatabaseArgs` configuration. - pub const fn with_database_log_level(mut self, log_level: LogLevel) -> Self { - self.log_level = convert_log_level(log_level); + pub const fn with_database_log_level(mut self, log_level: Option) -> Self { + if let Some(level) = log_level { + self.log_level = convert_log_level(level); + } self } diff --git a/crates/storage/provider/src/providers/rocksdb_stub.rs b/crates/storage/provider/src/providers/rocksdb_stub.rs new file mode 100644 index 0000000000..17fd3d2507 --- /dev/null +++ b/crates/storage/provider/src/providers/rocksdb_stub.rs @@ -0,0 +1,210 @@ +//! Stub implementation of `RocksDB` provider. +//! +//! This module provides placeholder types that allow the code to compile when `RocksDB` is not +//! available (either on non-Unix platforms or when the `rocksdb` feature is not enabled). +//! Operations will produce errors if actually attempted. + +use reth_db_api::table::{Encode, Table}; +use reth_storage_errors::{ + db::LogLevel, + provider::{ProviderError::UnsupportedProvider, ProviderResult}, +}; +use std::path::Path; + +/// A stub `RocksDB` provider. +/// +/// This type exists to allow code to compile when `RocksDB` is not available (either on non-Unix +/// platforms or when the `rocksdb` feature is not enabled). When using this stub, the +/// `transaction_hash_numbers_in_rocksdb` flag should be set to `false` to ensure all operations +/// route to MDBX instead. +#[derive(Debug, Clone)] +pub struct RocksDBProvider; + +impl RocksDBProvider { + /// Creates a new stub `RocksDB` provider. + /// + /// On non-Unix platforms, this returns an error indicating `RocksDB` is not supported. + pub fn new(_path: impl AsRef) -> ProviderResult { + Ok(Self) + } + + /// Creates a new stub `RocksDB` provider builder. + pub fn builder(path: impl AsRef) -> RocksDBBuilder { + RocksDBBuilder::new(path) + } + + /// Get a value from `RocksDB` (stub implementation). + pub fn get(&self, _key: T::Key) -> ProviderResult> { + Err(UnsupportedProvider) + } + + /// Get a value from `RocksDB` using pre-encoded key (stub implementation). + pub const fn get_encoded( + &self, + _key: &::Encoded, + ) -> ProviderResult> { + Err(UnsupportedProvider) + } + + /// Put a value into `RocksDB` (stub implementation). + pub fn put(&self, _key: T::Key, _value: &T::Value) -> ProviderResult<()> { + Err(UnsupportedProvider) + } + + /// Put a value into `RocksDB` using pre-encoded key (stub implementation). + pub const fn put_encoded( + &self, + _key: &::Encoded, + _value: &T::Value, + ) -> ProviderResult<()> { + Err(UnsupportedProvider) + } + + /// Delete a value from `RocksDB` (stub implementation). + pub fn delete(&self, _key: T::Key) -> ProviderResult<()> { + Err(UnsupportedProvider) + } + + /// Write a batch of operations (stub implementation). + pub fn write_batch(&self, _f: F) -> ProviderResult<()> + where + F: FnOnce(&mut RocksDBBatch) -> ProviderResult<()>, + { + Err(UnsupportedProvider) + } + + /// Creates a new transaction (stub implementation). + pub const fn tx(&self) -> RocksTx { + RocksTx + } +} + +/// A stub batch writer for `RocksDB` on non-Unix platforms. +#[derive(Debug)] +pub struct RocksDBBatch; + +impl RocksDBBatch { + /// Puts a value into the batch (stub implementation). + pub fn put(&self, _key: T::Key, _value: &T::Value) -> ProviderResult<()> { + Err(UnsupportedProvider) + } + + /// Puts a value into the batch using pre-encoded key (stub implementation). + pub const fn put_encoded( + &self, + _key: &::Encoded, + _value: &T::Value, + ) -> ProviderResult<()> { + Err(UnsupportedProvider) + } + + /// Deletes a value from the batch (stub implementation). + pub fn delete(&self, _key: T::Key) -> ProviderResult<()> { + Err(UnsupportedProvider) + } +} + +/// A stub builder for `RocksDB` on non-Unix platforms. +#[derive(Debug)] +pub struct RocksDBBuilder; + +impl RocksDBBuilder { + /// Creates a new stub builder. + pub fn new>(_path: P) -> Self { + Self + } + + /// Adds a column family for a specific table type (stub implementation). + pub const fn with_table(self) -> Self { + self + } + + /// Enables metrics (stub implementation). + pub const fn with_metrics(self) -> Self { + self + } + + /// Enables `RocksDB` internal statistics collection (stub implementation). + pub const fn with_statistics(self) -> Self { + self + } + + /// Sets the log level from `DatabaseArgs` configuration (stub implementation). + pub const fn with_database_log_level(self, _log_level: Option) -> Self { + self + } + + /// Sets a custom block cache size (stub implementation). + pub const fn with_block_cache_size(self, _capacity_bytes: usize) -> Self { + self + } + + /// Build the `RocksDB` provider (stub implementation). + pub const fn build(self) -> ProviderResult { + Ok(RocksDBProvider) + } +} + +/// A stub transaction for `RocksDB`. +#[derive(Debug)] +pub struct RocksTx; + +impl RocksTx { + /// Gets a value from the specified table (stub implementation). + pub fn get(&self, _key: T::Key) -> ProviderResult> { + Err(UnsupportedProvider) + } + + /// Gets a value using pre-encoded key (stub implementation). + pub const fn get_encoded( + &self, + _key: &::Encoded, + ) -> ProviderResult> { + Err(UnsupportedProvider) + } + + /// Puts a value into the specified table (stub implementation). + pub fn put(&self, _key: T::Key, _value: &T::Value) -> ProviderResult<()> { + Err(UnsupportedProvider) + } + + /// Puts a value using pre-encoded key (stub implementation). + pub const fn put_encoded( + &self, + _key: &::Encoded, + _value: &T::Value, + ) -> ProviderResult<()> { + Err(UnsupportedProvider) + } + + /// Deletes a value from the specified table (stub implementation). + pub fn delete(&self, _key: T::Key) -> ProviderResult<()> { + Err(UnsupportedProvider) + } + + /// Creates an iterator for the specified table (stub implementation). + pub const fn iter(&self) -> ProviderResult> { + Err(UnsupportedProvider) + } + + /// Creates an iterator starting from the given key (stub implementation). + pub fn iter_from(&self, _key: T::Key) -> ProviderResult> { + Err(UnsupportedProvider) + } + + /// Commits the transaction (stub implementation). + pub const fn commit(self) -> ProviderResult<()> { + Err(UnsupportedProvider) + } + + /// Rolls back the transaction (stub implementation). + pub const fn rollback(self) -> ProviderResult<()> { + Err(UnsupportedProvider) + } +} + +/// A stub iterator for `RocksDB` transactions. +#[derive(Debug)] +pub struct RocksTxIter<'a, T> { + _marker: std::marker::PhantomData<(&'a (), T)>, +} diff --git a/crates/storage/provider/src/test_utils/mod.rs b/crates/storage/provider/src/test_utils/mod.rs index 5530c7411c..6b1e7c4b84 100644 --- a/crates/storage/provider/src/test_utils/mod.rs +++ b/crates/storage/provider/src/test_utils/mod.rs @@ -1,11 +1,13 @@ use crate::{ - providers::{NodeTypesForProvider, ProviderNodeTypes, StaticFileProvider}, + providers::{NodeTypesForProvider, ProviderNodeTypes, RocksDBProvider, StaticFileProvider}, HashingWriter, ProviderFactory, TrieWriter, }; use alloy_primitives::B256; use reth_chainspec::{ChainSpec, MAINNET}; use reth_db::{ - test_utils::{create_test_rw_db, create_test_static_files_dir, TempDatabase}, + test_utils::{ + create_test_rocksdb_dir, create_test_rw_db, create_test_static_files_dir, TempDatabase, + }, DatabaseEnv, }; use reth_errors::ProviderResult; @@ -54,11 +56,13 @@ pub fn create_test_provider_factory_with_node_types( chain_spec: Arc, ) -> ProviderFactory>>> { let (static_dir, _) = create_test_static_files_dir(); + let (rocksdb_dir, _) = create_test_rocksdb_dir(); let db = create_test_rw_db(); ProviderFactory::new( db, chain_spec, StaticFileProvider::read_write(static_dir.keep()).expect("static file provider"), + RocksDBProvider::new(&rocksdb_dir).expect("failed to create test RocksDB provider"), ) .expect("failed to create test provider factory") } diff --git a/crates/storage/provider/src/test_utils/noop.rs b/crates/storage/provider/src/test_utils/noop.rs index 85501f4e71..61609c3758 100644 --- a/crates/storage/provider/src/test_utils/noop.rs +++ b/crates/storage/provider/src/test_utils/noop.rs @@ -1,8 +1,8 @@ //! Additional testing support for `NoopProvider`. use crate::{ - providers::{StaticFileProvider, StaticFileProviderRWRefMut}, - StaticFileProviderFactory, + providers::{RocksDBProvider, StaticFileProvider, StaticFileProviderRWRefMut}, + RocksDBProviderFactory, StaticFileProviderFactory, }; use reth_errors::{ProviderError, ProviderResult}; use reth_primitives_traits::NodePrimitives; @@ -24,3 +24,9 @@ impl StaticFileProviderFactory for NoopProvid Err(ProviderError::ReadOnlyStaticFileAccess) } } + +impl RocksDBProviderFactory for NoopProvider { + fn rocksdb_provider(&self) -> RocksDBProvider { + RocksDBProvider::builder(PathBuf::default()).build().unwrap() + } +} diff --git a/crates/storage/provider/src/traits/full.rs b/crates/storage/provider/src/traits/full.rs index 6fe88a6640..c6a69e2fbf 100644 --- a/crates/storage/provider/src/traits/full.rs +++ b/crates/storage/provider/src/traits/full.rs @@ -2,8 +2,9 @@ use crate::{ AccountReader, BlockReader, BlockReaderIdExt, ChainSpecProvider, ChangeSetReader, - DatabaseProviderFactory, HashedPostStateProvider, PruneCheckpointReader, StageCheckpointReader, - StateProviderFactory, StateReader, StaticFileProviderFactory, TrieReader, + DatabaseProviderFactory, HashedPostStateProvider, PruneCheckpointReader, + RocksDBProviderFactory, StageCheckpointReader, StateProviderFactory, StateReader, + StaticFileProviderFactory, TrieReader, }; use reth_chain_state::{CanonStateSubscriptions, ForkChoiceSubscriptions}; use reth_node_types::{BlockTy, HeaderTy, NodeTypesWithDB, ReceiptTy, TxTy}; @@ -17,6 +18,7 @@ pub trait FullProvider: Provider: BlockReader + TrieReader + StageCheckpointReader + PruneCheckpointReader, > + NodePrimitivesProvider + StaticFileProviderFactory + + RocksDBProviderFactory + BlockReaderIdExt< Transaction = TxTy, Block = BlockTy, @@ -44,6 +46,7 @@ impl FullProvider for T where Provider: BlockReader + TrieReader + StageCheckpointReader + PruneCheckpointReader, > + NodePrimitivesProvider + StaticFileProviderFactory + + RocksDBProviderFactory + BlockReaderIdExt< Transaction = TxTy, Block = BlockTy, diff --git a/crates/storage/provider/src/traits/mod.rs b/crates/storage/provider/src/traits/mod.rs index 2837a4505f..1e43cdbbd7 100644 --- a/crates/storage/provider/src/traits/mod.rs +++ b/crates/storage/provider/src/traits/mod.rs @@ -8,5 +8,8 @@ pub use reth_chainspec::ChainSpecProvider; mod static_file_provider; pub use static_file_provider::StaticFileProviderFactory; +mod rocksdb_provider; +pub use rocksdb_provider::RocksDBProviderFactory; + mod full; pub use full::FullProvider; diff --git a/crates/storage/provider/src/traits/rocksdb_provider.rs b/crates/storage/provider/src/traits/rocksdb_provider.rs new file mode 100644 index 0000000000..c1ffcd4358 --- /dev/null +++ b/crates/storage/provider/src/traits/rocksdb_provider.rs @@ -0,0 +1,9 @@ +use crate::providers::RocksDBProvider; + +/// `RocksDB` provider factory. +/// +/// This trait provides access to the `RocksDB` provider +pub trait RocksDBProviderFactory { + /// Returns the `RocksDB` provider. + fn rocksdb_provider(&self) -> RocksDBProvider; +} diff --git a/docs/vocs/docs/pages/cli/op-reth/db.mdx b/docs/vocs/docs/pages/cli/op-reth/db.mdx index 4de2e9c0bd..be7e33ec07 100644 --- a/docs/vocs/docs/pages/cli/op-reth/db.mdx +++ b/docs/vocs/docs/pages/cli/op-reth/db.mdx @@ -43,6 +43,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + --config The path to the configuration file to use diff --git a/docs/vocs/docs/pages/cli/op-reth/import-op.mdx b/docs/vocs/docs/pages/cli/op-reth/import-op.mdx index 4468b8f1c7..d60abada8d 100644 --- a/docs/vocs/docs/pages/cli/op-reth/import-op.mdx +++ b/docs/vocs/docs/pages/cli/op-reth/import-op.mdx @@ -27,6 +27,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + --config The path to the configuration file to use diff --git a/docs/vocs/docs/pages/cli/op-reth/import-receipts-op.mdx b/docs/vocs/docs/pages/cli/op-reth/import-receipts-op.mdx index 15a61e503c..85be78419d 100644 --- a/docs/vocs/docs/pages/cli/op-reth/import-receipts-op.mdx +++ b/docs/vocs/docs/pages/cli/op-reth/import-receipts-op.mdx @@ -27,6 +27,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + --config The path to the configuration file to use diff --git a/docs/vocs/docs/pages/cli/op-reth/init-state.mdx b/docs/vocs/docs/pages/cli/op-reth/init-state.mdx index 6643ecf004..f00042eb94 100644 --- a/docs/vocs/docs/pages/cli/op-reth/init-state.mdx +++ b/docs/vocs/docs/pages/cli/op-reth/init-state.mdx @@ -27,6 +27,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + --config The path to the configuration file to use diff --git a/docs/vocs/docs/pages/cli/op-reth/init.mdx b/docs/vocs/docs/pages/cli/op-reth/init.mdx index 09e28c848d..03384822a9 100644 --- a/docs/vocs/docs/pages/cli/op-reth/init.mdx +++ b/docs/vocs/docs/pages/cli/op-reth/init.mdx @@ -27,6 +27,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + --config The path to the configuration file to use diff --git a/docs/vocs/docs/pages/cli/op-reth/node.mdx b/docs/vocs/docs/pages/cli/op-reth/node.mdx index da2ed9ff28..ee1adfee78 100644 --- a/docs/vocs/docs/pages/cli/op-reth/node.mdx +++ b/docs/vocs/docs/pages/cli/op-reth/node.mdx @@ -71,6 +71,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + Networking: -d, --disable-discovery Disable the discovery service diff --git a/docs/vocs/docs/pages/cli/op-reth/p2p/body.mdx b/docs/vocs/docs/pages/cli/op-reth/p2p/body.mdx index b88f2f3bbb..de1b963862 100644 --- a/docs/vocs/docs/pages/cli/op-reth/p2p/body.mdx +++ b/docs/vocs/docs/pages/cli/op-reth/p2p/body.mdx @@ -248,6 +248,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + --config The path to the configuration file to use. diff --git a/docs/vocs/docs/pages/cli/op-reth/p2p/header.mdx b/docs/vocs/docs/pages/cli/op-reth/p2p/header.mdx index 64ae5b2427..d176568dd5 100644 --- a/docs/vocs/docs/pages/cli/op-reth/p2p/header.mdx +++ b/docs/vocs/docs/pages/cli/op-reth/p2p/header.mdx @@ -248,6 +248,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + --config The path to the configuration file to use. diff --git a/docs/vocs/docs/pages/cli/op-reth/prune.mdx b/docs/vocs/docs/pages/cli/op-reth/prune.mdx index 2c8e27eafb..ef0783da58 100644 --- a/docs/vocs/docs/pages/cli/op-reth/prune.mdx +++ b/docs/vocs/docs/pages/cli/op-reth/prune.mdx @@ -27,6 +27,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + --config The path to the configuration file to use diff --git a/docs/vocs/docs/pages/cli/op-reth/re-execute.mdx b/docs/vocs/docs/pages/cli/op-reth/re-execute.mdx index a38edea8f5..bd95fbab68 100644 --- a/docs/vocs/docs/pages/cli/op-reth/re-execute.mdx +++ b/docs/vocs/docs/pages/cli/op-reth/re-execute.mdx @@ -27,6 +27,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + --config The path to the configuration file to use diff --git a/docs/vocs/docs/pages/cli/op-reth/stage/drop.mdx b/docs/vocs/docs/pages/cli/op-reth/stage/drop.mdx index 1bda9307ad..5f96ed8ab6 100644 --- a/docs/vocs/docs/pages/cli/op-reth/stage/drop.mdx +++ b/docs/vocs/docs/pages/cli/op-reth/stage/drop.mdx @@ -27,6 +27,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + --config The path to the configuration file to use diff --git a/docs/vocs/docs/pages/cli/op-reth/stage/dump.mdx b/docs/vocs/docs/pages/cli/op-reth/stage/dump.mdx index 6987e59224..910ab7666f 100644 --- a/docs/vocs/docs/pages/cli/op-reth/stage/dump.mdx +++ b/docs/vocs/docs/pages/cli/op-reth/stage/dump.mdx @@ -34,6 +34,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + --config The path to the configuration file to use diff --git a/docs/vocs/docs/pages/cli/op-reth/stage/run.mdx b/docs/vocs/docs/pages/cli/op-reth/stage/run.mdx index e13c39e0be..c57a221e5f 100644 --- a/docs/vocs/docs/pages/cli/op-reth/stage/run.mdx +++ b/docs/vocs/docs/pages/cli/op-reth/stage/run.mdx @@ -27,6 +27,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + --config The path to the configuration file to use diff --git a/docs/vocs/docs/pages/cli/op-reth/stage/unwind.mdx b/docs/vocs/docs/pages/cli/op-reth/stage/unwind.mdx index c4aac0de47..d53a6e5b46 100644 --- a/docs/vocs/docs/pages/cli/op-reth/stage/unwind.mdx +++ b/docs/vocs/docs/pages/cli/op-reth/stage/unwind.mdx @@ -32,6 +32,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + --config The path to the configuration file to use diff --git a/docs/vocs/docs/pages/cli/reth/db.mdx b/docs/vocs/docs/pages/cli/reth/db.mdx index 90ea9d6881..da580f7e8e 100644 --- a/docs/vocs/docs/pages/cli/reth/db.mdx +++ b/docs/vocs/docs/pages/cli/reth/db.mdx @@ -43,6 +43,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + --config The path to the configuration file to use diff --git a/docs/vocs/docs/pages/cli/reth/download.mdx b/docs/vocs/docs/pages/cli/reth/download.mdx index 785213182a..9d897b8561 100644 --- a/docs/vocs/docs/pages/cli/reth/download.mdx +++ b/docs/vocs/docs/pages/cli/reth/download.mdx @@ -27,6 +27,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + --config The path to the configuration file to use diff --git a/docs/vocs/docs/pages/cli/reth/export-era.mdx b/docs/vocs/docs/pages/cli/reth/export-era.mdx index 02ac02965d..f43b294766 100644 --- a/docs/vocs/docs/pages/cli/reth/export-era.mdx +++ b/docs/vocs/docs/pages/cli/reth/export-era.mdx @@ -27,6 +27,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + --config The path to the configuration file to use diff --git a/docs/vocs/docs/pages/cli/reth/import-era.mdx b/docs/vocs/docs/pages/cli/reth/import-era.mdx index ece330c642..0528b83157 100644 --- a/docs/vocs/docs/pages/cli/reth/import-era.mdx +++ b/docs/vocs/docs/pages/cli/reth/import-era.mdx @@ -27,6 +27,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + --config The path to the configuration file to use diff --git a/docs/vocs/docs/pages/cli/reth/import.mdx b/docs/vocs/docs/pages/cli/reth/import.mdx index 25f96ac32a..79fc123c1f 100644 --- a/docs/vocs/docs/pages/cli/reth/import.mdx +++ b/docs/vocs/docs/pages/cli/reth/import.mdx @@ -27,6 +27,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + --config The path to the configuration file to use diff --git a/docs/vocs/docs/pages/cli/reth/init-state.mdx b/docs/vocs/docs/pages/cli/reth/init-state.mdx index 9819039691..471672c7a8 100644 --- a/docs/vocs/docs/pages/cli/reth/init-state.mdx +++ b/docs/vocs/docs/pages/cli/reth/init-state.mdx @@ -27,6 +27,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + --config The path to the configuration file to use diff --git a/docs/vocs/docs/pages/cli/reth/init.mdx b/docs/vocs/docs/pages/cli/reth/init.mdx index d1465158f5..4e55d4f169 100644 --- a/docs/vocs/docs/pages/cli/reth/init.mdx +++ b/docs/vocs/docs/pages/cli/reth/init.mdx @@ -27,6 +27,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + --config The path to the configuration file to use diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index d30216611a..38b59183e4 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -71,6 +71,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + Networking: -d, --disable-discovery Disable the discovery service diff --git a/docs/vocs/docs/pages/cli/reth/p2p/body.mdx b/docs/vocs/docs/pages/cli/reth/p2p/body.mdx index 0b351b18c9..ccbdfdcbfe 100644 --- a/docs/vocs/docs/pages/cli/reth/p2p/body.mdx +++ b/docs/vocs/docs/pages/cli/reth/p2p/body.mdx @@ -248,6 +248,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + --config The path to the configuration file to use. diff --git a/docs/vocs/docs/pages/cli/reth/p2p/header.mdx b/docs/vocs/docs/pages/cli/reth/p2p/header.mdx index bda050fcbc..f83bb8f14a 100644 --- a/docs/vocs/docs/pages/cli/reth/p2p/header.mdx +++ b/docs/vocs/docs/pages/cli/reth/p2p/header.mdx @@ -248,6 +248,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + --config The path to the configuration file to use. diff --git a/docs/vocs/docs/pages/cli/reth/prune.mdx b/docs/vocs/docs/pages/cli/reth/prune.mdx index 92aa365b8f..e5b143e133 100644 --- a/docs/vocs/docs/pages/cli/reth/prune.mdx +++ b/docs/vocs/docs/pages/cli/reth/prune.mdx @@ -27,6 +27,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + --config The path to the configuration file to use diff --git a/docs/vocs/docs/pages/cli/reth/re-execute.mdx b/docs/vocs/docs/pages/cli/reth/re-execute.mdx index 2743f4b491..4b455a9bff 100644 --- a/docs/vocs/docs/pages/cli/reth/re-execute.mdx +++ b/docs/vocs/docs/pages/cli/reth/re-execute.mdx @@ -27,6 +27,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + --config The path to the configuration file to use diff --git a/docs/vocs/docs/pages/cli/reth/stage/drop.mdx b/docs/vocs/docs/pages/cli/reth/stage/drop.mdx index 35a051f859..55ebb4725e 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/drop.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/drop.mdx @@ -27,6 +27,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + --config The path to the configuration file to use diff --git a/docs/vocs/docs/pages/cli/reth/stage/dump.mdx b/docs/vocs/docs/pages/cli/reth/stage/dump.mdx index 0ab8ab2d33..d0f2666979 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/dump.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/dump.mdx @@ -34,6 +34,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + --config The path to the configuration file to use diff --git a/docs/vocs/docs/pages/cli/reth/stage/run.mdx b/docs/vocs/docs/pages/cli/reth/stage/run.mdx index 41f083bc6f..f0e4f06bba 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/run.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/run.mdx @@ -27,6 +27,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + --config The path to the configuration file to use diff --git a/docs/vocs/docs/pages/cli/reth/stage/unwind.mdx b/docs/vocs/docs/pages/cli/reth/stage/unwind.mdx index 1460b4c81d..2ba873f682 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/unwind.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/unwind.mdx @@ -32,6 +32,9 @@ Datadir: --datadir.static-files The absolute path to store static files in. + --datadir.rocksdb + The absolute path to store `RocksDB` database in. + --config The path to the configuration file to use diff --git a/examples/rpc-db/src/main.rs b/examples/rpc-db/src/main.rs index 95edc54c42..4a0339fd00 100644 --- a/examples/rpc-db/src/main.rs +++ b/examples/rpc-db/src/main.rs @@ -24,7 +24,7 @@ use reth_ethereum::{ pool::noop::NoopTransactionPool, provider::{ db::{mdbx::DatabaseArguments, open_db_read_only, ClientVersion, DatabaseEnv}, - providers::{BlockchainProvider, StaticFileProvider}, + providers::{BlockchainProvider, RocksDBProvider, StaticFileProvider}, ProviderFactory, }, rpc::{ @@ -53,6 +53,7 @@ async fn main() -> eyre::Result<()> { db.clone(), spec.clone(), StaticFileProvider::read_only(db_path.join("static_files"), true)?, + RocksDBProvider::builder(db_path.join("rocksdb")).build().unwrap(), )?; // 2. Set up the blockchain provider using only the database provider and a noop for the tree to