chore: split db abstraction into new crate (#8594)

This commit is contained in:
Oliver
2024-06-04 23:45:57 +02:00
committed by GitHub
parent a8095740fc
commit 51a28f22da
183 changed files with 825 additions and 755 deletions

View File

@@ -17,6 +17,7 @@ reth-primitives.workspace = true
reth-storage-errors.workspace = true
reth-execution-errors.workspace = true
reth-db.workspace = true
reth-db-api.workspace = true
reth-evm.workspace = true
reth-revm.workspace = true
reth-provider.workspace = true
@@ -41,7 +42,7 @@ linked_hash_set = "0.1.4"
[dev-dependencies]
reth-db = { workspace = true, features = ["test-utils"] }
reth-primitives = { workspace = true , features = ["test-utils"] }
reth-primitives = { workspace = true, features = ["test-utils"] }
reth-provider = { workspace = true, features = ["test-utils"] }
reth-evm = { workspace = true, features = ["test-utils"] }
reth-testing-utils.workspace = true

View File

@@ -10,7 +10,7 @@ use reth_blockchain_tree_api::{
BlockAttachment, BlockStatus, BlockValidationKind, CanonicalOutcome, InsertPayloadOk,
};
use reth_consensus::{Consensus, ConsensusError};
use reth_db::database::Database;
use reth_db_api::database::Database;
use reth_evm::execute::BlockExecutorProvider;
use reth_execution_errors::{BlockExecutionError, BlockValidationError};
use reth_primitives::{
@@ -1377,7 +1377,8 @@ mod tests {
use assert_matches::assert_matches;
use linked_hash_set::LinkedHashSet;
use reth_consensus::test_utils::TestConsensus;
use reth_db::{tables, test_utils::TempDatabase, transaction::DbTxMut, DatabaseEnv};
use reth_db::{tables, test_utils::TempDatabase, DatabaseEnv};
use reth_db_api::transaction::DbTxMut;
use reth_evm::test_utils::MockExecutorProvider;
use reth_evm_ethereum::execute::EthExecutorProvider;
#[cfg(not(feature = "optimism"))]

View File

@@ -10,7 +10,7 @@ use reth_blockchain_tree_api::{
BlockAttachment, BlockValidationKind,
};
use reth_consensus::{Consensus, ConsensusError, PostExecutionInput};
use reth_db::database::Database;
use reth_db_api::database::Database;
use reth_evm::execute::{BlockExecutionOutput, BlockExecutorProvider, Executor};
use reth_execution_errors::BlockExecutionError;
use reth_primitives::{

View File

@@ -1,9 +1,8 @@
//! Blockchain tree externals.
use reth_consensus::Consensus;
use reth_db::{
cursor::DbCursorRO, database::Database, static_file::HeaderMask, tables, transaction::DbTx,
};
use reth_db::{static_file::HeaderMask, tables};
use reth_db_api::{cursor::DbCursorRO, database::Database, transaction::DbTx};
use reth_primitives::{BlockHash, BlockNumber, StaticFileSegment};
use reth_provider::{ProviderFactory, StaticFileProviderFactory, StatsReader};
use reth_storage_errors::provider::ProviderResult;

View File

@@ -7,7 +7,7 @@ use reth_blockchain_tree_api::{
BlockValidationKind, BlockchainTreeEngine, BlockchainTreeViewer, CanonicalOutcome,
InsertPayloadOk,
};
use reth_db::database::Database;
use reth_db_api::database::Database;
use reth_evm::execute::BlockExecutorProvider;
use reth_primitives::{
BlockHash, BlockNumHash, BlockNumber, Receipt, SealedBlock, SealedBlockWithSenders,

View File

@@ -17,7 +17,7 @@ reth-blockchain-tree-api.workspace = true
reth-primitives.workspace = true
reth-stages-api.workspace = true
reth-errors.workspace = true
reth-db.workspace = true
reth-db-api.workspace = true
reth-provider.workspace = true
reth-rpc-types.workspace = true
reth-tasks.workspace = true
@@ -74,5 +74,5 @@ optimism = [
"reth-provider/optimism",
"reth-blockchain-tree/optimism",
"reth-ethereum-consensus/optimism",
"reth-rpc/optimism"
"reth-rpc/optimism",
]

View File

@@ -6,7 +6,7 @@ use crate::{
};
use futures::FutureExt;
use metrics::Counter;
use reth_db::database::Database;
use reth_db_api::database::Database;
use reth_errors::{RethError, RethResult};
use reth_primitives::BlockNumber;
use reth_prune::{Pruner, PrunerError, PrunerWithResult};

View File

@@ -5,7 +5,7 @@ use crate::{
hooks::EngineHookDBAccessLevel,
};
use futures::FutureExt;
use reth_db::database::Database;
use reth_db_api::database::Database;
use reth_errors::RethResult;
use reth_primitives::{static_file::HighestStaticFiles, BlockNumber};
use reth_static_file::{StaticFileProducer, StaticFileProducerWithResult};

View File

@@ -4,7 +4,7 @@ use reth_blockchain_tree_api::{
error::{BlockchainTreeError, CanonicalError, InsertBlockError, InsertBlockErrorKind},
BlockStatus, BlockValidationKind, BlockchainTreeEngine, CanonicalOutcome, InsertPayloadOk,
};
use reth_db::database::Database;
use reth_db_api::database::Database;
use reth_engine_primitives::{EngineTypes, PayloadAttributes, PayloadBuilderAttributes};
use reth_errors::{BlockValidationError, ProviderResult, RethError, RethResult};
use reth_network_p2p::{
@@ -2178,7 +2178,8 @@ mod tests {
mod fork_choice_updated {
use super::*;
use reth_db::{tables, test_utils::create_test_static_files_dir, transaction::DbTxMut};
use reth_db::{tables, test_utils::create_test_static_files_dir};
use reth_db_api::transaction::DbTxMut;
use reth_primitives::U256;
use reth_provider::providers::StaticFileProvider;
use reth_rpc_types::engine::ForkchoiceUpdateError;

View File

@@ -5,7 +5,7 @@ use crate::{
ConsensusEngineLiveSyncProgress, EthBeaconConsensus,
};
use futures::FutureExt;
use reth_db::database::Database;
use reth_db_api::database::Database;
use reth_network_p2p::{
bodies::client::BodiesClient,
full_block::{FetchFullBlockFuture, FetchFullBlockRangeFuture, FullBlockClient},

View File

@@ -9,7 +9,7 @@ repository.workspace = true
[dependencies]
tempfile.workspace = true
reth-db.workspace = true
reth-db-api.workspace = true
rayon.workspace = true
[dev-dependencies]

View File

@@ -22,7 +22,7 @@ use std::{
};
use rayon::prelude::*;
use reth_db::table::{Compress, Encode, Key, Value};
use reth_db_api::table::{Compress, Encode, Key, Value};
use tempfile::{NamedTempFile, TempDir};
/// An ETL (extract, transform, load) data collector.

View File

@@ -23,6 +23,7 @@ reth-network-types.workspace = true
# optional deps for the test-utils feature
reth-db = { workspace = true, optional = true }
reth-db-api = { workspace = true, optional = true }
reth-testing-utils = { workspace = true, optional = true }
# eth
@@ -50,6 +51,7 @@ itertools.workspace = true
[dev-dependencies]
reth-db = { workspace = true, features = ["test-utils"] }
reth-db-api.workspace = true
reth-consensus = { workspace = true, features = ["test-utils"] }
reth-network-p2p = { workspace = true, features = ["test-utils"] }
reth-provider = { workspace = true, features = ["test-utils"] }
@@ -65,5 +67,11 @@ rand.workspace = true
tempfile.workspace = true
[features]
test-utils = ["dep:tempfile", "reth-db/test-utils", "reth-consensus/test-utils", "reth-network-p2p/test-utils", "reth-testing-utils"]
test-utils = [
"dep:tempfile",
"dep:reth-db-api",
"reth-db/test-utils",
"reth-consensus/test-utils",
"reth-network-p2p/test-utils",
"reth-testing-utils",
]

View File

@@ -2,7 +2,8 @@
#![allow(dead_code)]
use reth_db::{database::Database, tables, transaction::DbTxMut, DatabaseEnv};
use reth_db::{tables, DatabaseEnv};
use reth_db_api::{database::Database, transaction::DbTxMut};
use reth_network_p2p::bodies::response::BlockResponse;
use reth_primitives::{Block, BlockBody, SealedBlock, SealedHeader, B256};
use std::collections::HashMap;

View File

@@ -15,6 +15,7 @@ workspace = true
reth-primitives.workspace = true
reth-fs-util.workspace = true
reth-db = { workspace = true, features = ["mdbx"] }
reth-db-api.workspace = true
reth-storage-errors = { workspace = true, features = ["clap"] }
reth-provider.workspace = true
reth-network = { workspace = true, features = ["serde"] }

View File

@@ -9,7 +9,7 @@ use hyper::{
use metrics::describe_gauge;
use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle};
use metrics_util::layers::{PrefixLayer, Stack};
use reth_db::database_metrics::DatabaseMetrics;
use reth_db_api::database_metrics::DatabaseMetrics;
use reth_metrics::metrics::Unit;
use reth_provider::providers::StaticFileProvider;
use reth_tasks::TaskExecutor;

View File

@@ -13,7 +13,7 @@ use discv5::ListenConfig;
use metrics_exporter_prometheus::PrometheusHandle;
use once_cell::sync::Lazy;
use reth_config::{config::PruneConfig, Config};
use reth_db::{database::Database, database_metrics::DatabaseMetrics};
use reth_db_api::{database::Database, database_metrics::DatabaseMetrics};
use reth_network::{NetworkBuilder, NetworkConfig, NetworkManager};
use reth_network_p2p::headers::client::HeadersClient;
use reth_primitives::{

View File

@@ -1,5 +1,5 @@
//! Version information for reth.
use reth_db::models::client_version::ClientVersion;
use reth_db_api::models::ClientVersion;
use reth_rpc_types::engine::ClientCode;
/// The client code for Reth

View File

@@ -14,7 +14,7 @@ workspace = true
# reth
reth-evm.workspace = true
reth-provider.workspace = true
reth-db.workspace = true
reth-db-api.workspace = true
reth-engine-primitives.workspace = true
reth-transaction-pool.workspace = true
reth-network.workspace = true

View File

@@ -1,7 +1,7 @@
//! Traits for configuring a node.
use crate::{primitives::NodePrimitives, ConfigureEvm, EngineTypes};
use reth_db::{
use reth_db_api::{
database::Database,
database_metrics::{DatabaseMetadata, DatabaseMetrics},
};

View File

@@ -21,6 +21,7 @@ reth-exex.workspace = true
reth-evm.workspace = true
reth-provider.workspace = true
reth-db.workspace = true
reth-db-api.workspace = true
reth-rpc-engine-api.workspace = true
reth-rpc.workspace = true
reth-rpc-layer.workspace = true

View File

@@ -10,11 +10,13 @@ use crate::{
};
use futures::Future;
use reth_db::{
database::Database,
database_metrics::{DatabaseMetadata, DatabaseMetrics},
test_utils::{create_test_rw_db_with_path, tempdir_path, TempDatabase},
DatabaseEnv,
};
use reth_db_api::{
database::Database,
database_metrics::{DatabaseMetadata, DatabaseMetrics},
};
use reth_exex::ExExContext;
use reth_network::{NetworkBuilder, NetworkConfig, NetworkHandle};
use reth_node_api::{FullNodeTypes, FullNodeTypesAdapter, NodeTypes};

View File

@@ -5,7 +5,7 @@ use rayon::ThreadPoolBuilder;
use reth_auto_seal_consensus::MiningMode;
use reth_beacon_consensus::EthBeaconConsensus;
use reth_config::{config::EtlConfig, PruneConfig};
use reth_db::{database::Database, database_metrics::DatabaseMetrics};
use reth_db_api::{database::Database, database_metrics::DatabaseMetrics};
use reth_db_common::init::{init_genesis, InitDatabaseError};
use reth_downloaders::{bodies::noop::NoopBodiesDownloader, headers::noop::NoopHeaderDownloader};
use reth_evm::noop::NoopBlockExecutorProvider;

View File

@@ -2,7 +2,7 @@
use reth_config::{config::StageConfig, PruneConfig};
use reth_consensus::Consensus;
use reth_db::database::Database;
use reth_db_api::database::Database;
use reth_downloaders::{
bodies::bodies::BodiesDownloaderBuilder,
headers::reverse_headers::ReverseHeadersDownloaderBuilder,

View File

@@ -18,7 +18,7 @@ reth-network-api.workspace = true
reth-stages.workspace = true
reth-prune.workspace = true
reth-static-file.workspace = true
reth-db.workspace = true
reth-db-api.workspace = true
reth-primitives.workspace = true
reth-rpc-types.workspace = true

View File

@@ -5,7 +5,7 @@ use futures::Stream;
use reth_beacon_consensus::{
BeaconConsensusEngineEvent, ConsensusEngineLiveSyncProgress, ForkchoiceStatus,
};
use reth_db::{database::Database, database_metrics::DatabaseMetadata};
use reth_db_api::{database::Database, database_metrics::DatabaseMetadata};
use reth_network::{NetworkEvent, NetworkHandle};
use reth_network_api::PeersInfo;
use reth_primitives::{

View File

@@ -15,6 +15,7 @@ workspace = true
# reth
reth-primitives.workspace = true
reth-db.workspace = true
reth-db-api.workspace = true
reth-errors.workspace = true
reth-provider.workspace = true
reth-tokio-util.workspace = true

View File

@@ -1,6 +1,6 @@
use crate::{segments::SegmentSet, Pruner};
use reth_config::PruneConfig;
use reth_db::database::Database;
use reth_db_api::database::Database;
use reth_primitives::{FinishedExExHeight, PruneModes, MAINNET};
use reth_provider::ProviderFactory;
use std::time::Duration;

View File

@@ -5,7 +5,7 @@ use crate::{
segments::{PruneInput, Segment},
Metrics, PrunerError, PrunerEvent,
};
use reth_db::database::Database;
use reth_db_api::database::Database;
use reth_primitives::{
BlockNumber, FinishedExExHeight, PruneLimiter, PruneMode, PruneProgress, PrunePurpose,
PruneSegment, StaticFileSegment,

View File

@@ -4,7 +4,8 @@ use crate::{
},
PrunerError,
};
use reth_db::{database::Database, models::ShardedKey, tables};
use reth_db::tables;
use reth_db_api::{database::Database, models::ShardedKey};
use reth_primitives::{PruneInterruptReason, PruneMode, PruneProgress, PruneSegment};
use reth_provider::DatabaseProviderRW;
use tracing::{instrument, trace};

View File

@@ -5,10 +5,10 @@ use crate::{
PrunerError,
};
use itertools::Itertools;
use reth_db::{
use reth_db::tables;
use reth_db_api::{
cursor::{DbCursorRO, RangeWalker},
database::Database,
tables,
transaction::DbTxMut,
};
@@ -188,7 +188,8 @@ where
#[cfg(test)]
mod tests {
use assert_matches::assert_matches;
use reth_db::{tables, transaction::DbTx};
use reth_db::tables;
use reth_db_api::transaction::DbTx;
use reth_primitives::{
BlockNumber, PruneCheckpoint, PruneInterruptReason, PruneLimiter, PruneMode, PruneProgress,
PruneSegment, B256, U256,

View File

@@ -1,10 +1,11 @@
use reth_db::{
use reth_db::BlockNumberList;
use reth_db_api::{
cursor::{DbCursorRO, DbCursorRW},
database::Database,
models::ShardedKey,
table::Table,
transaction::DbTxMut,
BlockNumberList, DatabaseError,
DatabaseError,
};
use reth_primitives::BlockNumber;
use reth_provider::DatabaseProviderRW;

View File

@@ -14,7 +14,7 @@ pub use account_history::AccountHistory;
pub use headers::Headers;
pub use receipts::Receipts;
pub use receipts_by_logs::ReceiptsByLogs;
use reth_db::database::Database;
use reth_db_api::database::Database;
use reth_primitives::{
BlockNumber, PruneCheckpoint, PruneInterruptReason, PruneLimiter, PruneMode, PruneProgress,
PruneSegment, TxNumber,

View File

@@ -2,7 +2,8 @@ use crate::{
segments::{PruneInput, PruneOutput, PruneOutputCheckpoint, Segment},
PrunerError,
};
use reth_db::{database::Database, tables};
use reth_db::tables;
use reth_db_api::database::Database;
use reth_primitives::{PruneCheckpoint, PruneMode, PruneProgress, PruneSegment};
use reth_provider::{
errors::provider::ProviderResult, DatabaseProviderRW, PruneCheckpointWriter,

View File

@@ -2,7 +2,8 @@ use crate::{
segments::{PruneInput, PruneOutput, Segment},
PrunerError,
};
use reth_db::{database::Database, tables};
use reth_db::tables;
use reth_db_api::database::Database;
use reth_primitives::{
PruneCheckpoint, PruneMode, PruneProgress, PrunePurpose, PruneSegment, ReceiptsLogPruneConfig,
MINIMUM_PRUNING_DISTANCE,
@@ -216,7 +217,8 @@ impl<DB: Database> Segment<DB> for ReceiptsByLogs {
mod tests {
use crate::segments::{receipts_by_logs::ReceiptsByLogs, PruneInput, Segment};
use assert_matches::assert_matches;
use reth_db::{cursor::DbCursorRO, tables, transaction::DbTx};
use reth_db::tables;
use reth_db_api::{cursor::DbCursorRO, transaction::DbTx};
use reth_primitives::{PruneLimiter, PruneMode, PruneSegment, ReceiptsLogPruneConfig, B256};
use reth_provider::{PruneCheckpointReader, TransactionsProvider};
use reth_stages::test_utils::{StorageKind, TestStageDB};

View File

@@ -2,7 +2,8 @@ use crate::{
segments::{PruneInput, PruneOutput, PruneOutputCheckpoint, Segment},
PrunerError,
};
use reth_db::{database::Database, tables};
use reth_db::tables;
use reth_db_api::database::Database;
use reth_primitives::{PruneMode, PruneProgress, PruneSegment};
use reth_provider::{DatabaseProviderRW, TransactionsProvider};
use tracing::{instrument, trace};

View File

@@ -2,7 +2,7 @@ use crate::segments::{
AccountHistory, Receipts, ReceiptsByLogs, Segment, SenderRecovery, StorageHistory,
TransactionLookup,
};
use reth_db::database::Database;
use reth_db_api::database::Database;
use reth_primitives::PruneModes;
/// Collection of [Segment]. Thread-safe, allocated on the heap.

View File

@@ -4,10 +4,10 @@ use crate::{
},
PrunerError,
};
use reth_db::{
use reth_db::tables;
use reth_db_api::{
database::Database,
models::{storage_sharded_key::StorageShardedKey, BlockNumberAddress},
tables,
};
use reth_primitives::{PruneInterruptReason, PruneMode, PruneProgress, PruneSegment};
use reth_provider::DatabaseProviderRW;

View File

@@ -3,7 +3,8 @@ use crate::{
PrunerError,
};
use rayon::prelude::*;
use reth_db::{database::Database, tables};
use reth_db::tables;
use reth_db_api::database::Database;
use reth_primitives::{PruneMode, PruneProgress, PruneSegment};
use reth_provider::{DatabaseProviderRW, TransactionsProvider};
use tracing::{instrument, trace};

View File

@@ -2,7 +2,8 @@ use crate::{
segments::{PruneInput, PruneOutput, PruneOutputCheckpoint, Segment},
PrunerError,
};
use reth_db::{database::Database, tables};
use reth_db::tables;
use reth_db_api::database::Database;
use reth_primitives::{PruneMode, PruneProgress, PruneSegment};
use reth_provider::{DatabaseProviderRW, TransactionsProvider};
use tracing::{instrument, trace};

View File

@@ -14,7 +14,7 @@ workspace = true
# reth
reth-primitives.workspace = true
reth-provider.workspace = true
reth-db.workspace = true
reth-db-api.workspace = true
reth-static-file.workspace = true
reth-network-p2p.workspace = true
reth-tokio-util.workspace = true

View File

@@ -1,5 +1,5 @@
use crate::{pipeline::BoxedStage, MetricEventsSender, Pipeline, Stage, StageSet};
use reth_db::database::Database;
use reth_db_api::database::Database;
use reth_primitives::{stage::StageId, BlockNumber, B256};
use reth_provider::ProviderFactory;
use reth_static_file::StaticFileProducer;

View File

@@ -3,7 +3,7 @@ mod event;
pub use crate::pipeline::ctrl::ControlFlow;
pub use event::*;
use futures_util::Future;
use reth_db::database::Database;
use reth_db_api::database::Database;
use reth_primitives::{
constants::BEACON_CONSENSUS_REORG_UNWIND_DEPTH,
stage::{PipelineTarget, StageCheckpoint, StageId},

View File

@@ -1,5 +1,5 @@
use crate::Stage;
use reth_db::database::Database;
use reth_db_api::database::Database;
use reth_primitives::stage::StageId;
use std::{
collections::HashMap,

View File

@@ -1,5 +1,5 @@
use crate::error::StageError;
use reth_db::database::Database;
use reth_db_api::database::Database;
use reth_primitives::{
stage::{StageCheckpoint, StageId},
BlockNumber, TxNumber,

View File

@@ -1,7 +1,7 @@
#![allow(missing_docs)]
use crate::{ExecInput, ExecOutput, Stage, StageError, UnwindInput, UnwindOutput};
use reth_db::database::Database;
use reth_db_api::database::Database;
use reth_primitives::stage::StageId;
use reth_provider::DatabaseProviderRW;
use std::collections::VecDeque;

View File

@@ -17,6 +17,7 @@ reth-codecs.workspace = true
reth-config.workspace = true
reth-consensus.workspace = true
reth-db.workspace = true
reth-db-api.workspace = true
reth-etl.workspace = true
reth-evm.workspace = true
reth-exex.workspace = true

View File

@@ -1,8 +1,9 @@
#![allow(unreachable_pub)]
use super::constants;
use reth_db::{
cursor::DbCursorRO, database::Database, tables, transaction::DbTx, DatabaseError as DbError,
use reth_db::tables;
use reth_db_api::{
cursor::DbCursorRO, database::Database, transaction::DbTx, DatabaseError as DbError,
};
use reth_primitives::{stage::StageCheckpoint, BlockNumber};
use reth_stages::{

View File

@@ -1,11 +1,9 @@
#![allow(unreachable_pub)]
use itertools::concat;
use reth_db::{
use reth_db::{tables, test_utils::TempDatabase, DatabaseEnv};
use reth_db_api::{
cursor::DbCursorRO,
tables,
test_utils::TempDatabase,
transaction::{DbTx, DbTxMut},
DatabaseEnv,
};
use reth_primitives::{Account, Address, SealedBlock, B256, U256};
use reth_stages::{

View File

@@ -42,7 +42,7 @@ use crate::{
};
use reth_config::config::StageConfig;
use reth_consensus::Consensus;
use reth_db::database::Database;
use reth_db_api::database::Database;
use reth_evm::execute::BlockExecutorProvider;
use reth_network_p2p::{bodies::downloader::BodyDownloader, headers::downloader::HeaderDownloader};
use reth_primitives::PruneModes;

View File

@@ -6,11 +6,11 @@ use std::{
use futures_util::TryStreamExt;
use tracing::*;
use reth_db::{
use reth_db::tables;
use reth_db_api::{
cursor::{DbCursorRO, DbCursorRW},
database::Database,
models::{StoredBlockBodyIndices, StoredBlockOmmers, StoredBlockWithdrawals},
tables,
transaction::DbTxMut,
};
use reth_network_p2p::bodies::{downloader::BodyDownloader, response::BlockResponse};
@@ -614,24 +614,19 @@ mod tests {
}
mod test_utils {
use std::{
collections::{HashMap, VecDeque},
ops::RangeInclusive,
pin::Pin,
sync::Arc,
task::{Context, Poll},
use crate::{
stages::bodies::BodyStage,
test_utils::{
ExecuteStageTestRunner, StageTestRunner, TestRunnerError, TestStageDB,
UnwindStageTestRunner,
},
};
use futures_util::Stream;
use reth_db::{
use reth_db::{static_file::HeaderMask, tables, test_utils::TempDatabase, DatabaseEnv};
use reth_db_api::{
cursor::DbCursorRO,
models::{StoredBlockBodyIndices, StoredBlockOmmers},
static_file::HeaderMask,
tables,
test_utils::TempDatabase,
transaction::{DbTx, DbTxMut},
DatabaseEnv,
};
use reth_network_p2p::{
bodies::{
@@ -653,13 +648,12 @@ mod tests {
generators,
generators::{random_block_range, random_signed_tx},
};
use crate::{
stages::bodies::BodyStage,
test_utils::{
ExecuteStageTestRunner, StageTestRunner, TestRunnerError, TestStageDB,
UnwindStageTestRunner,
},
use std::{
collections::{HashMap, VecDeque},
ops::RangeInclusive,
pin::Pin,
sync::Arc,
task::{Context, Poll},
};
/// The block hash of the genesis block.

View File

@@ -1,9 +1,8 @@
use crate::stages::MERKLE_STAGE_DEFAULT_CLEAN_THRESHOLD;
use num_traits::Zero;
use reth_config::config::ExecutionConfig;
use reth_db::{
cursor::DbCursorRO, database::Database, static_file::HeaderMask, tables, transaction::DbTx,
};
use reth_db::{static_file::HeaderMask, tables};
use reth_db_api::{cursor::DbCursorRO, database::Database, transaction::DbTx};
use reth_evm::execute::{BatchBlockExecutionOutput, BatchExecutor, BlockExecutorProvider};
use reth_exex::{ExExManagerHandle, ExExNotification};
use reth_primitives::{
@@ -654,7 +653,7 @@ mod tests {
use crate::test_utils::TestStageDB;
use alloy_rlp::Decodable;
use assert_matches::assert_matches;
use reth_db::{models::AccountBeforeTx, transaction::DbTxMut};
use reth_db_api::{models::AccountBeforeTx, transaction::DbTxMut};
use reth_evm_ethereum::execute::EthExecutorProvider;
use reth_execution_errors::BlockValidationError;
use reth_primitives::{

View File

@@ -1,4 +1,4 @@
use reth_db::database::Database;
use reth_db_api::database::Database;
use reth_primitives::stage::{StageCheckpoint, StageId};
use reth_provider::DatabaseProviderRW;
use reth_stages_api::{ExecInput, ExecOutput, Stage, StageError, UnwindInput, UnwindOutput};

View File

@@ -1,11 +1,10 @@
use itertools::Itertools;
use reth_config::config::{EtlConfig, HashingConfig};
use reth_db::{
use reth_db::{tables, RawKey, RawTable, RawValue};
use reth_db_api::{
cursor::{DbCursorRO, DbCursorRW},
database::Database,
tables,
transaction::{DbTx, DbTxMut},
RawKey, RawTable, RawValue,
};
use reth_etl::Collector;
use reth_primitives::{
@@ -64,7 +63,7 @@ impl AccountHashingStage {
provider: &DatabaseProviderRW<DB>,
opts: SeedOpts,
) -> Result<Vec<(reth_primitives::Address, reth_primitives::Account)>, StageError> {
use reth_db::models::AccountBeforeTx;
use reth_db_api::models::AccountBeforeTx;
use reth_primitives::U256;
use reth_provider::providers::StaticFileWriter;
use reth_testing_utils::{

View File

@@ -1,12 +1,11 @@
use itertools::Itertools;
use reth_config::config::{EtlConfig, HashingConfig};
use reth_db::{
codecs::CompactU256,
use reth_db::tables;
use reth_db_api::{
cursor::{DbCursorRO, DbDupCursorRW},
database::Database,
models::BlockNumberAddress,
models::{BlockNumberAddress, CompactU256},
table::Decompress,
tables,
transaction::{DbTx, DbTxMut},
};
use reth_etl::Collector;
@@ -219,7 +218,7 @@ mod tests {
};
use assert_matches::assert_matches;
use rand::Rng;
use reth_db::{
use reth_db_api::{
cursor::{DbCursorRW, DbDupCursorRO},
models::StoredBlockBodyIndices,
};

View File

@@ -2,12 +2,11 @@ use futures_util::StreamExt;
use reth_codecs::Compact;
use reth_config::config::EtlConfig;
use reth_consensus::Consensus;
use reth_db::{
use reth_db::{tables, RawKey, RawTable, RawValue};
use reth_db_api::{
cursor::{DbCursorRO, DbCursorRW},
database::Database,
tables,
transaction::DbTxMut,
RawKey, RawTable, RawValue,
};
use reth_etl::Collector;
use reth_network_p2p::headers::{downloader::HeaderDownloader, error::HeadersDownloaderError};

View File

@@ -1,8 +1,7 @@
use super::{collect_history_indices, load_history_indices};
use reth_config::config::{EtlConfig, IndexHistoryConfig};
use reth_db::{
database::Database, models::ShardedKey, table::Decode, tables, transaction::DbTxMut,
};
use reth_db::tables;
use reth_db_api::{database::Database, models::ShardedKey, table::Decode, transaction::DbTxMut};
use reth_primitives::{
stage::{StageCheckpoint, StageId},
Address, PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment,
@@ -148,14 +147,14 @@ mod tests {
TestStageDB, UnwindStageTestRunner,
};
use itertools::Itertools;
use reth_db::{
use reth_db::BlockNumberList;
use reth_db_api::{
cursor::DbCursorRO,
models::{
sharded_key, sharded_key::NUM_OF_INDICES_IN_SHARD, AccountBeforeTx,
StoredBlockBodyIndices,
},
transaction::DbTx,
BlockNumberList,
};
use reth_primitives::{address, BlockNumber, B256};
use reth_provider::providers::StaticFileWriter;

View File

@@ -1,10 +1,10 @@
use super::{collect_history_indices, load_history_indices};
use reth_config::config::{EtlConfig, IndexHistoryConfig};
use reth_db::{
use reth_db::tables;
use reth_db_api::{
database::Database,
models::{storage_sharded_key::StorageShardedKey, AddressStorageKey, BlockNumberAddress},
table::Decode,
tables,
transaction::DbTxMut,
};
use reth_primitives::{
@@ -155,14 +155,14 @@ mod tests {
TestStageDB, UnwindStageTestRunner,
};
use itertools::Itertools;
use reth_db::{
use reth_db::BlockNumberList;
use reth_db_api::{
cursor::DbCursorRO,
models::{
sharded_key, storage_sharded_key::NUM_OF_INDICES_IN_SHARD, ShardedKey,
StoredBlockBodyIndices,
},
transaction::DbTx,
BlockNumberList,
};
use reth_primitives::{address, b256, Address, BlockNumber, StorageEntry, B256, U256};
use reth_provider::providers::StaticFileWriter;

View File

@@ -1,8 +1,8 @@
use reth_codecs::Compact;
use reth_consensus::ConsensusError;
use reth_db::{
use reth_db::tables;
use reth_db_api::{
database::Database,
tables,
transaction::{DbTx, DbTxMut},
};
use reth_primitives::{
@@ -367,7 +367,7 @@ mod tests {
TestRunnerError, TestStageDB, UnwindStageTestRunner,
};
use assert_matches::assert_matches;
use reth_db::cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO};
use reth_db_api::cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO};
use reth_primitives::{
keccak256, stage::StageUnitCheckpoint, SealedBlock, StaticFileSegment, StorageEntry, U256,
};

View File

@@ -43,14 +43,16 @@ mod tests {
use crate::test_utils::{StorageKind, TestStageDB};
use alloy_rlp::Decodable;
use reth_db::{
cursor::{DbCursorRO, DbCursorRW},
mdbx::{cursor::Cursor, RW},
table::Table,
tables,
test_utils::TempDatabase,
transaction::{DbTx, DbTxMut},
AccountsHistory, DatabaseEnv,
};
use reth_db_api::{
cursor::{DbCursorRO, DbCursorRW},
table::Table,
transaction::{DbTx, DbTxMut},
};
use reth_evm_ethereum::execute::EthExecutorProvider;
use reth_exex::ExExManagerHandle;
use reth_primitives::{

View File

@@ -1,12 +1,10 @@
use reth_config::config::SenderRecoveryConfig;
use reth_consensus::ConsensusError;
use reth_db::{
use reth_db::{static_file::TransactionMask, tables, RawValue};
use reth_db_api::{
cursor::DbCursorRW,
database::Database,
static_file::TransactionMask,
tables,
transaction::{DbTx, DbTxMut},
RawValue,
};
use reth_primitives::{
stage::{EntitiesCheckpoint, StageCheckpoint, StageId},
@@ -284,7 +282,7 @@ struct FailedSenderRecoveryError {
#[cfg(test)]
mod tests {
use assert_matches::assert_matches;
use reth_db::cursor::DbCursorRO;
use reth_db_api::cursor::DbCursorRO;
use reth_primitives::{
stage::StageUnitCheckpoint, BlockNumber, PruneCheckpoint, PruneMode, SealedBlock,
TransactionSigned, B256,

View File

@@ -1,11 +1,10 @@
use num_traits::Zero;
use reth_config::config::{EtlConfig, TransactionLookupConfig};
use reth_db::{
use reth_db::{tables, RawKey, RawValue};
use reth_db_api::{
cursor::{DbCursorRO, DbCursorRW},
database::Database,
tables,
transaction::{DbTx, DbTxMut},
RawKey, RawValue,
};
use reth_etl::Collector;
use reth_primitives::{

View File

@@ -1,11 +1,12 @@
//! Utils for `stages`.
use reth_config::config::EtlConfig;
use reth_db::{
use reth_db::BlockNumberList;
use reth_db_api::{
cursor::{DbCursorRO, DbCursorRW},
models::sharded_key::NUM_OF_INDICES_IN_SHARD,
table::{Decompress, Table},
transaction::{DbTx, DbTxMut},
BlockNumberList, DatabaseError,
DatabaseError,
};
use reth_etl::Collector;
use reth_primitives::BlockNumber;

View File

@@ -1,6 +1,6 @@
use super::TEST_STAGE_ID;
use crate::{StageSet, StageSetBuilder};
use reth_db::database::Database;
use reth_db_api::database::Database;
use reth_stages_api::{test_utils::TestStage, ExecOutput, StageError, UnwindOutput};
use std::collections::VecDeque;

View File

@@ -1,15 +1,18 @@
use reth_db::{
tables,
test_utils::{
create_test_rw_db, create_test_rw_db_with_path, create_test_static_files_dir, TempDatabase,
},
DatabaseEnv,
};
use reth_db_api::{
common::KeyValue,
cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO},
database::Database,
models::{AccountBeforeTx, StoredBlockBodyIndices},
table::Table,
tables,
test_utils::{
create_test_rw_db, create_test_rw_db_with_path, create_test_static_files_dir, TempDatabase,
},
transaction::{DbTx, DbTxMut},
DatabaseEnv, DatabaseError as DbError,
DatabaseError as DbError,
};
use reth_primitives::{
keccak256, Account, Address, BlockNumber, Receipt, SealedBlock, SealedHeader,

View File

@@ -15,6 +15,7 @@ workspace = true
# reth
reth-primitives.workspace = true
reth-db.workspace = true
reth-db-api.workspace = true
reth-provider.workspace = true
reth-storage-errors.workspace = true
reth-nippy-jar.workspace = true

View File

@@ -1,8 +1,6 @@
use crate::segments::{dataset_for_compression, prepare_jar, Segment, SegmentHeader};
use reth_db::{
cursor::DbCursorRO, database::Database, static_file::create_static_file_T1_T2_T3, tables,
transaction::DbTx, RawKey, RawTable,
};
use reth_db::{static_file::create_static_file_T1_T2_T3, tables, RawKey, RawTable};
use reth_db_api::{cursor::DbCursorRO, database::Database, transaction::DbTx};
use reth_primitives::{static_file::SegmentConfig, BlockNumber, StaticFileSegment};
use reth_provider::{
providers::{StaticFileProvider, StaticFileWriter},

View File

@@ -9,9 +9,8 @@ pub use headers::Headers;
mod receipts;
pub use receipts::Receipts;
use reth_db::{
cursor::DbCursorRO, database::Database, table::Table, transaction::DbTx, RawKey, RawTable,
};
use reth_db::{RawKey, RawTable};
use reth_db_api::{cursor::DbCursorRO, database::Database, table::Table, transaction::DbTx};
use reth_nippy_jar::NippyJar;
use reth_primitives::{
static_file::{

View File

@@ -1,8 +1,6 @@
use crate::segments::{dataset_for_compression, prepare_jar, Segment};
use reth_db::{
cursor::DbCursorRO, database::Database, static_file::create_static_file_T1, tables,
transaction::DbTx,
};
use reth_db::{static_file::create_static_file_T1, tables};
use reth_db_api::{cursor::DbCursorRO, database::Database, transaction::DbTx};
use reth_primitives::{
static_file::{SegmentConfig, SegmentHeader},
BlockNumber, StaticFileSegment, TxNumber,

View File

@@ -1,8 +1,6 @@
use crate::segments::{dataset_for_compression, prepare_jar, Segment};
use reth_db::{
cursor::DbCursorRO, database::Database, static_file::create_static_file_T1, tables,
transaction::DbTx,
};
use reth_db::{static_file::create_static_file_T1, tables};
use reth_db_api::{cursor::DbCursorRO, database::Database, transaction::DbTx};
use reth_primitives::{
static_file::{SegmentConfig, SegmentHeader},
BlockNumber, StaticFileSegment, TxNumber,

View File

@@ -3,7 +3,7 @@
use crate::{segments, segments::Segment, StaticFileProducerEvent};
use parking_lot::Mutex;
use rayon::prelude::*;
use reth_db::database::Database;
use reth_db_api::database::Database;
use reth_primitives::{static_file::HighestStaticFiles, BlockNumber, PruneModes};
use reth_provider::{providers::StaticFileWriter, ProviderFactory, StaticFileProviderFactory};
use reth_storage_errors::provider::ProviderResult;
@@ -228,7 +228,8 @@ mod tests {
StaticFileProducer, StaticFileProducerInner, StaticFileTargets,
};
use assert_matches::assert_matches;
use reth_db::{database::Database, test_utils::TempDatabase, transaction::DbTx, DatabaseEnv};
use reth_db::{test_utils::TempDatabase, DatabaseEnv};
use reth_db_api::{database::Database, transaction::DbTx};
use reth_primitives::{
static_file::HighestStaticFiles, PruneModes, StaticFileSegment, B256, U256,
};

View File

@@ -0,0 +1,71 @@
[package]
name = "reth-db-api"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
description = "Database abstraction used in reth."
[lints]
workspace = true
[dependencies]
# reth
reth-codecs.workspace = true
reth-primitives.workspace = true
reth-storage-errors.workspace = true
# codecs
modular-bitfield.workspace = true
parity-scale-codec = { version = "3.2.1", features = ["bytes"] }
serde = { workspace = true, default-features = false }
# metrics
metrics.workspace = true
# misc
derive_more.workspace = true
bytes.workspace = true
# arbitrary utils
arbitrary = { workspace = true, features = ["derive"], optional = true }
proptest = { workspace = true, optional = true }
proptest-derive = { workspace = true, optional = true }
[dev-dependencies]
# reth libs with arbitrary
reth-primitives = { workspace = true, features = ["arbitrary"] }
reth-codecs.workspace = true
rand.workspace = true
serde_json.workspace = true
test-fuzz.workspace = true
pprof = { workspace = true, features = [
"flamegraph",
"frame-pointer",
"criterion",
] }
criterion.workspace = true
iai-callgrind = "0.10.2"
arbitrary = { workspace = true, features = ["derive"] }
proptest.workspace = true
proptest-derive.workspace = true
paste.workspace = true
assert_matches.workspace = true
[features]
test-utils = ["arbitrary"]
arbitrary = [
"reth-primitives/arbitrary",
"dep:arbitrary",
"dep:proptest",
"dep:proptest-derive",
]
optimism = []

View File

@@ -1,4 +1,4 @@
use crate::{abstraction::table::*, DatabaseError};
use crate::{table::*, DatabaseError};
/// A key-value pair for table `T`.
pub type KeyValue<T> = (<T as Table>::Key, <T as Table>::Value);
@@ -17,23 +17,3 @@ pub type IterPairResult<T> = Option<Result<KeyValue<T>, DatabaseError>>;
/// A value only result for table `T`.
pub type ValueOnlyResult<T> = Result<Option<<T as Table>::Value>, DatabaseError>;
// Sealed trait helper to prevent misuse of the Database API.
mod sealed {
use crate::{database::Database, mock::DatabaseMock};
use std::sync::Arc;
/// Sealed trait to limit the implementers of the Database trait.
pub trait Sealed: Sized {}
impl<DB: Database> Sealed for &DB {}
impl<DB: Database> Sealed for Arc<DB> {}
#[cfg(feature = "mdbx")]
impl Sealed for crate::DatabaseEnv {}
impl Sealed for DatabaseMock {}
#[cfg(any(test, feature = "test-utils"))]
impl<DB: Database> Sealed for crate::test_utils::TempDatabase<DB> {}
}
pub(crate) use sealed::Sealed;

View File

@@ -1,5 +1,4 @@
use crate::{
abstraction::common::Sealed,
table::TableImporter,
transaction::{DbTx, DbTxMut},
DatabaseError,
@@ -9,7 +8,7 @@ use std::{fmt::Debug, sync::Arc};
/// Main Database trait that can open read-only and read-write transactions.
///
/// Sealed trait which cannot be implemented by 3rd parties, exposed only for consumption.
pub trait Database: Send + Sync + Sealed {
pub trait Database: Send + Sync {
/// Read-Only database transaction
type TX: DbTx + Send + Sync + Debug + 'static;
/// Read-Write database transaction

View File

@@ -0,0 +1,83 @@
//! reth's database abstraction layer.
//!
//! The database abstraction assumes that the underlying store is a KV store subdivided into tables.
//!
//! One or more changes are tied to a transaction that is atomically committed to the data store at
//! the same time. Strong consistency in what data is written and when is important for reth, so it
//! is not possible to write data to the database outside of using a transaction.
//!
//! Good starting points for this crate are:
//!
//! - [`Database`] for the main database abstraction
//! - [`DbTx`] (RO) and [`DbTxMut`] (RW) for the transaction abstractions.
//! - [`DbCursorRO`] (RO) and [`DbCursorRW`] (RW) for the cursor abstractions (see below).
//!
//! # Cursors and Walkers
//!
//! The abstraction also defines a couple of helpful abstractions for iterating and writing data:
//!
//! - **Cursors** ([`DbCursorRO`] / [`DbCursorRW`]) for iterating data in a table. Cursors are
//! assumed to resolve data in a sorted manner when iterating from start to finish, and it is safe
//! to assume that they are efficient at doing so.
//! - **Walkers** ([`Walker`] / [`RangeWalker`] / [`ReverseWalker`]) use cursors to walk the entries
//! in a table, either fully from a specific point, or over a range.
//!
//! Dup tables (see below) also have corresponding cursors and walkers (e.g. [`DbDupCursorRO`]).
//! These **should** be preferred when working with dup tables, as they provide additional methods
//! that are optimized for dup tables.
//!
//! # Tables
//!
//! reth has two types of tables: simple KV stores (one key, one value) and dup tables (one key,
//! many values). Dup tables can be efficient for certain types of data.
//!
//! Keys are de/serialized using the [`Encode`] and [`Decode`] traits, and values are de/serialized
//! ("compressed") using the [`Compress`] and [`Decompress`] traits.
//!
//! Tables implement the [`Table`] trait.
//!
//! [`Database`]: crate::database::Database
//! [`DbTx`]: crate::transaction::DbTx
//! [`DbTxMut`]: crate::transaction::DbTxMut
//! [`DbCursorRO`]: crate::cursor::DbCursorRO
//! [`DbCursorRW`]: crate::cursor::DbCursorRW
//! [`Walker`]: crate::cursor::Walker
//! [`RangeWalker`]: crate::cursor::RangeWalker
//! [`ReverseWalker`]: crate::cursor::ReverseWalker
//! [`DbDupCursorRO`]: crate::cursor::DbDupCursorRO
//! [`Encode`]: crate::table::Encode
//! [`Decode`]: crate::table::Decode
//! [`Compress`]: crate::table::Compress
//! [`Decompress`]: crate::table::Decompress
//! [`Table`]: crate::table::Table
#![doc(
html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png",
html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256",
issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/"
)]
#![cfg_attr(not(test), warn(unused_crate_dependencies))]
#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))]
// TODO: remove when https://github.com/proptest-rs/proptest/pull/427 is merged
#![allow(unknown_lints, non_local_definitions)]
/// Common types used throughout the abstraction.
pub mod common;
/// Cursor database traits.
pub mod cursor;
/// Database traits.
pub mod database;
/// Database metrics trait extensions.
pub mod database_metrics;
pub mod mock;
/// Table traits
pub mod table;
/// Transaction database traits.
pub mod transaction;
/// Re-exports
pub use reth_storage_errors::db::{DatabaseError, DatabaseWriteOperation};
pub mod models;
mod scale;
mod utils;

View File

@@ -11,7 +11,7 @@ use reth_codecs::{derive_arbitrary, Compact};
use reth_primitives::{Account, Address, BlockNumber, Buf, StorageKey};
use serde::{Deserialize, Serialize};
/// Account as it is saved inside [`AccountChangeSets`][crate::tables::AccountChangeSets].
/// Account as it is saved in the database.
///
/// [`Address`] is the subkey.
#[derive_arbitrary(compact)]
@@ -57,8 +57,7 @@ impl Compact for AccountBeforeTx {
}
}
/// [`BlockNumber`] concatenated with [`Address`]. Used as the key for
/// [`StorageChangeSets`](crate::tables::StorageChangeSets)
/// [`BlockNumber`] concatenated with [`Address`].
///
/// Since it's used as a key, it isn't compressed when encoding it.
#[derive(

View File

@@ -83,7 +83,7 @@ pub struct StoredBlockWithdrawals {
pub withdrawals: Withdrawals,
}
/// Hash of the block header. Value for [`CanonicalHeaders`][crate::tables::CanonicalHeaders]
/// Hash of the block header.
pub type HeaderHash = B256;
#[cfg(test)]

View File

@@ -1,10 +1,181 @@
//! Implements data structures specific to the database
use crate::{
models::client_version::ClientVersion,
table::{Compress, Decompress},
tables::models::*,
table::{Compress, Decode, Decompress, Encode},
DatabaseError,
};
use reth_codecs::{main_codec, Compact};
use reth_primitives::{stage::StageCheckpoint, trie::*, *};
use reth_primitives::{
stage::StageCheckpoint,
trie::{StoredNibbles, StoredNibblesSubKey, *},
Address, PruneSegment, B256, *,
};
pub mod accounts;
pub mod blocks;
pub mod client_version;
pub mod integer_list;
pub mod sharded_key;
pub mod storage_sharded_key;
pub use accounts::*;
pub use blocks::*;
pub use client_version::ClientVersion;
pub use sharded_key::ShardedKey;
/// Macro that implements [`Encode`] and [`Decode`] for uint types.
macro_rules! impl_uints {
($($name:tt),+) => {
$(
impl Encode for $name {
type Encoded = [u8; std::mem::size_of::<$name>()];
fn encode(self) -> Self::Encoded {
self.to_be_bytes()
}
}
impl Decode for $name {
fn decode<B: AsRef<[u8]>>(value: B) -> Result<Self, $crate::DatabaseError> {
Ok(
$name::from_be_bytes(
value.as_ref().try_into().map_err(|_| $crate::DatabaseError::Decode)?
)
)
}
}
)+
};
}
impl_uints!(u64, u32, u16, u8);
impl Encode for Vec<u8> {
type Encoded = Self;
fn encode(self) -> Self::Encoded {
self
}
}
impl Decode for Vec<u8> {
fn decode<B: AsRef<[u8]>>(value: B) -> Result<Self, DatabaseError> {
Ok(value.as_ref().to_vec())
}
}
impl Encode for Address {
type Encoded = [u8; 20];
fn encode(self) -> Self::Encoded {
self.0 .0
}
}
impl Decode for Address {
fn decode<B: AsRef<[u8]>>(value: B) -> Result<Self, DatabaseError> {
Ok(Self::from_slice(value.as_ref()))
}
}
impl Encode for B256 {
type Encoded = [u8; 32];
fn encode(self) -> Self::Encoded {
self.0
}
}
impl Decode for B256 {
fn decode<B: AsRef<[u8]>>(value: B) -> Result<Self, DatabaseError> {
Ok(Self::new(value.as_ref().try_into().map_err(|_| DatabaseError::Decode)?))
}
}
impl Encode for String {
type Encoded = Vec<u8>;
fn encode(self) -> Self::Encoded {
self.into_bytes()
}
}
impl Decode for String {
fn decode<B: AsRef<[u8]>>(value: B) -> Result<Self, DatabaseError> {
Self::from_utf8(value.as_ref().to_vec()).map_err(|_| DatabaseError::Decode)
}
}
impl Encode for StoredNibbles {
type Encoded = Vec<u8>;
// Delegate to the Compact implementation
fn encode(self) -> Self::Encoded {
let mut buf = Vec::with_capacity(self.0.len());
self.to_compact(&mut buf);
buf
}
}
impl Decode for StoredNibbles {
fn decode<B: AsRef<[u8]>>(value: B) -> Result<Self, DatabaseError> {
let buf = value.as_ref();
Ok(Self::from_compact(buf, buf.len()).0)
}
}
impl Encode for StoredNibblesSubKey {
type Encoded = Vec<u8>;
// Delegate to the Compact implementation
fn encode(self) -> Self::Encoded {
let mut buf = Vec::with_capacity(65);
self.to_compact(&mut buf);
buf
}
}
impl Decode for StoredNibblesSubKey {
fn decode<B: AsRef<[u8]>>(value: B) -> Result<Self, DatabaseError> {
let buf = value.as_ref();
Ok(Self::from_compact(buf, buf.len()).0)
}
}
impl Encode for PruneSegment {
type Encoded = [u8; 1];
fn encode(self) -> Self::Encoded {
let mut buf = [0u8];
self.to_compact(&mut buf.as_mut());
buf
}
}
impl Decode for PruneSegment {
fn decode<B: AsRef<[u8]>>(value: B) -> Result<Self, DatabaseError> {
let buf = value.as_ref();
Ok(Self::from_compact(buf, buf.len()).0)
}
}
impl Encode for ClientVersion {
type Encoded = Vec<u8>;
// Delegate to the Compact implementation
fn encode(self) -> Self::Encoded {
let mut buf = vec![];
self.to_compact(&mut buf);
buf
}
}
impl Decode for ClientVersion {
fn decode<B: AsRef<[u8]>>(value: B) -> Result<Self, DatabaseError> {
let buf = value.as_ref();
Ok(Self::from_compact(buf, buf.len()).0)
}
}
/// Implements compression for Compact type.
macro_rules! impl_compression_for_compact {
@@ -127,13 +298,7 @@ add_wrapper_struct!((ClientVersion, CompactClientVersion));
#[cfg(test)]
mod tests {
use crate::{
codecs::{
compact::{CompactClientVersion, CompactU64},
CompactU256,
},
models::{StoredBlockBodyIndices, StoredBlockOmmers, StoredBlockWithdrawals},
};
use super::*;
use reth_primitives::{
stage::{
AccountHashingCheckpoint, CheckpointBlockRange, EntitiesCheckpoint,

View File

@@ -1,13 +1,11 @@
//! Sharded key
use std::hash::Hash;
use crate::{
table::{Decode, Encode},
DatabaseError,
};
use reth_primitives::BlockNumber;
use serde::{Deserialize, Serialize};
use std::hash::Hash;
/// Number of indices in one shard.
pub const NUM_OF_INDICES_IN_SHARD: usize = 2_000;

View File

@@ -1,5 +1,4 @@
//! Storage sharded key
use crate::{
table::{Decode, Encode},
DatabaseError,

View File

@@ -1,5 +1,5 @@
use crate::{
abstraction::cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW},
cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW},
transaction::{DbTx, DbTxMut},
DatabaseError,
};
@@ -80,11 +80,8 @@ impl<T> Value for T where T: Compress + Decompress + Serialize {}
/// It allows for the use of codecs. See [`crate::models::ShardedKey`] for a custom
/// implementation.
pub trait Table: Send + Sync + Debug + 'static {
/// The dynamic type of the table.
const TABLE: crate::Tables;
/// The table's name.
const NAME: &'static str = Self::TABLE.name();
const NAME: &'static str;
/// Key element of `Table`.
///

View File

@@ -0,0 +1,34 @@
#[macro_export]
/// Implements the `Arbitrary` trait for types with fixed array types.
macro_rules! impl_fixed_arbitrary {
($(($name:ident, $size:expr)),*) => {
#[cfg(any(test, feature = "arbitrary"))]
use arbitrary::{Arbitrary, Unstructured};
$(
#[cfg(any(test, feature = "arbitrary"))]
impl<'a> Arbitrary<'a> for $name {
fn arbitrary(u: &mut Unstructured<'a>) -> Result<Self, arbitrary::Error> {
let mut buffer = vec![0; $size];
u.fill_buffer(buffer.as_mut_slice())?;
Decode::decode(buffer).map_err(|_| arbitrary::Error::IncorrectFormat)
}
}
#[cfg(any(test, feature = "arbitrary"))]
impl proptest::prelude::Arbitrary for $name {
type Parameters = ();
type Strategy = proptest::strategy::Map<
proptest::collection::VecStrategy<<u8 as proptest::arbitrary::Arbitrary>::Strategy>,
fn(Vec<u8>) -> Self,
>;
fn arbitrary_with(args: Self::Parameters) -> Self::Strategy {
use proptest::strategy::Strategy;
proptest::collection::vec(proptest::arbitrary::any_with::<u8>(args), $size)
.prop_map(move |vec| Decode::decode(vec).unwrap())
}
}
)+
};
}

View File

@@ -11,6 +11,7 @@ repository.workspace = true
# reth
reth-primitives.workspace = true
reth-db = { workspace = true, features = ["mdbx"] }
reth-db-api.workspace = true
reth-provider.workspace = true
reth-config.workspace = true
reth-trie.workspace = true
@@ -29,4 +30,4 @@ serde_json.workspace = true
tracing.workspace = true
[lints]
workspace = true
workspace = true

View File

@@ -2,7 +2,8 @@
use reth_codecs::Compact;
use reth_config::config::EtlConfig;
use reth_db::{database::Database, tables, transaction::DbTxMut, DatabaseError};
use reth_db::tables;
use reth_db_api::{database::Database, transaction::DbTxMut, DatabaseError};
use reth_etl::Collector;
use reth_primitives::{
stage::{StageCheckpoint, StageId},
@@ -522,12 +523,12 @@ struct GenesisAccountWithAddress {
#[cfg(test)]
mod tests {
use super::*;
use reth_db::{
use reth_db::DatabaseEnv;
use reth_db_api::{
cursor::DbCursorRO,
models::{storage_sharded_key::StorageShardedKey, ShardedKey},
table::{Table, TableRow},
transaction::DbTx,
DatabaseEnv,
};
use reth_primitives::{
Chain, Genesis, IntegerList, GOERLI, GOERLI_GENESIS_HASH, MAINNET, MAINNET_GENESIS_HASH,

View File

@@ -13,10 +13,10 @@ workspace = true
[dependencies]
# reth
reth-db-api.workspace = true
reth-primitives.workspace = true
reth-fs-util.workspace = true
reth-storage-errors.workspace = true
reth-codecs.workspace = true
reth-libmdbx = { workspace = true, optional = true, features = [
"return-borrowed",
"read-tx-timeouts",
@@ -26,8 +26,6 @@ reth-tracing.workspace = true
# codecs
serde = { workspace = true, default-features = false }
parity-scale-codec = { version = "3.2.1", features = ["bytes"] }
modular-bitfield.workspace = true
# metrics
reth-metrics.workspace = true
@@ -45,17 +43,11 @@ rustc-hash.workspace = true
sysinfo = "0.30"
# arbitrary utils
arbitrary = { workspace = true, features = ["derive"], optional = true }
proptest = { workspace = true, optional = true }
proptest-derive = { workspace = true, optional = true }
strum = { workspace = true, features = ["derive"] }
once_cell.workspace = true
[dev-dependencies]
# reth libs with arbitrary
reth-primitives = { workspace = true, features = ["arbitrary"] }
reth-codecs.workspace = true
rand.workspace = true
serde_json.workspace = true
@@ -72,7 +64,6 @@ iai-callgrind = "0.10.2"
arbitrary = { workspace = true, features = ["derive"] }
proptest.workspace = true
proptest-derive.workspace = true
paste.workspace = true
@@ -83,12 +74,7 @@ default = ["mdbx"]
test-utils = ["tempfile", "arbitrary"]
mdbx = ["reth-libmdbx"]
bench = []
arbitrary = [
"reth-primitives/arbitrary",
"dep:arbitrary",
"dep:proptest",
"dep:proptest-derive",
]
arbitrary = ["reth-primitives/arbitrary", "reth-db-api/arbitrary"]
optimism = []
[[bench]]

View File

@@ -3,10 +3,10 @@ use criterion::{
black_box, criterion_group, criterion_main, measurement::WallTime, BenchmarkGroup, Criterion,
};
use pprof::criterion::{Output, PProfProfiler};
use reth_db::{
use reth_db::tables::*;
use reth_db_api::{
cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW},
table::{Decode, Decompress, DupSort},
tables::*,
transaction::DbTx,
};

View File

@@ -9,7 +9,8 @@ use proptest::{
strategy::{Strategy, ValueTree},
test_runner::TestRunner,
};
use reth_db::{cursor::DbCursorRW, TransactionHashNumbers};
use reth_db::TransactionHashNumbers;
use reth_db_api::cursor::DbCursorRW;
use std::collections::HashSet;
criterion_group! {

View File

@@ -4,7 +4,7 @@ use iai_callgrind::{
library_benchmark, library_benchmark_group, LibraryBenchmarkConfig, RegressionConfig,
};
use paste::paste;
use reth_db::table::{Compress, Decode, Decompress, Encode, Table};
use reth_db_api::table::{Compress, Decode, Decompress, Encode, Table};
mod utils;
use utils::*;

View File

@@ -1,9 +1,8 @@
use reth_db::{
use reth_db::{test_utils::create_test_rw_db_with_path, DatabaseEnv};
use reth_db_api::{
database::Database,
table::{Compress, Encode, Table, TableRow},
test_utils::create_test_rw_db_with_path,
transaction::DbTxMut,
DatabaseEnv,
};
use reth_fs_util as fs;
use reth_primitives::Bytes;

View File

@@ -1,14 +0,0 @@
/// Common types used throughout the abstraction.
pub mod common;
/// Cursor database traits.
pub mod cursor;
/// Database traits.
pub mod database;
/// Database metrics trait extensions.
pub mod database_metrics;
/// mock
pub mod mock;
/// Table traits
pub mod table;
/// Transaction database traits.
pub mod transaction;

View File

@@ -1,15 +1,17 @@
//! Cursor wrapper for libmdbx-sys.
use crate::{
metrics::{DatabaseEnvMetrics, Operation},
tables::utils::*,
DatabaseError,
};
use reth_db_api::{
common::{PairResult, ValueOnlyResult},
cursor::{
DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW, DupWalker, RangeWalker,
ReverseWalker, Walker,
},
metrics::{DatabaseEnvMetrics, Operation},
table::{Compress, Decode, Decompress, DupSort, Encode, Table},
tables::utils::*,
DatabaseError,
};
use reth_libmdbx::{Error as MDBXError, TransactionKind, WriteFlags, RO, RW};
use reth_storage_errors::db::{DatabaseErrorInfo, DatabaseWriteError, DatabaseWriteOperation};
@@ -52,7 +54,7 @@ impl<K: TransactionKind, T: Table> Cursor<K, T> {
f: impl FnOnce(&mut Self) -> R,
) -> R {
if let Some(metrics) = self.metrics.as_ref().cloned() {
metrics.record_operation(T::TABLE, operation, value_size, || f(self))
metrics.record_operation(T::NAME, operation, value_size, || f(self))
} else {
f(self)
}

View File

@@ -1,19 +1,21 @@
//! Module that interacts with MDBX.
use crate::{
cursor::{DbCursorRO, DbCursorRW},
database::Database,
database_metrics::{DatabaseMetadata, DatabaseMetadataValue, DatabaseMetrics},
lockfile::StorageLock,
metrics::DatabaseEnvMetrics,
models::client_version::ClientVersion,
tables::{self, TableType, Tables},
transaction::{DbTx, DbTxMut},
utils::default_page_size,
DatabaseError,
};
use eyre::Context;
use metrics::{gauge, Label};
use reth_db_api::{
cursor::{DbCursorRO, DbCursorRW},
database::Database,
database_metrics::{DatabaseMetadata, DatabaseMetadataValue, DatabaseMetrics},
models::client_version::ClientVersion,
transaction::{DbTx, DbTxMut},
};
use reth_libmdbx::{
DatabaseFlags, Environment, EnvironmentFlags, Geometry, MaxReadTransactionDuration, Mode,
PageSize, SyncMode, RO, RW,
@@ -459,15 +461,17 @@ impl Deref for DatabaseEnv {
mod tests {
use super::*;
use crate::{
abstraction::table::{Encode, Table},
cursor::{DbDupCursorRO, DbDupCursorRW, ReverseWalker, Walker},
models::{AccountBeforeTx, ShardedKey},
tables::{
AccountsHistory, CanonicalHeaders, Headers, PlainAccountState, PlainStorageState,
},
test_utils::*,
AccountChangeSets,
};
use reth_db_api::{
cursor::{DbDupCursorRO, DbDupCursorRW, ReverseWalker, Walker},
models::{AccountBeforeTx, ShardedKey},
table::{Encode, Table},
};
use reth_libmdbx::Error;
use reth_primitives::{Account, Address, Header, IntegerList, StorageEntry, B256, U256};
use reth_storage_errors::db::{DatabaseWriteError, DatabaseWriteOperation};

View File

@@ -3,12 +3,13 @@
use super::cursor::Cursor;
use crate::{
metrics::{DatabaseEnvMetrics, Operation, TransactionMode, TransactionOutcome},
table::{Compress, DupSort, Encode, Table, TableImporter},
tables::{utils::decode_one, Tables},
transaction::{DbTx, DbTxMut},
tables::utils::decode_one,
DatabaseError,
};
use once_cell::sync::OnceCell;
use reth_db_api::{
table::{Compress, DupSort, Encode, Table, TableImporter},
transaction::{DbTx, DbTxMut},
};
use reth_libmdbx::{ffi::DBI, CommitLatency, Transaction, TransactionKind, WriteFlags, RW};
use reth_storage_errors::db::{DatabaseWriteError, DatabaseWriteOperation};
use reth_tracing::tracing::{debug, trace, warn};
@@ -36,10 +37,6 @@ pub struct Tx<K: TransactionKind> {
///
/// If [Some], then metrics are reported.
metrics_handler: Option<MetricsHandler<K>>,
/// Database table handle cache.
// TODO: Use `std::sync::OnceLock` once `get_or_try_init` is stable.
db_handles: [OnceCell<DBI>; Tables::COUNT],
}
impl<K: TransactionKind> Tx<K> {
@@ -69,14 +66,7 @@ impl<K: TransactionKind> Tx<K> {
#[inline]
const fn new_inner(inner: Transaction<K>, metrics_handler: Option<MetricsHandler<K>>) -> Self {
// NOTE: These constants are needed to initialize `OnceCell` at compile-time, as array
// initialization is not allowed with non-Copy types, and `const { }` blocks are not stable
// yet.
#[allow(clippy::declare_interior_mutable_const)]
const ONCECELL_DBI_NEW: OnceCell<DBI> = OnceCell::new();
#[allow(clippy::declare_interior_mutable_const)]
const DB_HANDLES: [OnceCell<DBI>; Tables::COUNT] = [ONCECELL_DBI_NEW; Tables::COUNT];
Self { inner, db_handles: DB_HANDLES, metrics_handler }
Self { inner, metrics_handler }
}
/// Gets this transaction ID.
@@ -86,14 +76,10 @@ impl<K: TransactionKind> Tx<K> {
/// Gets a table database handle if it exists, otherwise creates it.
pub fn get_dbi<T: Table>(&self) -> Result<DBI, DatabaseError> {
self.db_handles[T::TABLE as usize]
.get_or_try_init(|| {
self.inner
.open_db(Some(T::NAME))
.map(|db| db.dbi())
.map_err(|e| DatabaseError::Open(e.into()))
})
.copied()
self.inner
.open_db(Some(T::NAME))
.map(|db| db.dbi())
.map_err(|e| DatabaseError::Open(e.into()))
}
/// Create db Cursor
@@ -170,7 +156,7 @@ impl<K: TransactionKind> Tx<K> {
metrics_handler.log_backtrace_on_long_read_transaction();
metrics_handler
.env_metrics
.record_operation(T::TABLE, operation, value_size, || f(&self.inner))
.record_operation(T::NAME, operation, value_size, || f(&self.inner))
} else {
f(&self.inner)
}
@@ -391,10 +377,8 @@ impl DbTxMut for Tx<RW> {
#[cfg(test)]
mod tests {
use crate::{
database::Database, mdbx::DatabaseArguments, models::client_version::ClientVersion, tables,
transaction::DbTx, DatabaseEnv, DatabaseEnvKind,
};
use crate::{mdbx::DatabaseArguments, tables, DatabaseEnv, DatabaseEnvKind};
use reth_db_api::{database::Database, models::ClientVersion, transaction::DbTx};
use reth_libmdbx::MaxReadTransactionDuration;
use reth_storage_errors::db::DatabaseError;
use std::{sync::atomic::Ordering, thread::sleep, time::Duration};

Some files were not shown because too many files have changed in this diff Show More