mirror of
https://github.com/paradigmxyz/reth.git
synced 2026-04-30 03:01:58 -04:00
chore: split db abstraction into new crate (#8594)
This commit is contained in:
@@ -17,6 +17,7 @@ reth-primitives.workspace = true
|
||||
reth-storage-errors.workspace = true
|
||||
reth-execution-errors.workspace = true
|
||||
reth-db.workspace = true
|
||||
reth-db-api.workspace = true
|
||||
reth-evm.workspace = true
|
||||
reth-revm.workspace = true
|
||||
reth-provider.workspace = true
|
||||
@@ -41,7 +42,7 @@ linked_hash_set = "0.1.4"
|
||||
|
||||
[dev-dependencies]
|
||||
reth-db = { workspace = true, features = ["test-utils"] }
|
||||
reth-primitives = { workspace = true , features = ["test-utils"] }
|
||||
reth-primitives = { workspace = true, features = ["test-utils"] }
|
||||
reth-provider = { workspace = true, features = ["test-utils"] }
|
||||
reth-evm = { workspace = true, features = ["test-utils"] }
|
||||
reth-testing-utils.workspace = true
|
||||
|
||||
@@ -10,7 +10,7 @@ use reth_blockchain_tree_api::{
|
||||
BlockAttachment, BlockStatus, BlockValidationKind, CanonicalOutcome, InsertPayloadOk,
|
||||
};
|
||||
use reth_consensus::{Consensus, ConsensusError};
|
||||
use reth_db::database::Database;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_evm::execute::BlockExecutorProvider;
|
||||
use reth_execution_errors::{BlockExecutionError, BlockValidationError};
|
||||
use reth_primitives::{
|
||||
@@ -1377,7 +1377,8 @@ mod tests {
|
||||
use assert_matches::assert_matches;
|
||||
use linked_hash_set::LinkedHashSet;
|
||||
use reth_consensus::test_utils::TestConsensus;
|
||||
use reth_db::{tables, test_utils::TempDatabase, transaction::DbTxMut, DatabaseEnv};
|
||||
use reth_db::{tables, test_utils::TempDatabase, DatabaseEnv};
|
||||
use reth_db_api::transaction::DbTxMut;
|
||||
use reth_evm::test_utils::MockExecutorProvider;
|
||||
use reth_evm_ethereum::execute::EthExecutorProvider;
|
||||
#[cfg(not(feature = "optimism"))]
|
||||
|
||||
@@ -10,7 +10,7 @@ use reth_blockchain_tree_api::{
|
||||
BlockAttachment, BlockValidationKind,
|
||||
};
|
||||
use reth_consensus::{Consensus, ConsensusError, PostExecutionInput};
|
||||
use reth_db::database::Database;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_evm::execute::{BlockExecutionOutput, BlockExecutorProvider, Executor};
|
||||
use reth_execution_errors::BlockExecutionError;
|
||||
use reth_primitives::{
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
//! Blockchain tree externals.
|
||||
|
||||
use reth_consensus::Consensus;
|
||||
use reth_db::{
|
||||
cursor::DbCursorRO, database::Database, static_file::HeaderMask, tables, transaction::DbTx,
|
||||
};
|
||||
use reth_db::{static_file::HeaderMask, tables};
|
||||
use reth_db_api::{cursor::DbCursorRO, database::Database, transaction::DbTx};
|
||||
use reth_primitives::{BlockHash, BlockNumber, StaticFileSegment};
|
||||
use reth_provider::{ProviderFactory, StaticFileProviderFactory, StatsReader};
|
||||
use reth_storage_errors::provider::ProviderResult;
|
||||
|
||||
@@ -7,7 +7,7 @@ use reth_blockchain_tree_api::{
|
||||
BlockValidationKind, BlockchainTreeEngine, BlockchainTreeViewer, CanonicalOutcome,
|
||||
InsertPayloadOk,
|
||||
};
|
||||
use reth_db::database::Database;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_evm::execute::BlockExecutorProvider;
|
||||
use reth_primitives::{
|
||||
BlockHash, BlockNumHash, BlockNumber, Receipt, SealedBlock, SealedBlockWithSenders,
|
||||
|
||||
@@ -17,7 +17,7 @@ reth-blockchain-tree-api.workspace = true
|
||||
reth-primitives.workspace = true
|
||||
reth-stages-api.workspace = true
|
||||
reth-errors.workspace = true
|
||||
reth-db.workspace = true
|
||||
reth-db-api.workspace = true
|
||||
reth-provider.workspace = true
|
||||
reth-rpc-types.workspace = true
|
||||
reth-tasks.workspace = true
|
||||
@@ -74,5 +74,5 @@ optimism = [
|
||||
"reth-provider/optimism",
|
||||
"reth-blockchain-tree/optimism",
|
||||
"reth-ethereum-consensus/optimism",
|
||||
"reth-rpc/optimism"
|
||||
"reth-rpc/optimism",
|
||||
]
|
||||
|
||||
@@ -6,7 +6,7 @@ use crate::{
|
||||
};
|
||||
use futures::FutureExt;
|
||||
use metrics::Counter;
|
||||
use reth_db::database::Database;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_errors::{RethError, RethResult};
|
||||
use reth_primitives::BlockNumber;
|
||||
use reth_prune::{Pruner, PrunerError, PrunerWithResult};
|
||||
|
||||
@@ -5,7 +5,7 @@ use crate::{
|
||||
hooks::EngineHookDBAccessLevel,
|
||||
};
|
||||
use futures::FutureExt;
|
||||
use reth_db::database::Database;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_errors::RethResult;
|
||||
use reth_primitives::{static_file::HighestStaticFiles, BlockNumber};
|
||||
use reth_static_file::{StaticFileProducer, StaticFileProducerWithResult};
|
||||
|
||||
@@ -4,7 +4,7 @@ use reth_blockchain_tree_api::{
|
||||
error::{BlockchainTreeError, CanonicalError, InsertBlockError, InsertBlockErrorKind},
|
||||
BlockStatus, BlockValidationKind, BlockchainTreeEngine, CanonicalOutcome, InsertPayloadOk,
|
||||
};
|
||||
use reth_db::database::Database;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_engine_primitives::{EngineTypes, PayloadAttributes, PayloadBuilderAttributes};
|
||||
use reth_errors::{BlockValidationError, ProviderResult, RethError, RethResult};
|
||||
use reth_network_p2p::{
|
||||
@@ -2178,7 +2178,8 @@ mod tests {
|
||||
|
||||
mod fork_choice_updated {
|
||||
use super::*;
|
||||
use reth_db::{tables, test_utils::create_test_static_files_dir, transaction::DbTxMut};
|
||||
use reth_db::{tables, test_utils::create_test_static_files_dir};
|
||||
use reth_db_api::transaction::DbTxMut;
|
||||
use reth_primitives::U256;
|
||||
use reth_provider::providers::StaticFileProvider;
|
||||
use reth_rpc_types::engine::ForkchoiceUpdateError;
|
||||
|
||||
@@ -5,7 +5,7 @@ use crate::{
|
||||
ConsensusEngineLiveSyncProgress, EthBeaconConsensus,
|
||||
};
|
||||
use futures::FutureExt;
|
||||
use reth_db::database::Database;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_network_p2p::{
|
||||
bodies::client::BodiesClient,
|
||||
full_block::{FetchFullBlockFuture, FetchFullBlockRangeFuture, FullBlockClient},
|
||||
|
||||
@@ -9,7 +9,7 @@ repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
tempfile.workspace = true
|
||||
reth-db.workspace = true
|
||||
reth-db-api.workspace = true
|
||||
rayon.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
@@ -22,7 +22,7 @@ use std::{
|
||||
};
|
||||
|
||||
use rayon::prelude::*;
|
||||
use reth_db::table::{Compress, Encode, Key, Value};
|
||||
use reth_db_api::table::{Compress, Encode, Key, Value};
|
||||
use tempfile::{NamedTempFile, TempDir};
|
||||
|
||||
/// An ETL (extract, transform, load) data collector.
|
||||
|
||||
@@ -23,6 +23,7 @@ reth-network-types.workspace = true
|
||||
|
||||
# optional deps for the test-utils feature
|
||||
reth-db = { workspace = true, optional = true }
|
||||
reth-db-api = { workspace = true, optional = true }
|
||||
reth-testing-utils = { workspace = true, optional = true }
|
||||
|
||||
# eth
|
||||
@@ -50,6 +51,7 @@ itertools.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
reth-db = { workspace = true, features = ["test-utils"] }
|
||||
reth-db-api.workspace = true
|
||||
reth-consensus = { workspace = true, features = ["test-utils"] }
|
||||
reth-network-p2p = { workspace = true, features = ["test-utils"] }
|
||||
reth-provider = { workspace = true, features = ["test-utils"] }
|
||||
@@ -65,5 +67,11 @@ rand.workspace = true
|
||||
tempfile.workspace = true
|
||||
|
||||
[features]
|
||||
test-utils = ["dep:tempfile", "reth-db/test-utils", "reth-consensus/test-utils", "reth-network-p2p/test-utils", "reth-testing-utils"]
|
||||
|
||||
test-utils = [
|
||||
"dep:tempfile",
|
||||
"dep:reth-db-api",
|
||||
"reth-db/test-utils",
|
||||
"reth-consensus/test-utils",
|
||||
"reth-network-p2p/test-utils",
|
||||
"reth-testing-utils",
|
||||
]
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
|
||||
#![allow(dead_code)]
|
||||
|
||||
use reth_db::{database::Database, tables, transaction::DbTxMut, DatabaseEnv};
|
||||
use reth_db::{tables, DatabaseEnv};
|
||||
use reth_db_api::{database::Database, transaction::DbTxMut};
|
||||
use reth_network_p2p::bodies::response::BlockResponse;
|
||||
use reth_primitives::{Block, BlockBody, SealedBlock, SealedHeader, B256};
|
||||
use std::collections::HashMap;
|
||||
|
||||
@@ -15,6 +15,7 @@ workspace = true
|
||||
reth-primitives.workspace = true
|
||||
reth-fs-util.workspace = true
|
||||
reth-db = { workspace = true, features = ["mdbx"] }
|
||||
reth-db-api.workspace = true
|
||||
reth-storage-errors = { workspace = true, features = ["clap"] }
|
||||
reth-provider.workspace = true
|
||||
reth-network = { workspace = true, features = ["serde"] }
|
||||
|
||||
@@ -9,7 +9,7 @@ use hyper::{
|
||||
use metrics::describe_gauge;
|
||||
use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle};
|
||||
use metrics_util::layers::{PrefixLayer, Stack};
|
||||
use reth_db::database_metrics::DatabaseMetrics;
|
||||
use reth_db_api::database_metrics::DatabaseMetrics;
|
||||
use reth_metrics::metrics::Unit;
|
||||
use reth_provider::providers::StaticFileProvider;
|
||||
use reth_tasks::TaskExecutor;
|
||||
|
||||
@@ -13,7 +13,7 @@ use discv5::ListenConfig;
|
||||
use metrics_exporter_prometheus::PrometheusHandle;
|
||||
use once_cell::sync::Lazy;
|
||||
use reth_config::{config::PruneConfig, Config};
|
||||
use reth_db::{database::Database, database_metrics::DatabaseMetrics};
|
||||
use reth_db_api::{database::Database, database_metrics::DatabaseMetrics};
|
||||
use reth_network::{NetworkBuilder, NetworkConfig, NetworkManager};
|
||||
use reth_network_p2p::headers::client::HeadersClient;
|
||||
use reth_primitives::{
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
//! Version information for reth.
|
||||
use reth_db::models::client_version::ClientVersion;
|
||||
use reth_db_api::models::ClientVersion;
|
||||
use reth_rpc_types::engine::ClientCode;
|
||||
|
||||
/// The client code for Reth
|
||||
|
||||
@@ -14,7 +14,7 @@ workspace = true
|
||||
# reth
|
||||
reth-evm.workspace = true
|
||||
reth-provider.workspace = true
|
||||
reth-db.workspace = true
|
||||
reth-db-api.workspace = true
|
||||
reth-engine-primitives.workspace = true
|
||||
reth-transaction-pool.workspace = true
|
||||
reth-network.workspace = true
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
//! Traits for configuring a node.
|
||||
|
||||
use crate::{primitives::NodePrimitives, ConfigureEvm, EngineTypes};
|
||||
use reth_db::{
|
||||
use reth_db_api::{
|
||||
database::Database,
|
||||
database_metrics::{DatabaseMetadata, DatabaseMetrics},
|
||||
};
|
||||
|
||||
@@ -21,6 +21,7 @@ reth-exex.workspace = true
|
||||
reth-evm.workspace = true
|
||||
reth-provider.workspace = true
|
||||
reth-db.workspace = true
|
||||
reth-db-api.workspace = true
|
||||
reth-rpc-engine-api.workspace = true
|
||||
reth-rpc.workspace = true
|
||||
reth-rpc-layer.workspace = true
|
||||
|
||||
@@ -10,11 +10,13 @@ use crate::{
|
||||
};
|
||||
use futures::Future;
|
||||
use reth_db::{
|
||||
database::Database,
|
||||
database_metrics::{DatabaseMetadata, DatabaseMetrics},
|
||||
test_utils::{create_test_rw_db_with_path, tempdir_path, TempDatabase},
|
||||
DatabaseEnv,
|
||||
};
|
||||
use reth_db_api::{
|
||||
database::Database,
|
||||
database_metrics::{DatabaseMetadata, DatabaseMetrics},
|
||||
};
|
||||
use reth_exex::ExExContext;
|
||||
use reth_network::{NetworkBuilder, NetworkConfig, NetworkHandle};
|
||||
use reth_node_api::{FullNodeTypes, FullNodeTypesAdapter, NodeTypes};
|
||||
|
||||
@@ -5,7 +5,7 @@ use rayon::ThreadPoolBuilder;
|
||||
use reth_auto_seal_consensus::MiningMode;
|
||||
use reth_beacon_consensus::EthBeaconConsensus;
|
||||
use reth_config::{config::EtlConfig, PruneConfig};
|
||||
use reth_db::{database::Database, database_metrics::DatabaseMetrics};
|
||||
use reth_db_api::{database::Database, database_metrics::DatabaseMetrics};
|
||||
use reth_db_common::init::{init_genesis, InitDatabaseError};
|
||||
use reth_downloaders::{bodies::noop::NoopBodiesDownloader, headers::noop::NoopHeaderDownloader};
|
||||
use reth_evm::noop::NoopBlockExecutorProvider;
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
use reth_config::{config::StageConfig, PruneConfig};
|
||||
use reth_consensus::Consensus;
|
||||
use reth_db::database::Database;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_downloaders::{
|
||||
bodies::bodies::BodiesDownloaderBuilder,
|
||||
headers::reverse_headers::ReverseHeadersDownloaderBuilder,
|
||||
|
||||
@@ -18,7 +18,7 @@ reth-network-api.workspace = true
|
||||
reth-stages.workspace = true
|
||||
reth-prune.workspace = true
|
||||
reth-static-file.workspace = true
|
||||
reth-db.workspace = true
|
||||
reth-db-api.workspace = true
|
||||
reth-primitives.workspace = true
|
||||
reth-rpc-types.workspace = true
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ use futures::Stream;
|
||||
use reth_beacon_consensus::{
|
||||
BeaconConsensusEngineEvent, ConsensusEngineLiveSyncProgress, ForkchoiceStatus,
|
||||
};
|
||||
use reth_db::{database::Database, database_metrics::DatabaseMetadata};
|
||||
use reth_db_api::{database::Database, database_metrics::DatabaseMetadata};
|
||||
use reth_network::{NetworkEvent, NetworkHandle};
|
||||
use reth_network_api::PeersInfo;
|
||||
use reth_primitives::{
|
||||
|
||||
@@ -15,6 +15,7 @@ workspace = true
|
||||
# reth
|
||||
reth-primitives.workspace = true
|
||||
reth-db.workspace = true
|
||||
reth-db-api.workspace = true
|
||||
reth-errors.workspace = true
|
||||
reth-provider.workspace = true
|
||||
reth-tokio-util.workspace = true
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use crate::{segments::SegmentSet, Pruner};
|
||||
use reth_config::PruneConfig;
|
||||
use reth_db::database::Database;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_primitives::{FinishedExExHeight, PruneModes, MAINNET};
|
||||
use reth_provider::ProviderFactory;
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -5,7 +5,7 @@ use crate::{
|
||||
segments::{PruneInput, Segment},
|
||||
Metrics, PrunerError, PrunerEvent,
|
||||
};
|
||||
use reth_db::database::Database;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_primitives::{
|
||||
BlockNumber, FinishedExExHeight, PruneLimiter, PruneMode, PruneProgress, PrunePurpose,
|
||||
PruneSegment, StaticFileSegment,
|
||||
|
||||
@@ -4,7 +4,8 @@ use crate::{
|
||||
},
|
||||
PrunerError,
|
||||
};
|
||||
use reth_db::{database::Database, models::ShardedKey, tables};
|
||||
use reth_db::tables;
|
||||
use reth_db_api::{database::Database, models::ShardedKey};
|
||||
use reth_primitives::{PruneInterruptReason, PruneMode, PruneProgress, PruneSegment};
|
||||
use reth_provider::DatabaseProviderRW;
|
||||
use tracing::{instrument, trace};
|
||||
|
||||
@@ -5,10 +5,10 @@ use crate::{
|
||||
PrunerError,
|
||||
};
|
||||
use itertools::Itertools;
|
||||
use reth_db::{
|
||||
use reth_db::tables;
|
||||
use reth_db_api::{
|
||||
cursor::{DbCursorRO, RangeWalker},
|
||||
database::Database,
|
||||
tables,
|
||||
transaction::DbTxMut,
|
||||
};
|
||||
|
||||
@@ -188,7 +188,8 @@ where
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use assert_matches::assert_matches;
|
||||
use reth_db::{tables, transaction::DbTx};
|
||||
use reth_db::tables;
|
||||
use reth_db_api::transaction::DbTx;
|
||||
use reth_primitives::{
|
||||
BlockNumber, PruneCheckpoint, PruneInterruptReason, PruneLimiter, PruneMode, PruneProgress,
|
||||
PruneSegment, B256, U256,
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
use reth_db::{
|
||||
use reth_db::BlockNumberList;
|
||||
use reth_db_api::{
|
||||
cursor::{DbCursorRO, DbCursorRW},
|
||||
database::Database,
|
||||
models::ShardedKey,
|
||||
table::Table,
|
||||
transaction::DbTxMut,
|
||||
BlockNumberList, DatabaseError,
|
||||
DatabaseError,
|
||||
};
|
||||
use reth_primitives::BlockNumber;
|
||||
use reth_provider::DatabaseProviderRW;
|
||||
|
||||
@@ -14,7 +14,7 @@ pub use account_history::AccountHistory;
|
||||
pub use headers::Headers;
|
||||
pub use receipts::Receipts;
|
||||
pub use receipts_by_logs::ReceiptsByLogs;
|
||||
use reth_db::database::Database;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_primitives::{
|
||||
BlockNumber, PruneCheckpoint, PruneInterruptReason, PruneLimiter, PruneMode, PruneProgress,
|
||||
PruneSegment, TxNumber,
|
||||
|
||||
@@ -2,7 +2,8 @@ use crate::{
|
||||
segments::{PruneInput, PruneOutput, PruneOutputCheckpoint, Segment},
|
||||
PrunerError,
|
||||
};
|
||||
use reth_db::{database::Database, tables};
|
||||
use reth_db::tables;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_primitives::{PruneCheckpoint, PruneMode, PruneProgress, PruneSegment};
|
||||
use reth_provider::{
|
||||
errors::provider::ProviderResult, DatabaseProviderRW, PruneCheckpointWriter,
|
||||
|
||||
@@ -2,7 +2,8 @@ use crate::{
|
||||
segments::{PruneInput, PruneOutput, Segment},
|
||||
PrunerError,
|
||||
};
|
||||
use reth_db::{database::Database, tables};
|
||||
use reth_db::tables;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_primitives::{
|
||||
PruneCheckpoint, PruneMode, PruneProgress, PrunePurpose, PruneSegment, ReceiptsLogPruneConfig,
|
||||
MINIMUM_PRUNING_DISTANCE,
|
||||
@@ -216,7 +217,8 @@ impl<DB: Database> Segment<DB> for ReceiptsByLogs {
|
||||
mod tests {
|
||||
use crate::segments::{receipts_by_logs::ReceiptsByLogs, PruneInput, Segment};
|
||||
use assert_matches::assert_matches;
|
||||
use reth_db::{cursor::DbCursorRO, tables, transaction::DbTx};
|
||||
use reth_db::tables;
|
||||
use reth_db_api::{cursor::DbCursorRO, transaction::DbTx};
|
||||
use reth_primitives::{PruneLimiter, PruneMode, PruneSegment, ReceiptsLogPruneConfig, B256};
|
||||
use reth_provider::{PruneCheckpointReader, TransactionsProvider};
|
||||
use reth_stages::test_utils::{StorageKind, TestStageDB};
|
||||
|
||||
@@ -2,7 +2,8 @@ use crate::{
|
||||
segments::{PruneInput, PruneOutput, PruneOutputCheckpoint, Segment},
|
||||
PrunerError,
|
||||
};
|
||||
use reth_db::{database::Database, tables};
|
||||
use reth_db::tables;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_primitives::{PruneMode, PruneProgress, PruneSegment};
|
||||
use reth_provider::{DatabaseProviderRW, TransactionsProvider};
|
||||
use tracing::{instrument, trace};
|
||||
|
||||
@@ -2,7 +2,7 @@ use crate::segments::{
|
||||
AccountHistory, Receipts, ReceiptsByLogs, Segment, SenderRecovery, StorageHistory,
|
||||
TransactionLookup,
|
||||
};
|
||||
use reth_db::database::Database;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_primitives::PruneModes;
|
||||
|
||||
/// Collection of [Segment]. Thread-safe, allocated on the heap.
|
||||
|
||||
@@ -4,10 +4,10 @@ use crate::{
|
||||
},
|
||||
PrunerError,
|
||||
};
|
||||
use reth_db::{
|
||||
use reth_db::tables;
|
||||
use reth_db_api::{
|
||||
database::Database,
|
||||
models::{storage_sharded_key::StorageShardedKey, BlockNumberAddress},
|
||||
tables,
|
||||
};
|
||||
use reth_primitives::{PruneInterruptReason, PruneMode, PruneProgress, PruneSegment};
|
||||
use reth_provider::DatabaseProviderRW;
|
||||
|
||||
@@ -3,7 +3,8 @@ use crate::{
|
||||
PrunerError,
|
||||
};
|
||||
use rayon::prelude::*;
|
||||
use reth_db::{database::Database, tables};
|
||||
use reth_db::tables;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_primitives::{PruneMode, PruneProgress, PruneSegment};
|
||||
use reth_provider::{DatabaseProviderRW, TransactionsProvider};
|
||||
use tracing::{instrument, trace};
|
||||
|
||||
@@ -2,7 +2,8 @@ use crate::{
|
||||
segments::{PruneInput, PruneOutput, PruneOutputCheckpoint, Segment},
|
||||
PrunerError,
|
||||
};
|
||||
use reth_db::{database::Database, tables};
|
||||
use reth_db::tables;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_primitives::{PruneMode, PruneProgress, PruneSegment};
|
||||
use reth_provider::{DatabaseProviderRW, TransactionsProvider};
|
||||
use tracing::{instrument, trace};
|
||||
|
||||
@@ -14,7 +14,7 @@ workspace = true
|
||||
# reth
|
||||
reth-primitives.workspace = true
|
||||
reth-provider.workspace = true
|
||||
reth-db.workspace = true
|
||||
reth-db-api.workspace = true
|
||||
reth-static-file.workspace = true
|
||||
reth-network-p2p.workspace = true
|
||||
reth-tokio-util.workspace = true
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use crate::{pipeline::BoxedStage, MetricEventsSender, Pipeline, Stage, StageSet};
|
||||
use reth_db::database::Database;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_primitives::{stage::StageId, BlockNumber, B256};
|
||||
use reth_provider::ProviderFactory;
|
||||
use reth_static_file::StaticFileProducer;
|
||||
|
||||
@@ -3,7 +3,7 @@ mod event;
|
||||
pub use crate::pipeline::ctrl::ControlFlow;
|
||||
pub use event::*;
|
||||
use futures_util::Future;
|
||||
use reth_db::database::Database;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_primitives::{
|
||||
constants::BEACON_CONSENSUS_REORG_UNWIND_DEPTH,
|
||||
stage::{PipelineTarget, StageCheckpoint, StageId},
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use crate::Stage;
|
||||
use reth_db::database::Database;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_primitives::stage::StageId;
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use crate::error::StageError;
|
||||
use reth_db::database::Database;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_primitives::{
|
||||
stage::{StageCheckpoint, StageId},
|
||||
BlockNumber, TxNumber,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
#![allow(missing_docs)]
|
||||
|
||||
use crate::{ExecInput, ExecOutput, Stage, StageError, UnwindInput, UnwindOutput};
|
||||
use reth_db::database::Database;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_primitives::stage::StageId;
|
||||
use reth_provider::DatabaseProviderRW;
|
||||
use std::collections::VecDeque;
|
||||
|
||||
@@ -17,6 +17,7 @@ reth-codecs.workspace = true
|
||||
reth-config.workspace = true
|
||||
reth-consensus.workspace = true
|
||||
reth-db.workspace = true
|
||||
reth-db-api.workspace = true
|
||||
reth-etl.workspace = true
|
||||
reth-evm.workspace = true
|
||||
reth-exex.workspace = true
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
#![allow(unreachable_pub)]
|
||||
|
||||
use super::constants;
|
||||
use reth_db::{
|
||||
cursor::DbCursorRO, database::Database, tables, transaction::DbTx, DatabaseError as DbError,
|
||||
use reth_db::tables;
|
||||
use reth_db_api::{
|
||||
cursor::DbCursorRO, database::Database, transaction::DbTx, DatabaseError as DbError,
|
||||
};
|
||||
use reth_primitives::{stage::StageCheckpoint, BlockNumber};
|
||||
use reth_stages::{
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
#![allow(unreachable_pub)]
|
||||
use itertools::concat;
|
||||
use reth_db::{
|
||||
use reth_db::{tables, test_utils::TempDatabase, DatabaseEnv};
|
||||
use reth_db_api::{
|
||||
cursor::DbCursorRO,
|
||||
tables,
|
||||
test_utils::TempDatabase,
|
||||
transaction::{DbTx, DbTxMut},
|
||||
DatabaseEnv,
|
||||
};
|
||||
use reth_primitives::{Account, Address, SealedBlock, B256, U256};
|
||||
use reth_stages::{
|
||||
|
||||
@@ -42,7 +42,7 @@ use crate::{
|
||||
};
|
||||
use reth_config::config::StageConfig;
|
||||
use reth_consensus::Consensus;
|
||||
use reth_db::database::Database;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_evm::execute::BlockExecutorProvider;
|
||||
use reth_network_p2p::{bodies::downloader::BodyDownloader, headers::downloader::HeaderDownloader};
|
||||
use reth_primitives::PruneModes;
|
||||
|
||||
@@ -6,11 +6,11 @@ use std::{
|
||||
use futures_util::TryStreamExt;
|
||||
use tracing::*;
|
||||
|
||||
use reth_db::{
|
||||
use reth_db::tables;
|
||||
use reth_db_api::{
|
||||
cursor::{DbCursorRO, DbCursorRW},
|
||||
database::Database,
|
||||
models::{StoredBlockBodyIndices, StoredBlockOmmers, StoredBlockWithdrawals},
|
||||
tables,
|
||||
transaction::DbTxMut,
|
||||
};
|
||||
use reth_network_p2p::bodies::{downloader::BodyDownloader, response::BlockResponse};
|
||||
@@ -614,24 +614,19 @@ mod tests {
|
||||
}
|
||||
|
||||
mod test_utils {
|
||||
use std::{
|
||||
collections::{HashMap, VecDeque},
|
||||
ops::RangeInclusive,
|
||||
pin::Pin,
|
||||
sync::Arc,
|
||||
task::{Context, Poll},
|
||||
use crate::{
|
||||
stages::bodies::BodyStage,
|
||||
test_utils::{
|
||||
ExecuteStageTestRunner, StageTestRunner, TestRunnerError, TestStageDB,
|
||||
UnwindStageTestRunner,
|
||||
},
|
||||
};
|
||||
|
||||
use futures_util::Stream;
|
||||
|
||||
use reth_db::{
|
||||
use reth_db::{static_file::HeaderMask, tables, test_utils::TempDatabase, DatabaseEnv};
|
||||
use reth_db_api::{
|
||||
cursor::DbCursorRO,
|
||||
models::{StoredBlockBodyIndices, StoredBlockOmmers},
|
||||
static_file::HeaderMask,
|
||||
tables,
|
||||
test_utils::TempDatabase,
|
||||
transaction::{DbTx, DbTxMut},
|
||||
DatabaseEnv,
|
||||
};
|
||||
use reth_network_p2p::{
|
||||
bodies::{
|
||||
@@ -653,13 +648,12 @@ mod tests {
|
||||
generators,
|
||||
generators::{random_block_range, random_signed_tx},
|
||||
};
|
||||
|
||||
use crate::{
|
||||
stages::bodies::BodyStage,
|
||||
test_utils::{
|
||||
ExecuteStageTestRunner, StageTestRunner, TestRunnerError, TestStageDB,
|
||||
UnwindStageTestRunner,
|
||||
},
|
||||
use std::{
|
||||
collections::{HashMap, VecDeque},
|
||||
ops::RangeInclusive,
|
||||
pin::Pin,
|
||||
sync::Arc,
|
||||
task::{Context, Poll},
|
||||
};
|
||||
|
||||
/// The block hash of the genesis block.
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
use crate::stages::MERKLE_STAGE_DEFAULT_CLEAN_THRESHOLD;
|
||||
use num_traits::Zero;
|
||||
use reth_config::config::ExecutionConfig;
|
||||
use reth_db::{
|
||||
cursor::DbCursorRO, database::Database, static_file::HeaderMask, tables, transaction::DbTx,
|
||||
};
|
||||
use reth_db::{static_file::HeaderMask, tables};
|
||||
use reth_db_api::{cursor::DbCursorRO, database::Database, transaction::DbTx};
|
||||
use reth_evm::execute::{BatchBlockExecutionOutput, BatchExecutor, BlockExecutorProvider};
|
||||
use reth_exex::{ExExManagerHandle, ExExNotification};
|
||||
use reth_primitives::{
|
||||
@@ -654,7 +653,7 @@ mod tests {
|
||||
use crate::test_utils::TestStageDB;
|
||||
use alloy_rlp::Decodable;
|
||||
use assert_matches::assert_matches;
|
||||
use reth_db::{models::AccountBeforeTx, transaction::DbTxMut};
|
||||
use reth_db_api::{models::AccountBeforeTx, transaction::DbTxMut};
|
||||
use reth_evm_ethereum::execute::EthExecutorProvider;
|
||||
use reth_execution_errors::BlockValidationError;
|
||||
use reth_primitives::{
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use reth_db::database::Database;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_primitives::stage::{StageCheckpoint, StageId};
|
||||
use reth_provider::DatabaseProviderRW;
|
||||
use reth_stages_api::{ExecInput, ExecOutput, Stage, StageError, UnwindInput, UnwindOutput};
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
use itertools::Itertools;
|
||||
use reth_config::config::{EtlConfig, HashingConfig};
|
||||
use reth_db::{
|
||||
use reth_db::{tables, RawKey, RawTable, RawValue};
|
||||
use reth_db_api::{
|
||||
cursor::{DbCursorRO, DbCursorRW},
|
||||
database::Database,
|
||||
tables,
|
||||
transaction::{DbTx, DbTxMut},
|
||||
RawKey, RawTable, RawValue,
|
||||
};
|
||||
use reth_etl::Collector;
|
||||
use reth_primitives::{
|
||||
@@ -64,7 +63,7 @@ impl AccountHashingStage {
|
||||
provider: &DatabaseProviderRW<DB>,
|
||||
opts: SeedOpts,
|
||||
) -> Result<Vec<(reth_primitives::Address, reth_primitives::Account)>, StageError> {
|
||||
use reth_db::models::AccountBeforeTx;
|
||||
use reth_db_api::models::AccountBeforeTx;
|
||||
use reth_primitives::U256;
|
||||
use reth_provider::providers::StaticFileWriter;
|
||||
use reth_testing_utils::{
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
use itertools::Itertools;
|
||||
use reth_config::config::{EtlConfig, HashingConfig};
|
||||
use reth_db::{
|
||||
codecs::CompactU256,
|
||||
use reth_db::tables;
|
||||
use reth_db_api::{
|
||||
cursor::{DbCursorRO, DbDupCursorRW},
|
||||
database::Database,
|
||||
models::BlockNumberAddress,
|
||||
models::{BlockNumberAddress, CompactU256},
|
||||
table::Decompress,
|
||||
tables,
|
||||
transaction::{DbTx, DbTxMut},
|
||||
};
|
||||
use reth_etl::Collector;
|
||||
@@ -219,7 +218,7 @@ mod tests {
|
||||
};
|
||||
use assert_matches::assert_matches;
|
||||
use rand::Rng;
|
||||
use reth_db::{
|
||||
use reth_db_api::{
|
||||
cursor::{DbCursorRW, DbDupCursorRO},
|
||||
models::StoredBlockBodyIndices,
|
||||
};
|
||||
|
||||
@@ -2,12 +2,11 @@ use futures_util::StreamExt;
|
||||
use reth_codecs::Compact;
|
||||
use reth_config::config::EtlConfig;
|
||||
use reth_consensus::Consensus;
|
||||
use reth_db::{
|
||||
use reth_db::{tables, RawKey, RawTable, RawValue};
|
||||
use reth_db_api::{
|
||||
cursor::{DbCursorRO, DbCursorRW},
|
||||
database::Database,
|
||||
tables,
|
||||
transaction::DbTxMut,
|
||||
RawKey, RawTable, RawValue,
|
||||
};
|
||||
use reth_etl::Collector;
|
||||
use reth_network_p2p::headers::{downloader::HeaderDownloader, error::HeadersDownloaderError};
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
use super::{collect_history_indices, load_history_indices};
|
||||
use reth_config::config::{EtlConfig, IndexHistoryConfig};
|
||||
use reth_db::{
|
||||
database::Database, models::ShardedKey, table::Decode, tables, transaction::DbTxMut,
|
||||
};
|
||||
use reth_db::tables;
|
||||
use reth_db_api::{database::Database, models::ShardedKey, table::Decode, transaction::DbTxMut};
|
||||
use reth_primitives::{
|
||||
stage::{StageCheckpoint, StageId},
|
||||
Address, PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment,
|
||||
@@ -148,14 +147,14 @@ mod tests {
|
||||
TestStageDB, UnwindStageTestRunner,
|
||||
};
|
||||
use itertools::Itertools;
|
||||
use reth_db::{
|
||||
use reth_db::BlockNumberList;
|
||||
use reth_db_api::{
|
||||
cursor::DbCursorRO,
|
||||
models::{
|
||||
sharded_key, sharded_key::NUM_OF_INDICES_IN_SHARD, AccountBeforeTx,
|
||||
StoredBlockBodyIndices,
|
||||
},
|
||||
transaction::DbTx,
|
||||
BlockNumberList,
|
||||
};
|
||||
use reth_primitives::{address, BlockNumber, B256};
|
||||
use reth_provider::providers::StaticFileWriter;
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
use super::{collect_history_indices, load_history_indices};
|
||||
use reth_config::config::{EtlConfig, IndexHistoryConfig};
|
||||
use reth_db::{
|
||||
use reth_db::tables;
|
||||
use reth_db_api::{
|
||||
database::Database,
|
||||
models::{storage_sharded_key::StorageShardedKey, AddressStorageKey, BlockNumberAddress},
|
||||
table::Decode,
|
||||
tables,
|
||||
transaction::DbTxMut,
|
||||
};
|
||||
use reth_primitives::{
|
||||
@@ -155,14 +155,14 @@ mod tests {
|
||||
TestStageDB, UnwindStageTestRunner,
|
||||
};
|
||||
use itertools::Itertools;
|
||||
use reth_db::{
|
||||
use reth_db::BlockNumberList;
|
||||
use reth_db_api::{
|
||||
cursor::DbCursorRO,
|
||||
models::{
|
||||
sharded_key, storage_sharded_key::NUM_OF_INDICES_IN_SHARD, ShardedKey,
|
||||
StoredBlockBodyIndices,
|
||||
},
|
||||
transaction::DbTx,
|
||||
BlockNumberList,
|
||||
};
|
||||
use reth_primitives::{address, b256, Address, BlockNumber, StorageEntry, B256, U256};
|
||||
use reth_provider::providers::StaticFileWriter;
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
use reth_codecs::Compact;
|
||||
use reth_consensus::ConsensusError;
|
||||
use reth_db::{
|
||||
use reth_db::tables;
|
||||
use reth_db_api::{
|
||||
database::Database,
|
||||
tables,
|
||||
transaction::{DbTx, DbTxMut},
|
||||
};
|
||||
use reth_primitives::{
|
||||
@@ -367,7 +367,7 @@ mod tests {
|
||||
TestRunnerError, TestStageDB, UnwindStageTestRunner,
|
||||
};
|
||||
use assert_matches::assert_matches;
|
||||
use reth_db::cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO};
|
||||
use reth_db_api::cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO};
|
||||
use reth_primitives::{
|
||||
keccak256, stage::StageUnitCheckpoint, SealedBlock, StaticFileSegment, StorageEntry, U256,
|
||||
};
|
||||
|
||||
@@ -43,14 +43,16 @@ mod tests {
|
||||
use crate::test_utils::{StorageKind, TestStageDB};
|
||||
use alloy_rlp::Decodable;
|
||||
use reth_db::{
|
||||
cursor::{DbCursorRO, DbCursorRW},
|
||||
mdbx::{cursor::Cursor, RW},
|
||||
table::Table,
|
||||
tables,
|
||||
test_utils::TempDatabase,
|
||||
transaction::{DbTx, DbTxMut},
|
||||
AccountsHistory, DatabaseEnv,
|
||||
};
|
||||
use reth_db_api::{
|
||||
cursor::{DbCursorRO, DbCursorRW},
|
||||
table::Table,
|
||||
transaction::{DbTx, DbTxMut},
|
||||
};
|
||||
use reth_evm_ethereum::execute::EthExecutorProvider;
|
||||
use reth_exex::ExExManagerHandle;
|
||||
use reth_primitives::{
|
||||
|
||||
@@ -1,12 +1,10 @@
|
||||
use reth_config::config::SenderRecoveryConfig;
|
||||
use reth_consensus::ConsensusError;
|
||||
use reth_db::{
|
||||
use reth_db::{static_file::TransactionMask, tables, RawValue};
|
||||
use reth_db_api::{
|
||||
cursor::DbCursorRW,
|
||||
database::Database,
|
||||
static_file::TransactionMask,
|
||||
tables,
|
||||
transaction::{DbTx, DbTxMut},
|
||||
RawValue,
|
||||
};
|
||||
use reth_primitives::{
|
||||
stage::{EntitiesCheckpoint, StageCheckpoint, StageId},
|
||||
@@ -284,7 +282,7 @@ struct FailedSenderRecoveryError {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use assert_matches::assert_matches;
|
||||
use reth_db::cursor::DbCursorRO;
|
||||
use reth_db_api::cursor::DbCursorRO;
|
||||
use reth_primitives::{
|
||||
stage::StageUnitCheckpoint, BlockNumber, PruneCheckpoint, PruneMode, SealedBlock,
|
||||
TransactionSigned, B256,
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
use num_traits::Zero;
|
||||
use reth_config::config::{EtlConfig, TransactionLookupConfig};
|
||||
use reth_db::{
|
||||
use reth_db::{tables, RawKey, RawValue};
|
||||
use reth_db_api::{
|
||||
cursor::{DbCursorRO, DbCursorRW},
|
||||
database::Database,
|
||||
tables,
|
||||
transaction::{DbTx, DbTxMut},
|
||||
RawKey, RawValue,
|
||||
};
|
||||
use reth_etl::Collector;
|
||||
use reth_primitives::{
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
//! Utils for `stages`.
|
||||
use reth_config::config::EtlConfig;
|
||||
use reth_db::{
|
||||
use reth_db::BlockNumberList;
|
||||
use reth_db_api::{
|
||||
cursor::{DbCursorRO, DbCursorRW},
|
||||
models::sharded_key::NUM_OF_INDICES_IN_SHARD,
|
||||
table::{Decompress, Table},
|
||||
transaction::{DbTx, DbTxMut},
|
||||
BlockNumberList, DatabaseError,
|
||||
DatabaseError,
|
||||
};
|
||||
use reth_etl::Collector;
|
||||
use reth_primitives::BlockNumber;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use super::TEST_STAGE_ID;
|
||||
use crate::{StageSet, StageSetBuilder};
|
||||
use reth_db::database::Database;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_stages_api::{test_utils::TestStage, ExecOutput, StageError, UnwindOutput};
|
||||
use std::collections::VecDeque;
|
||||
|
||||
|
||||
@@ -1,15 +1,18 @@
|
||||
use reth_db::{
|
||||
tables,
|
||||
test_utils::{
|
||||
create_test_rw_db, create_test_rw_db_with_path, create_test_static_files_dir, TempDatabase,
|
||||
},
|
||||
DatabaseEnv,
|
||||
};
|
||||
use reth_db_api::{
|
||||
common::KeyValue,
|
||||
cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO},
|
||||
database::Database,
|
||||
models::{AccountBeforeTx, StoredBlockBodyIndices},
|
||||
table::Table,
|
||||
tables,
|
||||
test_utils::{
|
||||
create_test_rw_db, create_test_rw_db_with_path, create_test_static_files_dir, TempDatabase,
|
||||
},
|
||||
transaction::{DbTx, DbTxMut},
|
||||
DatabaseEnv, DatabaseError as DbError,
|
||||
DatabaseError as DbError,
|
||||
};
|
||||
use reth_primitives::{
|
||||
keccak256, Account, Address, BlockNumber, Receipt, SealedBlock, SealedHeader,
|
||||
|
||||
@@ -15,6 +15,7 @@ workspace = true
|
||||
# reth
|
||||
reth-primitives.workspace = true
|
||||
reth-db.workspace = true
|
||||
reth-db-api.workspace = true
|
||||
reth-provider.workspace = true
|
||||
reth-storage-errors.workspace = true
|
||||
reth-nippy-jar.workspace = true
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
use crate::segments::{dataset_for_compression, prepare_jar, Segment, SegmentHeader};
|
||||
use reth_db::{
|
||||
cursor::DbCursorRO, database::Database, static_file::create_static_file_T1_T2_T3, tables,
|
||||
transaction::DbTx, RawKey, RawTable,
|
||||
};
|
||||
use reth_db::{static_file::create_static_file_T1_T2_T3, tables, RawKey, RawTable};
|
||||
use reth_db_api::{cursor::DbCursorRO, database::Database, transaction::DbTx};
|
||||
use reth_primitives::{static_file::SegmentConfig, BlockNumber, StaticFileSegment};
|
||||
use reth_provider::{
|
||||
providers::{StaticFileProvider, StaticFileWriter},
|
||||
|
||||
@@ -9,9 +9,8 @@ pub use headers::Headers;
|
||||
mod receipts;
|
||||
pub use receipts::Receipts;
|
||||
|
||||
use reth_db::{
|
||||
cursor::DbCursorRO, database::Database, table::Table, transaction::DbTx, RawKey, RawTable,
|
||||
};
|
||||
use reth_db::{RawKey, RawTable};
|
||||
use reth_db_api::{cursor::DbCursorRO, database::Database, table::Table, transaction::DbTx};
|
||||
use reth_nippy_jar::NippyJar;
|
||||
use reth_primitives::{
|
||||
static_file::{
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
use crate::segments::{dataset_for_compression, prepare_jar, Segment};
|
||||
use reth_db::{
|
||||
cursor::DbCursorRO, database::Database, static_file::create_static_file_T1, tables,
|
||||
transaction::DbTx,
|
||||
};
|
||||
use reth_db::{static_file::create_static_file_T1, tables};
|
||||
use reth_db_api::{cursor::DbCursorRO, database::Database, transaction::DbTx};
|
||||
use reth_primitives::{
|
||||
static_file::{SegmentConfig, SegmentHeader},
|
||||
BlockNumber, StaticFileSegment, TxNumber,
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
use crate::segments::{dataset_for_compression, prepare_jar, Segment};
|
||||
use reth_db::{
|
||||
cursor::DbCursorRO, database::Database, static_file::create_static_file_T1, tables,
|
||||
transaction::DbTx,
|
||||
};
|
||||
use reth_db::{static_file::create_static_file_T1, tables};
|
||||
use reth_db_api::{cursor::DbCursorRO, database::Database, transaction::DbTx};
|
||||
use reth_primitives::{
|
||||
static_file::{SegmentConfig, SegmentHeader},
|
||||
BlockNumber, StaticFileSegment, TxNumber,
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
use crate::{segments, segments::Segment, StaticFileProducerEvent};
|
||||
use parking_lot::Mutex;
|
||||
use rayon::prelude::*;
|
||||
use reth_db::database::Database;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_primitives::{static_file::HighestStaticFiles, BlockNumber, PruneModes};
|
||||
use reth_provider::{providers::StaticFileWriter, ProviderFactory, StaticFileProviderFactory};
|
||||
use reth_storage_errors::provider::ProviderResult;
|
||||
@@ -228,7 +228,8 @@ mod tests {
|
||||
StaticFileProducer, StaticFileProducerInner, StaticFileTargets,
|
||||
};
|
||||
use assert_matches::assert_matches;
|
||||
use reth_db::{database::Database, test_utils::TempDatabase, transaction::DbTx, DatabaseEnv};
|
||||
use reth_db::{test_utils::TempDatabase, DatabaseEnv};
|
||||
use reth_db_api::{database::Database, transaction::DbTx};
|
||||
use reth_primitives::{
|
||||
static_file::HighestStaticFiles, PruneModes, StaticFileSegment, B256, U256,
|
||||
};
|
||||
|
||||
71
crates/storage/db-api/Cargo.toml
Normal file
71
crates/storage/db-api/Cargo.toml
Normal file
@@ -0,0 +1,71 @@
|
||||
[package]
|
||||
name = "reth-db-api"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
description = "Database abstraction used in reth."
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
# reth
|
||||
reth-codecs.workspace = true
|
||||
reth-primitives.workspace = true
|
||||
reth-storage-errors.workspace = true
|
||||
|
||||
# codecs
|
||||
modular-bitfield.workspace = true
|
||||
parity-scale-codec = { version = "3.2.1", features = ["bytes"] }
|
||||
serde = { workspace = true, default-features = false }
|
||||
|
||||
# metrics
|
||||
metrics.workspace = true
|
||||
|
||||
# misc
|
||||
derive_more.workspace = true
|
||||
bytes.workspace = true
|
||||
|
||||
# arbitrary utils
|
||||
arbitrary = { workspace = true, features = ["derive"], optional = true }
|
||||
proptest = { workspace = true, optional = true }
|
||||
proptest-derive = { workspace = true, optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
# reth libs with arbitrary
|
||||
reth-primitives = { workspace = true, features = ["arbitrary"] }
|
||||
reth-codecs.workspace = true
|
||||
|
||||
rand.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
||||
test-fuzz.workspace = true
|
||||
|
||||
pprof = { workspace = true, features = [
|
||||
"flamegraph",
|
||||
"frame-pointer",
|
||||
"criterion",
|
||||
] }
|
||||
criterion.workspace = true
|
||||
iai-callgrind = "0.10.2"
|
||||
|
||||
arbitrary = { workspace = true, features = ["derive"] }
|
||||
proptest.workspace = true
|
||||
proptest-derive.workspace = true
|
||||
|
||||
paste.workspace = true
|
||||
|
||||
assert_matches.workspace = true
|
||||
|
||||
[features]
|
||||
test-utils = ["arbitrary"]
|
||||
arbitrary = [
|
||||
"reth-primitives/arbitrary",
|
||||
"dep:arbitrary",
|
||||
"dep:proptest",
|
||||
"dep:proptest-derive",
|
||||
]
|
||||
optimism = []
|
||||
@@ -1,4 +1,4 @@
|
||||
use crate::{abstraction::table::*, DatabaseError};
|
||||
use crate::{table::*, DatabaseError};
|
||||
|
||||
/// A key-value pair for table `T`.
|
||||
pub type KeyValue<T> = (<T as Table>::Key, <T as Table>::Value);
|
||||
@@ -17,23 +17,3 @@ pub type IterPairResult<T> = Option<Result<KeyValue<T>, DatabaseError>>;
|
||||
|
||||
/// A value only result for table `T`.
|
||||
pub type ValueOnlyResult<T> = Result<Option<<T as Table>::Value>, DatabaseError>;
|
||||
|
||||
// Sealed trait helper to prevent misuse of the Database API.
|
||||
mod sealed {
|
||||
use crate::{database::Database, mock::DatabaseMock};
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Sealed trait to limit the implementers of the Database trait.
|
||||
pub trait Sealed: Sized {}
|
||||
|
||||
impl<DB: Database> Sealed for &DB {}
|
||||
impl<DB: Database> Sealed for Arc<DB> {}
|
||||
|
||||
#[cfg(feature = "mdbx")]
|
||||
impl Sealed for crate::DatabaseEnv {}
|
||||
impl Sealed for DatabaseMock {}
|
||||
|
||||
#[cfg(any(test, feature = "test-utils"))]
|
||||
impl<DB: Database> Sealed for crate::test_utils::TempDatabase<DB> {}
|
||||
}
|
||||
pub(crate) use sealed::Sealed;
|
||||
@@ -1,5 +1,4 @@
|
||||
use crate::{
|
||||
abstraction::common::Sealed,
|
||||
table::TableImporter,
|
||||
transaction::{DbTx, DbTxMut},
|
||||
DatabaseError,
|
||||
@@ -9,7 +8,7 @@ use std::{fmt::Debug, sync::Arc};
|
||||
/// Main Database trait that can open read-only and read-write transactions.
|
||||
///
|
||||
/// Sealed trait which cannot be implemented by 3rd parties, exposed only for consumption.
|
||||
pub trait Database: Send + Sync + Sealed {
|
||||
pub trait Database: Send + Sync {
|
||||
/// Read-Only database transaction
|
||||
type TX: DbTx + Send + Sync + Debug + 'static;
|
||||
/// Read-Write database transaction
|
||||
83
crates/storage/db-api/src/lib.rs
Normal file
83
crates/storage/db-api/src/lib.rs
Normal file
@@ -0,0 +1,83 @@
|
||||
//! reth's database abstraction layer.
|
||||
//!
|
||||
//! The database abstraction assumes that the underlying store is a KV store subdivided into tables.
|
||||
//!
|
||||
//! One or more changes are tied to a transaction that is atomically committed to the data store at
|
||||
//! the same time. Strong consistency in what data is written and when is important for reth, so it
|
||||
//! is not possible to write data to the database outside of using a transaction.
|
||||
//!
|
||||
//! Good starting points for this crate are:
|
||||
//!
|
||||
//! - [`Database`] for the main database abstraction
|
||||
//! - [`DbTx`] (RO) and [`DbTxMut`] (RW) for the transaction abstractions.
|
||||
//! - [`DbCursorRO`] (RO) and [`DbCursorRW`] (RW) for the cursor abstractions (see below).
|
||||
//!
|
||||
//! # Cursors and Walkers
|
||||
//!
|
||||
//! The abstraction also defines a couple of helpful abstractions for iterating and writing data:
|
||||
//!
|
||||
//! - **Cursors** ([`DbCursorRO`] / [`DbCursorRW`]) for iterating data in a table. Cursors are
|
||||
//! assumed to resolve data in a sorted manner when iterating from start to finish, and it is safe
|
||||
//! to assume that they are efficient at doing so.
|
||||
//! - **Walkers** ([`Walker`] / [`RangeWalker`] / [`ReverseWalker`]) use cursors to walk the entries
|
||||
//! in a table, either fully from a specific point, or over a range.
|
||||
//!
|
||||
//! Dup tables (see below) also have corresponding cursors and walkers (e.g. [`DbDupCursorRO`]).
|
||||
//! These **should** be preferred when working with dup tables, as they provide additional methods
|
||||
//! that are optimized for dup tables.
|
||||
//!
|
||||
//! # Tables
|
||||
//!
|
||||
//! reth has two types of tables: simple KV stores (one key, one value) and dup tables (one key,
|
||||
//! many values). Dup tables can be efficient for certain types of data.
|
||||
//!
|
||||
//! Keys are de/serialized using the [`Encode`] and [`Decode`] traits, and values are de/serialized
|
||||
//! ("compressed") using the [`Compress`] and [`Decompress`] traits.
|
||||
//!
|
||||
//! Tables implement the [`Table`] trait.
|
||||
//!
|
||||
//! [`Database`]: crate::database::Database
|
||||
//! [`DbTx`]: crate::transaction::DbTx
|
||||
//! [`DbTxMut`]: crate::transaction::DbTxMut
|
||||
//! [`DbCursorRO`]: crate::cursor::DbCursorRO
|
||||
//! [`DbCursorRW`]: crate::cursor::DbCursorRW
|
||||
//! [`Walker`]: crate::cursor::Walker
|
||||
//! [`RangeWalker`]: crate::cursor::RangeWalker
|
||||
//! [`ReverseWalker`]: crate::cursor::ReverseWalker
|
||||
//! [`DbDupCursorRO`]: crate::cursor::DbDupCursorRO
|
||||
//! [`Encode`]: crate::table::Encode
|
||||
//! [`Decode`]: crate::table::Decode
|
||||
//! [`Compress`]: crate::table::Compress
|
||||
//! [`Decompress`]: crate::table::Decompress
|
||||
//! [`Table`]: crate::table::Table
|
||||
|
||||
#![doc(
|
||||
html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png",
|
||||
html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256",
|
||||
issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/"
|
||||
)]
|
||||
#![cfg_attr(not(test), warn(unused_crate_dependencies))]
|
||||
#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))]
|
||||
// TODO: remove when https://github.com/proptest-rs/proptest/pull/427 is merged
|
||||
#![allow(unknown_lints, non_local_definitions)]
|
||||
|
||||
/// Common types used throughout the abstraction.
|
||||
pub mod common;
|
||||
/// Cursor database traits.
|
||||
pub mod cursor;
|
||||
/// Database traits.
|
||||
pub mod database;
|
||||
/// Database metrics trait extensions.
|
||||
pub mod database_metrics;
|
||||
pub mod mock;
|
||||
/// Table traits
|
||||
pub mod table;
|
||||
/// Transaction database traits.
|
||||
pub mod transaction;
|
||||
/// Re-exports
|
||||
pub use reth_storage_errors::db::{DatabaseError, DatabaseWriteOperation};
|
||||
|
||||
pub mod models;
|
||||
mod scale;
|
||||
|
||||
mod utils;
|
||||
@@ -11,7 +11,7 @@ use reth_codecs::{derive_arbitrary, Compact};
|
||||
use reth_primitives::{Account, Address, BlockNumber, Buf, StorageKey};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Account as it is saved inside [`AccountChangeSets`][crate::tables::AccountChangeSets].
|
||||
/// Account as it is saved in the database.
|
||||
///
|
||||
/// [`Address`] is the subkey.
|
||||
#[derive_arbitrary(compact)]
|
||||
@@ -57,8 +57,7 @@ impl Compact for AccountBeforeTx {
|
||||
}
|
||||
}
|
||||
|
||||
/// [`BlockNumber`] concatenated with [`Address`]. Used as the key for
|
||||
/// [`StorageChangeSets`](crate::tables::StorageChangeSets)
|
||||
/// [`BlockNumber`] concatenated with [`Address`].
|
||||
///
|
||||
/// Since it's used as a key, it isn't compressed when encoding it.
|
||||
#[derive(
|
||||
@@ -83,7 +83,7 @@ pub struct StoredBlockWithdrawals {
|
||||
pub withdrawals: Withdrawals,
|
||||
}
|
||||
|
||||
/// Hash of the block header. Value for [`CanonicalHeaders`][crate::tables::CanonicalHeaders]
|
||||
/// Hash of the block header.
|
||||
pub type HeaderHash = B256;
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1,10 +1,181 @@
|
||||
//! Implements data structures specific to the database
|
||||
|
||||
use crate::{
|
||||
models::client_version::ClientVersion,
|
||||
table::{Compress, Decompress},
|
||||
tables::models::*,
|
||||
table::{Compress, Decode, Decompress, Encode},
|
||||
DatabaseError,
|
||||
};
|
||||
use reth_codecs::{main_codec, Compact};
|
||||
use reth_primitives::{stage::StageCheckpoint, trie::*, *};
|
||||
use reth_primitives::{
|
||||
stage::StageCheckpoint,
|
||||
trie::{StoredNibbles, StoredNibblesSubKey, *},
|
||||
Address, PruneSegment, B256, *,
|
||||
};
|
||||
|
||||
pub mod accounts;
|
||||
pub mod blocks;
|
||||
pub mod client_version;
|
||||
pub mod integer_list;
|
||||
pub mod sharded_key;
|
||||
pub mod storage_sharded_key;
|
||||
|
||||
pub use accounts::*;
|
||||
pub use blocks::*;
|
||||
pub use client_version::ClientVersion;
|
||||
pub use sharded_key::ShardedKey;
|
||||
|
||||
/// Macro that implements [`Encode`] and [`Decode`] for uint types.
|
||||
macro_rules! impl_uints {
|
||||
($($name:tt),+) => {
|
||||
$(
|
||||
impl Encode for $name {
|
||||
type Encoded = [u8; std::mem::size_of::<$name>()];
|
||||
|
||||
fn encode(self) -> Self::Encoded {
|
||||
self.to_be_bytes()
|
||||
}
|
||||
}
|
||||
|
||||
impl Decode for $name {
|
||||
fn decode<B: AsRef<[u8]>>(value: B) -> Result<Self, $crate::DatabaseError> {
|
||||
Ok(
|
||||
$name::from_be_bytes(
|
||||
value.as_ref().try_into().map_err(|_| $crate::DatabaseError::Decode)?
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
)+
|
||||
};
|
||||
}
|
||||
|
||||
impl_uints!(u64, u32, u16, u8);
|
||||
|
||||
impl Encode for Vec<u8> {
|
||||
type Encoded = Self;
|
||||
|
||||
fn encode(self) -> Self::Encoded {
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl Decode for Vec<u8> {
|
||||
fn decode<B: AsRef<[u8]>>(value: B) -> Result<Self, DatabaseError> {
|
||||
Ok(value.as_ref().to_vec())
|
||||
}
|
||||
}
|
||||
|
||||
impl Encode for Address {
|
||||
type Encoded = [u8; 20];
|
||||
|
||||
fn encode(self) -> Self::Encoded {
|
||||
self.0 .0
|
||||
}
|
||||
}
|
||||
|
||||
impl Decode for Address {
|
||||
fn decode<B: AsRef<[u8]>>(value: B) -> Result<Self, DatabaseError> {
|
||||
Ok(Self::from_slice(value.as_ref()))
|
||||
}
|
||||
}
|
||||
|
||||
impl Encode for B256 {
|
||||
type Encoded = [u8; 32];
|
||||
|
||||
fn encode(self) -> Self::Encoded {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl Decode for B256 {
|
||||
fn decode<B: AsRef<[u8]>>(value: B) -> Result<Self, DatabaseError> {
|
||||
Ok(Self::new(value.as_ref().try_into().map_err(|_| DatabaseError::Decode)?))
|
||||
}
|
||||
}
|
||||
|
||||
impl Encode for String {
|
||||
type Encoded = Vec<u8>;
|
||||
|
||||
fn encode(self) -> Self::Encoded {
|
||||
self.into_bytes()
|
||||
}
|
||||
}
|
||||
|
||||
impl Decode for String {
|
||||
fn decode<B: AsRef<[u8]>>(value: B) -> Result<Self, DatabaseError> {
|
||||
Self::from_utf8(value.as_ref().to_vec()).map_err(|_| DatabaseError::Decode)
|
||||
}
|
||||
}
|
||||
|
||||
impl Encode for StoredNibbles {
|
||||
type Encoded = Vec<u8>;
|
||||
|
||||
// Delegate to the Compact implementation
|
||||
fn encode(self) -> Self::Encoded {
|
||||
let mut buf = Vec::with_capacity(self.0.len());
|
||||
self.to_compact(&mut buf);
|
||||
buf
|
||||
}
|
||||
}
|
||||
|
||||
impl Decode for StoredNibbles {
|
||||
fn decode<B: AsRef<[u8]>>(value: B) -> Result<Self, DatabaseError> {
|
||||
let buf = value.as_ref();
|
||||
Ok(Self::from_compact(buf, buf.len()).0)
|
||||
}
|
||||
}
|
||||
|
||||
impl Encode for StoredNibblesSubKey {
|
||||
type Encoded = Vec<u8>;
|
||||
|
||||
// Delegate to the Compact implementation
|
||||
fn encode(self) -> Self::Encoded {
|
||||
let mut buf = Vec::with_capacity(65);
|
||||
self.to_compact(&mut buf);
|
||||
buf
|
||||
}
|
||||
}
|
||||
|
||||
impl Decode for StoredNibblesSubKey {
|
||||
fn decode<B: AsRef<[u8]>>(value: B) -> Result<Self, DatabaseError> {
|
||||
let buf = value.as_ref();
|
||||
Ok(Self::from_compact(buf, buf.len()).0)
|
||||
}
|
||||
}
|
||||
|
||||
impl Encode for PruneSegment {
|
||||
type Encoded = [u8; 1];
|
||||
|
||||
fn encode(self) -> Self::Encoded {
|
||||
let mut buf = [0u8];
|
||||
self.to_compact(&mut buf.as_mut());
|
||||
buf
|
||||
}
|
||||
}
|
||||
|
||||
impl Decode for PruneSegment {
|
||||
fn decode<B: AsRef<[u8]>>(value: B) -> Result<Self, DatabaseError> {
|
||||
let buf = value.as_ref();
|
||||
Ok(Self::from_compact(buf, buf.len()).0)
|
||||
}
|
||||
}
|
||||
|
||||
impl Encode for ClientVersion {
|
||||
type Encoded = Vec<u8>;
|
||||
|
||||
// Delegate to the Compact implementation
|
||||
fn encode(self) -> Self::Encoded {
|
||||
let mut buf = vec![];
|
||||
self.to_compact(&mut buf);
|
||||
buf
|
||||
}
|
||||
}
|
||||
|
||||
impl Decode for ClientVersion {
|
||||
fn decode<B: AsRef<[u8]>>(value: B) -> Result<Self, DatabaseError> {
|
||||
let buf = value.as_ref();
|
||||
Ok(Self::from_compact(buf, buf.len()).0)
|
||||
}
|
||||
}
|
||||
|
||||
/// Implements compression for Compact type.
|
||||
macro_rules! impl_compression_for_compact {
|
||||
@@ -127,13 +298,7 @@ add_wrapper_struct!((ClientVersion, CompactClientVersion));
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::{
|
||||
codecs::{
|
||||
compact::{CompactClientVersion, CompactU64},
|
||||
CompactU256,
|
||||
},
|
||||
models::{StoredBlockBodyIndices, StoredBlockOmmers, StoredBlockWithdrawals},
|
||||
};
|
||||
use super::*;
|
||||
use reth_primitives::{
|
||||
stage::{
|
||||
AccountHashingCheckpoint, CheckpointBlockRange, EntitiesCheckpoint,
|
||||
@@ -1,13 +1,11 @@
|
||||
//! Sharded key
|
||||
|
||||
use std::hash::Hash;
|
||||
|
||||
use crate::{
|
||||
table::{Decode, Encode},
|
||||
DatabaseError,
|
||||
};
|
||||
use reth_primitives::BlockNumber;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::hash::Hash;
|
||||
|
||||
/// Number of indices in one shard.
|
||||
pub const NUM_OF_INDICES_IN_SHARD: usize = 2_000;
|
||||
@@ -1,5 +1,4 @@
|
||||
//! Storage sharded key
|
||||
|
||||
use crate::{
|
||||
table::{Decode, Encode},
|
||||
DatabaseError,
|
||||
@@ -1,5 +1,5 @@
|
||||
use crate::{
|
||||
abstraction::cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW},
|
||||
cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW},
|
||||
transaction::{DbTx, DbTxMut},
|
||||
DatabaseError,
|
||||
};
|
||||
@@ -80,11 +80,8 @@ impl<T> Value for T where T: Compress + Decompress + Serialize {}
|
||||
/// It allows for the use of codecs. See [`crate::models::ShardedKey`] for a custom
|
||||
/// implementation.
|
||||
pub trait Table: Send + Sync + Debug + 'static {
|
||||
/// The dynamic type of the table.
|
||||
const TABLE: crate::Tables;
|
||||
|
||||
/// The table's name.
|
||||
const NAME: &'static str = Self::TABLE.name();
|
||||
const NAME: &'static str;
|
||||
|
||||
/// Key element of `Table`.
|
||||
///
|
||||
34
crates/storage/db-api/src/utils.rs
Normal file
34
crates/storage/db-api/src/utils.rs
Normal file
@@ -0,0 +1,34 @@
|
||||
#[macro_export]
|
||||
/// Implements the `Arbitrary` trait for types with fixed array types.
|
||||
macro_rules! impl_fixed_arbitrary {
|
||||
($(($name:ident, $size:expr)),*) => {
|
||||
#[cfg(any(test, feature = "arbitrary"))]
|
||||
use arbitrary::{Arbitrary, Unstructured};
|
||||
$(
|
||||
#[cfg(any(test, feature = "arbitrary"))]
|
||||
impl<'a> Arbitrary<'a> for $name {
|
||||
fn arbitrary(u: &mut Unstructured<'a>) -> Result<Self, arbitrary::Error> {
|
||||
let mut buffer = vec![0; $size];
|
||||
u.fill_buffer(buffer.as_mut_slice())?;
|
||||
|
||||
Decode::decode(buffer).map_err(|_| arbitrary::Error::IncorrectFormat)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "arbitrary"))]
|
||||
impl proptest::prelude::Arbitrary for $name {
|
||||
type Parameters = ();
|
||||
type Strategy = proptest::strategy::Map<
|
||||
proptest::collection::VecStrategy<<u8 as proptest::arbitrary::Arbitrary>::Strategy>,
|
||||
fn(Vec<u8>) -> Self,
|
||||
>;
|
||||
|
||||
fn arbitrary_with(args: Self::Parameters) -> Self::Strategy {
|
||||
use proptest::strategy::Strategy;
|
||||
proptest::collection::vec(proptest::arbitrary::any_with::<u8>(args), $size)
|
||||
.prop_map(move |vec| Decode::decode(vec).unwrap())
|
||||
}
|
||||
}
|
||||
)+
|
||||
};
|
||||
}
|
||||
@@ -11,6 +11,7 @@ repository.workspace = true
|
||||
# reth
|
||||
reth-primitives.workspace = true
|
||||
reth-db = { workspace = true, features = ["mdbx"] }
|
||||
reth-db-api.workspace = true
|
||||
reth-provider.workspace = true
|
||||
reth-config.workspace = true
|
||||
reth-trie.workspace = true
|
||||
@@ -29,4 +30,4 @@ serde_json.workspace = true
|
||||
tracing.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
workspace = true
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
|
||||
use reth_codecs::Compact;
|
||||
use reth_config::config::EtlConfig;
|
||||
use reth_db::{database::Database, tables, transaction::DbTxMut, DatabaseError};
|
||||
use reth_db::tables;
|
||||
use reth_db_api::{database::Database, transaction::DbTxMut, DatabaseError};
|
||||
use reth_etl::Collector;
|
||||
use reth_primitives::{
|
||||
stage::{StageCheckpoint, StageId},
|
||||
@@ -522,12 +523,12 @@ struct GenesisAccountWithAddress {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use reth_db::{
|
||||
use reth_db::DatabaseEnv;
|
||||
use reth_db_api::{
|
||||
cursor::DbCursorRO,
|
||||
models::{storage_sharded_key::StorageShardedKey, ShardedKey},
|
||||
table::{Table, TableRow},
|
||||
transaction::DbTx,
|
||||
DatabaseEnv,
|
||||
};
|
||||
use reth_primitives::{
|
||||
Chain, Genesis, IntegerList, GOERLI, GOERLI_GENESIS_HASH, MAINNET, MAINNET_GENESIS_HASH,
|
||||
|
||||
@@ -13,10 +13,10 @@ workspace = true
|
||||
|
||||
[dependencies]
|
||||
# reth
|
||||
reth-db-api.workspace = true
|
||||
reth-primitives.workspace = true
|
||||
reth-fs-util.workspace = true
|
||||
reth-storage-errors.workspace = true
|
||||
reth-codecs.workspace = true
|
||||
reth-libmdbx = { workspace = true, optional = true, features = [
|
||||
"return-borrowed",
|
||||
"read-tx-timeouts",
|
||||
@@ -26,8 +26,6 @@ reth-tracing.workspace = true
|
||||
|
||||
# codecs
|
||||
serde = { workspace = true, default-features = false }
|
||||
parity-scale-codec = { version = "3.2.1", features = ["bytes"] }
|
||||
modular-bitfield.workspace = true
|
||||
|
||||
# metrics
|
||||
reth-metrics.workspace = true
|
||||
@@ -45,17 +43,11 @@ rustc-hash.workspace = true
|
||||
sysinfo = "0.30"
|
||||
|
||||
# arbitrary utils
|
||||
arbitrary = { workspace = true, features = ["derive"], optional = true }
|
||||
proptest = { workspace = true, optional = true }
|
||||
proptest-derive = { workspace = true, optional = true }
|
||||
strum = { workspace = true, features = ["derive"] }
|
||||
once_cell.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
# reth libs with arbitrary
|
||||
reth-primitives = { workspace = true, features = ["arbitrary"] }
|
||||
reth-codecs.workspace = true
|
||||
|
||||
rand.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
||||
@@ -72,7 +64,6 @@ iai-callgrind = "0.10.2"
|
||||
|
||||
arbitrary = { workspace = true, features = ["derive"] }
|
||||
proptest.workspace = true
|
||||
proptest-derive.workspace = true
|
||||
|
||||
paste.workspace = true
|
||||
|
||||
@@ -83,12 +74,7 @@ default = ["mdbx"]
|
||||
test-utils = ["tempfile", "arbitrary"]
|
||||
mdbx = ["reth-libmdbx"]
|
||||
bench = []
|
||||
arbitrary = [
|
||||
"reth-primitives/arbitrary",
|
||||
"dep:arbitrary",
|
||||
"dep:proptest",
|
||||
"dep:proptest-derive",
|
||||
]
|
||||
arbitrary = ["reth-primitives/arbitrary", "reth-db-api/arbitrary"]
|
||||
optimism = []
|
||||
|
||||
[[bench]]
|
||||
|
||||
@@ -3,10 +3,10 @@ use criterion::{
|
||||
black_box, criterion_group, criterion_main, measurement::WallTime, BenchmarkGroup, Criterion,
|
||||
};
|
||||
use pprof::criterion::{Output, PProfProfiler};
|
||||
use reth_db::{
|
||||
use reth_db::tables::*;
|
||||
use reth_db_api::{
|
||||
cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW},
|
||||
table::{Decode, Decompress, DupSort},
|
||||
tables::*,
|
||||
transaction::DbTx,
|
||||
};
|
||||
|
||||
|
||||
@@ -9,7 +9,8 @@ use proptest::{
|
||||
strategy::{Strategy, ValueTree},
|
||||
test_runner::TestRunner,
|
||||
};
|
||||
use reth_db::{cursor::DbCursorRW, TransactionHashNumbers};
|
||||
use reth_db::TransactionHashNumbers;
|
||||
use reth_db_api::cursor::DbCursorRW;
|
||||
use std::collections::HashSet;
|
||||
|
||||
criterion_group! {
|
||||
|
||||
@@ -4,7 +4,7 @@ use iai_callgrind::{
|
||||
library_benchmark, library_benchmark_group, LibraryBenchmarkConfig, RegressionConfig,
|
||||
};
|
||||
use paste::paste;
|
||||
use reth_db::table::{Compress, Decode, Decompress, Encode, Table};
|
||||
use reth_db_api::table::{Compress, Decode, Decompress, Encode, Table};
|
||||
|
||||
mod utils;
|
||||
use utils::*;
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
use reth_db::{
|
||||
use reth_db::{test_utils::create_test_rw_db_with_path, DatabaseEnv};
|
||||
use reth_db_api::{
|
||||
database::Database,
|
||||
table::{Compress, Encode, Table, TableRow},
|
||||
test_utils::create_test_rw_db_with_path,
|
||||
transaction::DbTxMut,
|
||||
DatabaseEnv,
|
||||
};
|
||||
use reth_fs_util as fs;
|
||||
use reth_primitives::Bytes;
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
/// Common types used throughout the abstraction.
|
||||
pub mod common;
|
||||
/// Cursor database traits.
|
||||
pub mod cursor;
|
||||
/// Database traits.
|
||||
pub mod database;
|
||||
/// Database metrics trait extensions.
|
||||
pub mod database_metrics;
|
||||
/// mock
|
||||
pub mod mock;
|
||||
/// Table traits
|
||||
pub mod table;
|
||||
/// Transaction database traits.
|
||||
pub mod transaction;
|
||||
@@ -1,15 +1,17 @@
|
||||
//! Cursor wrapper for libmdbx-sys.
|
||||
|
||||
use crate::{
|
||||
metrics::{DatabaseEnvMetrics, Operation},
|
||||
tables::utils::*,
|
||||
DatabaseError,
|
||||
};
|
||||
use reth_db_api::{
|
||||
common::{PairResult, ValueOnlyResult},
|
||||
cursor::{
|
||||
DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW, DupWalker, RangeWalker,
|
||||
ReverseWalker, Walker,
|
||||
},
|
||||
metrics::{DatabaseEnvMetrics, Operation},
|
||||
table::{Compress, Decode, Decompress, DupSort, Encode, Table},
|
||||
tables::utils::*,
|
||||
DatabaseError,
|
||||
};
|
||||
use reth_libmdbx::{Error as MDBXError, TransactionKind, WriteFlags, RO, RW};
|
||||
use reth_storage_errors::db::{DatabaseErrorInfo, DatabaseWriteError, DatabaseWriteOperation};
|
||||
@@ -52,7 +54,7 @@ impl<K: TransactionKind, T: Table> Cursor<K, T> {
|
||||
f: impl FnOnce(&mut Self) -> R,
|
||||
) -> R {
|
||||
if let Some(metrics) = self.metrics.as_ref().cloned() {
|
||||
metrics.record_operation(T::TABLE, operation, value_size, || f(self))
|
||||
metrics.record_operation(T::NAME, operation, value_size, || f(self))
|
||||
} else {
|
||||
f(self)
|
||||
}
|
||||
|
||||
@@ -1,19 +1,21 @@
|
||||
//! Module that interacts with MDBX.
|
||||
|
||||
use crate::{
|
||||
cursor::{DbCursorRO, DbCursorRW},
|
||||
database::Database,
|
||||
database_metrics::{DatabaseMetadata, DatabaseMetadataValue, DatabaseMetrics},
|
||||
lockfile::StorageLock,
|
||||
metrics::DatabaseEnvMetrics,
|
||||
models::client_version::ClientVersion,
|
||||
tables::{self, TableType, Tables},
|
||||
transaction::{DbTx, DbTxMut},
|
||||
utils::default_page_size,
|
||||
DatabaseError,
|
||||
};
|
||||
use eyre::Context;
|
||||
use metrics::{gauge, Label};
|
||||
use reth_db_api::{
|
||||
cursor::{DbCursorRO, DbCursorRW},
|
||||
database::Database,
|
||||
database_metrics::{DatabaseMetadata, DatabaseMetadataValue, DatabaseMetrics},
|
||||
models::client_version::ClientVersion,
|
||||
transaction::{DbTx, DbTxMut},
|
||||
};
|
||||
use reth_libmdbx::{
|
||||
DatabaseFlags, Environment, EnvironmentFlags, Geometry, MaxReadTransactionDuration, Mode,
|
||||
PageSize, SyncMode, RO, RW,
|
||||
@@ -459,15 +461,17 @@ impl Deref for DatabaseEnv {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::{
|
||||
abstraction::table::{Encode, Table},
|
||||
cursor::{DbDupCursorRO, DbDupCursorRW, ReverseWalker, Walker},
|
||||
models::{AccountBeforeTx, ShardedKey},
|
||||
tables::{
|
||||
AccountsHistory, CanonicalHeaders, Headers, PlainAccountState, PlainStorageState,
|
||||
},
|
||||
test_utils::*,
|
||||
AccountChangeSets,
|
||||
};
|
||||
use reth_db_api::{
|
||||
cursor::{DbDupCursorRO, DbDupCursorRW, ReverseWalker, Walker},
|
||||
models::{AccountBeforeTx, ShardedKey},
|
||||
table::{Encode, Table},
|
||||
};
|
||||
use reth_libmdbx::Error;
|
||||
use reth_primitives::{Account, Address, Header, IntegerList, StorageEntry, B256, U256};
|
||||
use reth_storage_errors::db::{DatabaseWriteError, DatabaseWriteOperation};
|
||||
|
||||
@@ -3,12 +3,13 @@
|
||||
use super::cursor::Cursor;
|
||||
use crate::{
|
||||
metrics::{DatabaseEnvMetrics, Operation, TransactionMode, TransactionOutcome},
|
||||
table::{Compress, DupSort, Encode, Table, TableImporter},
|
||||
tables::{utils::decode_one, Tables},
|
||||
transaction::{DbTx, DbTxMut},
|
||||
tables::utils::decode_one,
|
||||
DatabaseError,
|
||||
};
|
||||
use once_cell::sync::OnceCell;
|
||||
use reth_db_api::{
|
||||
table::{Compress, DupSort, Encode, Table, TableImporter},
|
||||
transaction::{DbTx, DbTxMut},
|
||||
};
|
||||
use reth_libmdbx::{ffi::DBI, CommitLatency, Transaction, TransactionKind, WriteFlags, RW};
|
||||
use reth_storage_errors::db::{DatabaseWriteError, DatabaseWriteOperation};
|
||||
use reth_tracing::tracing::{debug, trace, warn};
|
||||
@@ -36,10 +37,6 @@ pub struct Tx<K: TransactionKind> {
|
||||
///
|
||||
/// If [Some], then metrics are reported.
|
||||
metrics_handler: Option<MetricsHandler<K>>,
|
||||
|
||||
/// Database table handle cache.
|
||||
// TODO: Use `std::sync::OnceLock` once `get_or_try_init` is stable.
|
||||
db_handles: [OnceCell<DBI>; Tables::COUNT],
|
||||
}
|
||||
|
||||
impl<K: TransactionKind> Tx<K> {
|
||||
@@ -69,14 +66,7 @@ impl<K: TransactionKind> Tx<K> {
|
||||
|
||||
#[inline]
|
||||
const fn new_inner(inner: Transaction<K>, metrics_handler: Option<MetricsHandler<K>>) -> Self {
|
||||
// NOTE: These constants are needed to initialize `OnceCell` at compile-time, as array
|
||||
// initialization is not allowed with non-Copy types, and `const { }` blocks are not stable
|
||||
// yet.
|
||||
#[allow(clippy::declare_interior_mutable_const)]
|
||||
const ONCECELL_DBI_NEW: OnceCell<DBI> = OnceCell::new();
|
||||
#[allow(clippy::declare_interior_mutable_const)]
|
||||
const DB_HANDLES: [OnceCell<DBI>; Tables::COUNT] = [ONCECELL_DBI_NEW; Tables::COUNT];
|
||||
Self { inner, db_handles: DB_HANDLES, metrics_handler }
|
||||
Self { inner, metrics_handler }
|
||||
}
|
||||
|
||||
/// Gets this transaction ID.
|
||||
@@ -86,14 +76,10 @@ impl<K: TransactionKind> Tx<K> {
|
||||
|
||||
/// Gets a table database handle if it exists, otherwise creates it.
|
||||
pub fn get_dbi<T: Table>(&self) -> Result<DBI, DatabaseError> {
|
||||
self.db_handles[T::TABLE as usize]
|
||||
.get_or_try_init(|| {
|
||||
self.inner
|
||||
.open_db(Some(T::NAME))
|
||||
.map(|db| db.dbi())
|
||||
.map_err(|e| DatabaseError::Open(e.into()))
|
||||
})
|
||||
.copied()
|
||||
self.inner
|
||||
.open_db(Some(T::NAME))
|
||||
.map(|db| db.dbi())
|
||||
.map_err(|e| DatabaseError::Open(e.into()))
|
||||
}
|
||||
|
||||
/// Create db Cursor
|
||||
@@ -170,7 +156,7 @@ impl<K: TransactionKind> Tx<K> {
|
||||
metrics_handler.log_backtrace_on_long_read_transaction();
|
||||
metrics_handler
|
||||
.env_metrics
|
||||
.record_operation(T::TABLE, operation, value_size, || f(&self.inner))
|
||||
.record_operation(T::NAME, operation, value_size, || f(&self.inner))
|
||||
} else {
|
||||
f(&self.inner)
|
||||
}
|
||||
@@ -391,10 +377,8 @@ impl DbTxMut for Tx<RW> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::{
|
||||
database::Database, mdbx::DatabaseArguments, models::client_version::ClientVersion, tables,
|
||||
transaction::DbTx, DatabaseEnv, DatabaseEnvKind,
|
||||
};
|
||||
use crate::{mdbx::DatabaseArguments, tables, DatabaseEnv, DatabaseEnvKind};
|
||||
use reth_db_api::{database::Database, models::ClientVersion, transaction::DbTx};
|
||||
use reth_libmdbx::MaxReadTransactionDuration;
|
||||
use reth_storage_errors::db::DatabaseError;
|
||||
use std::{sync::atomic::Ordering, thread::sleep, time::Duration};
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user