feat(db): add helper enum for table name (#2935)

Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
This commit is contained in:
Mateusz
2023-06-22 20:04:01 +02:00
committed by GitHub
parent 0dfb757a16
commit 6810cd1295
10 changed files with 333 additions and 200 deletions

View File

@@ -1,8 +1,7 @@
use crate::utils::DbTool;
use clap::Parser;
use eyre::WrapErr;
use reth_db::{database::Database, table::Table, tables};
use serde::Deserialize;
use reth_db::{database::Database, table::Table, TableType, TableViewer, Tables};
use tracing::error;
/// The arguments for the `reth db get` command
@@ -12,75 +11,60 @@ pub struct Command {
///
/// NOTE: The dupsort tables are not supported now.
#[arg()]
pub table: String, // TODO: Convert to enum
pub table: Tables,
/// The key to get content for
/// The key to get content for
#[arg(value_parser = maybe_json_value_parser)]
pub key: String,
}
impl Command {
/// Execute `db get` command
pub fn execute<DB: Database>(self, mut tool: DbTool<'_, DB>) -> eyre::Result<()> {
macro_rules! table_get {
([$($table:ident),*]) => {
match self.table.as_str() {
$(stringify!($table) => {
let table_key = self.table_key::<tables::$table>().wrap_err("Could not parse the given table key.")?;
pub fn execute<DB: Database>(self, tool: &DbTool<'_, DB>) -> eyre::Result<()> {
if self.table.table_type() == TableType::DupSort {
error!(target: "reth::cli", "Unsupported table.");
match tool.get::<tables::$table>(table_key)? {
Some(content) => {
println!("{}", serde_json::to_string_pretty(&content)?);
}
None => {
error!(target: "reth::cli", "No content for the given table key.");
},
};
return Ok(());
},)*
_ => {
error!(target: "reth::cli", "Unknown or unsupported table.");
return Ok(());
}
}
}
return Ok(())
}
table_get!([
CanonicalHeaders,
HeaderTD,
HeaderNumbers,
Headers,
BlockBodyIndices,
BlockOmmers,
BlockWithdrawals,
TransactionBlock,
Transactions,
TxHashNumber,
Receipts,
PlainAccountState,
Bytecodes,
AccountHistory,
StorageHistory,
HashedAccount,
AccountsTrie,
TxSenders,
SyncStage,
SyncStageProgress
]);
self.table.view(&GetValueViewer { tool, args: &self })?;
Ok(())
}
/// Get an instance of key for given table
fn table_key<T: Table>(&self) -> Result<T::Key, eyre::Error>
where
for<'a> T::Key: Deserialize<'a>,
{
assert_eq!(T::NAME, self.table);
pub fn table_key<T: Table>(&self) -> Result<T::Key, eyre::Error> {
assert_eq!(T::NAME, self.table.name());
serde_json::from_str::<T::Key>(&self.key).map_err(|e| eyre::eyre!(e))
}
}
struct GetValueViewer<'a, DB: Database> {
tool: &'a DbTool<'a, DB>,
args: &'a Command,
}
impl<DB: Database> TableViewer<()> for GetValueViewer<'_, DB> {
type Error = eyre::Report;
fn view<T: Table>(&self) -> Result<(), Self::Error> {
// get a key for given table
let key = self.args.table_key::<T>()?;
match self.tool.get::<T>(key)? {
Some(content) => {
println!("{}", serde_json::to_string_pretty(&content)?);
}
None => {
error!(target: "reth::cli", "No content for the given table key.");
}
};
Ok(())
}
}
/// Map the user input value to json
fn maybe_json_value_parser(value: &str) -> Result<String, eyre::Error> {
if serde_json::from_str::<serde::de::IgnoredAny>(value).is_ok() {

85
bin/reth/src/db/list.rs Normal file
View File

@@ -0,0 +1,85 @@
use crate::utils::DbTool;
use clap::Parser;
use super::tui::DbListTUI;
use eyre::WrapErr;
use reth_db::{
database::Database,
mdbx::{Env, WriteMap},
table::Table,
TableType, TableViewer, Tables,
};
use tracing::error;
const DEFAULT_NUM_ITEMS: &str = "5";
#[derive(Parser, Debug)]
/// The arguments for the `reth db list` command
pub struct Command {
/// The table name
table: Tables,
/// Skip first N entries
#[arg(long, short, default_value = "0")]
skip: usize,
/// Reverse the order of the entries. If enabled last table entries are read.
#[arg(long, short, default_value = "false")]
reverse: bool,
/// How many items to take from the walker
#[arg(long, short, default_value = DEFAULT_NUM_ITEMS)]
len: usize,
/// Dump as JSON instead of using TUI.
#[arg(long, short)]
json: bool,
}
impl Command {
/// Execute `db list` command
pub fn execute(self, tool: &DbTool<'_, Env<WriteMap>>) -> eyre::Result<()> {
if self.table.table_type() == TableType::DupSort {
error!(target: "reth::cli", "Unsupported table.");
}
self.table.view(&ListTableViewer { tool, args: &self })?;
Ok(())
}
}
struct ListTableViewer<'a> {
tool: &'a DbTool<'a, Env<WriteMap>>,
args: &'a Command,
}
impl TableViewer<()> for ListTableViewer<'_> {
type Error = eyre::Report;
fn view<T: Table>(&self) -> Result<(), Self::Error> {
self.tool.db.view(|tx| {
let table_db = tx.inner.open_db(Some(self.args.table.name())).wrap_err("Could not open db.")?;
let stats = tx.inner.db_stat(&table_db).wrap_err(format!("Could not find table: {}", stringify!($table)))?;
let total_entries = stats.entries();
if self.args.skip > total_entries - 1 {
error!(
target: "reth::cli",
"Start index {start} is greater than the final entry index ({final_entry_idx}) in the table {table}",
start = self.args.skip,
final_entry_idx = total_entries - 1,
table = self.args.table.name()
);
return Ok(());
}
if self.args.json {
let list_result = self.tool.list::<T>(self.args.skip, self.args.len, self.args.reverse)?.into_iter().collect::<Vec<_>>();
println!("{}", serde_json::to_string_pretty(&list_result)?);
Ok(())
} else {
DbListTUI::<_, T>::new(|skip, count| {
self.tool.list::<T>(skip, count, self.args.reverse).unwrap()
}, self.args.skip, self.args.len, total_entries).run()
}
})??;
Ok(())
}
}

View File

@@ -10,15 +10,15 @@ use eyre::WrapErr;
use human_bytes::human_bytes;
use reth_db::{
database::Database,
tables,
version::{get_db_version, DatabaseVersionError, DB_VERSION},
Tables,
};
use reth_primitives::ChainSpec;
use reth_staged_sync::utils::init::init_db;
use std::sync::Arc;
use tracing::error;
mod get;
mod list;
/// DB List TUI
mod tui;
@@ -57,15 +57,13 @@ pub struct Command {
command: Subcommands,
}
const DEFAULT_NUM_ITEMS: &str = "5";
#[derive(Subcommand, Debug)]
/// `reth db` subcommands
pub enum Subcommands {
/// Lists all the tables, their entry count and their size
Stats,
/// Lists the contents of a table
List(ListArgs),
List(list::Command),
/// Gets the content of a table for the given key
Get(get::Command),
/// Deletes all database entries
@@ -76,25 +74,6 @@ pub enum Subcommands {
Path,
}
#[derive(Parser, Debug)]
/// The arguments for the `reth db list` command
pub struct ListArgs {
/// The table name
table: String, // TODO: Convert to enum
/// Skip first N entries
#[arg(long, short, default_value = "0")]
skip: usize,
/// Reverse the order of the entries. If enabled last table entries are read.
#[arg(long, short, default_value = "false")]
reverse: bool,
/// How many items to take from the walker
#[arg(long, short, default_value = DEFAULT_NUM_ITEMS)]
len: usize,
/// Dump as JSON instead of using TUI.
#[arg(long, short)]
json: bool,
}
impl Command {
/// Execute `db` command
pub async fn execute(self) -> eyre::Result<()> {
@@ -122,7 +101,7 @@ impl Command {
tool.db.view(|tx| {
let mut tables =
tables::TABLES.iter().map(|(_, name)| name).collect::<Vec<_>>();
Tables::ALL.iter().map(|table| table.name()).collect::<Vec<_>>();
tables.sort();
for table in tables {
let table_db =
@@ -157,75 +136,11 @@ impl Command {
println!("{stats_table}");
}
Subcommands::List(args) => {
macro_rules! table_tui {
($arg:expr, $start:expr, $len:expr => [$($table:ident),*]) => {
match $arg {
$(stringify!($table) => {
tool.db.view(|tx| {
let table_db = tx.inner.open_db(Some(stringify!($table))).wrap_err("Could not open db.")?;
let stats = tx.inner.db_stat(&table_db).wrap_err(format!("Could not find table: {}", stringify!($table)))?;
let total_entries = stats.entries();
if $start > total_entries - 1 {
error!(
target: "reth::cli",
"Start index {start} is greater than the final entry index ({final_entry_idx}) in the table {table}",
start = $start,
final_entry_idx = total_entries - 1,
table = stringify!($table)
);
return Ok(());
}
if args.json {
let list_result = tool.list::<tables::$table>(args.skip, args.len,args.reverse)?.into_iter().collect::<Vec<_>>();
println!("{}", serde_json::to_string_pretty(&list_result)?);
Ok(())
} else {
tui::DbListTUI::<_, tables::$table>::new(|skip, count| {
tool.list::<tables::$table>(skip, count, args.reverse).unwrap()
}, $start, $len, total_entries).run()
}
})??
},)*
_ => {
error!(target: "reth::cli", "Unknown table.");
return Ok(());
}
}
}
}
table_tui!(args.table.as_str(), args.skip, args.len => [
CanonicalHeaders,
HeaderTD,
HeaderNumbers,
Headers,
BlockBodyIndices,
BlockOmmers,
BlockWithdrawals,
TransactionBlock,
Transactions,
TxHashNumber,
Receipts,
PlainStorageState,
PlainAccountState,
Bytecodes,
AccountHistory,
StorageHistory,
AccountChangeSet,
StorageChangeSet,
HashedAccount,
HashedStorage,
AccountsTrie,
StoragesTrie,
TxSenders,
SyncStage,
SyncStageProgress
]);
Subcommands::List(command) => {
command.execute(&tool)?;
}
Subcommands::Get(command) => {
command.execute(tool)?;
command.execute(&tool)?;
}
Subcommands::Drop => {
tool.drop(db_path)?;

View File

@@ -73,7 +73,7 @@ pub(crate) async fn initialize_with_db_metrics(
// TODO: A generic stats abstraction for other DB types to deduplicate this and `reth db
// stats`
let _ = db.view(|tx| {
for table in tables::TABLES.iter().map(|(_, name)| name) {
for table in tables::Tables::ALL.iter().map(|table| table.name()) {
let table_db =
tx.inner.open_db(Some(table)).wrap_err("Could not open db.")?;
@@ -89,10 +89,10 @@ pub(crate) async fn initialize_with_db_metrics(
let num_pages = leaf_pages + branch_pages + overflow_pages;
let table_size = page_size * num_pages;
absolute_counter!("db.table_size", table_size as u64, "table" => *table);
absolute_counter!("db.table_pages", leaf_pages as u64, "table" => *table, "type" => "leaf");
absolute_counter!("db.table_pages", branch_pages as u64, "table" => *table, "type" => "branch");
absolute_counter!("db.table_pages", overflow_pages as u64, "table" => *table, "type" => "overflow");
absolute_counter!("db.table_size", table_size as u64, "table" => table);
absolute_counter!("db.table_pages", leaf_pages as u64, "table" => table, "type" => "leaf");
absolute_counter!("db.table_pages", branch_pages as u64, "table" => table, "type" => "branch");
absolute_counter!("db.table_pages", overflow_pages as u64, "table" => table, "type" => "overflow");
}
Ok::<(), eyre::Report>(())

View File

@@ -71,7 +71,7 @@ impl<'a, DB: Database> DbTool<'a, DB> {
/// Grabs the contents of the table within a certain index range and places the
/// entries into a [`HashMap`][std::collections::HashMap].
pub fn list<T: Table>(
&mut self,
&self,
skip: usize,
len: usize,
reverse: bool,
@@ -90,7 +90,7 @@ impl<'a, DB: Database> DbTool<'a, DB> {
}
/// Grabs the content of the table for the given key
pub fn get<T: Table>(&mut self, key: T::Key) -> Result<Option<T::Value>> {
pub fn get<T: Table>(&self, key: T::Key) -> Result<Option<T::Value>> {
self.db.view(|tx| tx.get::<T>(key))?.map_err(|e| eyre::eyre!(e))
}

View File

@@ -4,7 +4,7 @@ use crate::{
DatabaseError,
};
use serde::Serialize;
use serde::{Deserialize, Serialize};
use std::{
fmt::Debug,
marker::{Send, Sync},
@@ -53,9 +53,9 @@ pub trait Decode: Send + Sync + Sized + Debug {
}
/// Generic trait that enforces the database key to implement [`Encode`] and [`Decode`].
pub trait Key: Encode + Decode + Ord + Clone {}
pub trait Key: Encode + Decode + Ord + Clone + Serialize + for<'a> Deserialize<'a> {}
impl<T> Key for T where T: Encode + Decode + Ord + Clone {}
impl<T> Key for T where T: Encode + Decode + Ord + Clone + Serialize + for<'a> Deserialize<'a> {}
/// Generic trait that enforces the database value to implement [`Compress`] and [`Decompress`].
pub trait Value: Compress + Decompress + Serialize {}

View File

@@ -2,7 +2,7 @@
use crate::{
database::{Database, DatabaseGAT},
tables::{TableType, TABLES},
tables::{TableType, Tables},
utils::default_page_size,
DatabaseError,
};
@@ -67,7 +67,7 @@ impl<E: EnvironmentKind> Env<E> {
let env = Env {
inner: Environment::new()
.set_max_dbs(TABLES.len())
.set_max_dbs(Tables::ALL.len())
.set_geometry(Geometry {
// Maximum database size of 4 terabytes
size: Some(0..(4 * TERABYTE)),
@@ -96,13 +96,14 @@ impl<E: EnvironmentKind> Env<E> {
pub fn create_tables(&self) -> Result<(), DatabaseError> {
let tx = self.inner.begin_rw_txn().map_err(|e| DatabaseError::InitTransaction(e.into()))?;
for (table_type, table) in TABLES {
let flags = match table_type {
for table in Tables::ALL {
let flags = match table.table_type() {
TableType::Table => DatabaseFlags::default(),
TableType::DupSort => DatabaseFlags::DUP_SORT,
};
tx.create_db(Some(table), flags).map_err(|e| DatabaseError::TableCreation(e.into()))?;
tx.create_db(Some(table.name()), flags)
.map_err(|e| DatabaseError::TableCreation(e.into()))?;
}
tx.commit().map_err(|e| DatabaseError::Commit(e.into()))?;

View File

@@ -3,14 +3,14 @@
use super::cursor::Cursor;
use crate::{
table::{Compress, DupSort, Encode, Table, TableImporter},
tables::{utils::decode_one, NUM_TABLES, TABLES},
tables::{utils::decode_one, Tables, NUM_TABLES},
transaction::{DbTx, DbTxGAT, DbTxMut, DbTxMutGAT},
DatabaseError,
};
use parking_lot::RwLock;
use reth_libmdbx::{EnvironmentKind, Transaction, TransactionKind, WriteFlags, DBI, RW};
use reth_metrics::metrics::{self, histogram};
use std::{marker::PhantomData, sync::Arc, time::Instant};
use std::{marker::PhantomData, str::FromStr, sync::Arc, time::Instant};
/// Wrapper for the libmdbx transaction.
#[derive(Debug)]
@@ -39,13 +39,9 @@ impl<'env, K: TransactionKind, E: EnvironmentKind> Tx<'env, K, E> {
pub fn get_dbi<T: Table>(&self) -> Result<DBI, DatabaseError> {
let mut handles = self.db_handles.write();
let table_index = TABLES
.iter()
.enumerate()
.find_map(|(idx, (_, table))| (table == &T::NAME).then_some(idx))
.expect("Requested table should be part of `TABLES`.");
let table = Tables::from_str(T::NAME).expect("Requested table should be part of `Tables`.");
let dbi_handle = handles.get_mut(table_index).expect("should exist");
let dbi_handle = handles.get_mut(table as usize).expect("should exist");
if dbi_handle.is_none() {
*dbi_handle = Some(
self.inner

View File

@@ -17,7 +17,9 @@ pub mod models;
mod raw;
pub(crate) mod utils;
use crate::abstraction::table::Table;
pub use raw::{RawDupSort, RawKey, RawTable, RawValue};
use std::{fmt::Display, str::FromStr};
/// Declaration of all Database tables.
use crate::{
@@ -40,7 +42,7 @@ use reth_primitives::{
};
/// Enum for the types of tables present in libmdbx.
#[derive(Debug)]
#[derive(Debug, PartialEq, Copy, Clone)]
pub enum TableType {
/// key value table
Table,
@@ -51,34 +53,138 @@ pub enum TableType {
/// Number of tables that should be present inside database.
pub const NUM_TABLES: usize = 25;
/// Default tables that should be present inside database.
pub const TABLES: [(TableType, &str); NUM_TABLES] = [
(TableType::Table, CanonicalHeaders::const_name()),
(TableType::Table, HeaderTD::const_name()),
(TableType::Table, HeaderNumbers::const_name()),
(TableType::Table, Headers::const_name()),
(TableType::Table, BlockBodyIndices::const_name()),
(TableType::Table, BlockOmmers::const_name()),
(TableType::Table, BlockWithdrawals::const_name()),
(TableType::Table, TransactionBlock::const_name()),
(TableType::Table, Transactions::const_name()),
(TableType::Table, TxHashNumber::const_name()),
(TableType::Table, Receipts::const_name()),
(TableType::Table, PlainAccountState::const_name()),
(TableType::DupSort, PlainStorageState::const_name()),
(TableType::Table, Bytecodes::const_name()),
(TableType::Table, AccountHistory::const_name()),
(TableType::Table, StorageHistory::const_name()),
(TableType::DupSort, AccountChangeSet::const_name()),
(TableType::DupSort, StorageChangeSet::const_name()),
(TableType::Table, HashedAccount::const_name()),
(TableType::DupSort, HashedStorage::const_name()),
(TableType::Table, AccountsTrie::const_name()),
(TableType::DupSort, StoragesTrie::const_name()),
(TableType::Table, TxSenders::const_name()),
(TableType::Table, SyncStage::const_name()),
(TableType::Table, SyncStageProgress::const_name()),
];
/// The general purpose of this is to use with a combination of Tables enum,
/// by implementing a `TableViewer` trait you can operate on db tables in an abstract way.
///
/// # Example
///
/// ```
/// use reth_db::{ table::Table, TableViewer, Tables };
/// use std::str::FromStr;
///
/// let headers = Tables::from_str("Headers").unwrap();
/// let transactions = Tables::from_str("Transactions").unwrap();
///
/// struct MyTableViewer;
///
/// impl TableViewer<()> for MyTableViewer {
/// type Error = &'static str;
///
/// fn view<T: Table>(&self) -> Result<(), Self::Error> {
/// // operate on table in generic way
/// Ok(())
/// }
/// }
///
/// let viewer = MyTableViewer {};
///
/// let _ = headers.view(&viewer);
/// let _ = transactions.view(&viewer);
/// ```
pub trait TableViewer<R> {
/// type of error to return
type Error;
/// operate on table in generic way
fn view<T: Table>(&self) -> Result<R, Self::Error>;
}
macro_rules! tables {
([$(($table:ident, $type:expr)),*]) => {
#[derive(Debug, PartialEq, Copy, Clone)]
/// Default tables that should be present inside database.
pub enum Tables {
$(
#[doc = concat!("Represents a ", stringify!($table), " table")]
$table,
)*
}
impl Tables {
/// Array of all tables in database
pub const ALL: [Tables; NUM_TABLES] = [$(Tables::$table,)*];
/// The name of the given table in database
pub const fn name(&self) -> &str {
match self {
$(Tables::$table => {
$table::NAME
},)*
}
}
/// The type of the given table in database
pub const fn table_type(&self) -> TableType {
match self {
$(Tables::$table => {
$type
},)*
}
}
/// Allows to operate on specific table type
pub fn view<T, R>(&self, visitor: &T) -> Result<R, T::Error>
where
T: TableViewer<R>,
{
match self {
$(Tables::$table => {
visitor.view::<$table>()
},)*
}
}
}
impl Display for Tables {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.name())
}
}
impl FromStr for Tables {
type Err = String;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
$($table::NAME => {
return Ok(Tables::$table)
},)*
_ => {
return Err("Unknown table".to_string())
}
}
}
}
};
}
tables!([
(CanonicalHeaders, TableType::Table),
(HeaderTD, TableType::Table),
(HeaderNumbers, TableType::Table),
(Headers, TableType::Table),
(BlockBodyIndices, TableType::Table),
(BlockOmmers, TableType::Table),
(BlockWithdrawals, TableType::Table),
(TransactionBlock, TableType::Table),
(Transactions, TableType::Table),
(TxHashNumber, TableType::Table),
(Receipts, TableType::Table),
(PlainAccountState, TableType::Table),
(PlainStorageState, TableType::DupSort),
(Bytecodes, TableType::Table),
(AccountHistory, TableType::Table),
(StorageHistory, TableType::Table),
(AccountChangeSet, TableType::DupSort),
(StorageChangeSet, TableType::DupSort),
(HashedAccount, TableType::Table),
(HashedStorage, TableType::DupSort),
(AccountsTrie, TableType::Table),
(StoragesTrie, TableType::DupSort),
(TxSenders, TableType::Table),
(SyncStage, TableType::Table),
(SyncStageProgress, TableType::Table)
]);
#[macro_export]
/// Macro to declare key value table.
@@ -315,3 +421,49 @@ table!(
pub type BlockNumberList = IntegerList;
/// Encoded stage id.
pub type StageId = String;
#[cfg(test)]
mod tests {
use std::str::FromStr;
use crate::*;
const TABLES: [(TableType, &str); NUM_TABLES] = [
(TableType::Table, CanonicalHeaders::const_name()),
(TableType::Table, HeaderTD::const_name()),
(TableType::Table, HeaderNumbers::const_name()),
(TableType::Table, Headers::const_name()),
(TableType::Table, BlockBodyIndices::const_name()),
(TableType::Table, BlockOmmers::const_name()),
(TableType::Table, BlockWithdrawals::const_name()),
(TableType::Table, TransactionBlock::const_name()),
(TableType::Table, Transactions::const_name()),
(TableType::Table, TxHashNumber::const_name()),
(TableType::Table, Receipts::const_name()),
(TableType::Table, PlainAccountState::const_name()),
(TableType::DupSort, PlainStorageState::const_name()),
(TableType::Table, Bytecodes::const_name()),
(TableType::Table, AccountHistory::const_name()),
(TableType::Table, StorageHistory::const_name()),
(TableType::DupSort, AccountChangeSet::const_name()),
(TableType::DupSort, StorageChangeSet::const_name()),
(TableType::Table, HashedAccount::const_name()),
(TableType::DupSort, HashedStorage::const_name()),
(TableType::Table, AccountsTrie::const_name()),
(TableType::DupSort, StoragesTrie::const_name()),
(TableType::Table, TxSenders::const_name()),
(TableType::Table, SyncStage::const_name()),
(TableType::Table, SyncStageProgress::const_name()),
];
#[test]
fn parse_table_from_str() {
for (table_index, &(table_type, table_name)) in TABLES.iter().enumerate() {
let table = Tables::from_str(table_name).unwrap();
assert_eq!(table as usize, table_index);
assert_eq!(table.table_type(), table_type);
assert_eq!(table.name(), table_name);
}
}
}

View File

@@ -2,7 +2,7 @@ use crate::{
table::{Compress, Decode, Decompress, DupSort, Encode, Key, Table, Value},
DatabaseError,
};
use serde::Serialize;
use serde::{Deserialize, Serialize};
/// Raw table that can be used to access any table and its data in raw mode.
/// This is useful for delayed decoding/encoding of data.
@@ -39,7 +39,7 @@ impl<T: DupSort> DupSort for RawDupSort<T> {
}
/// Raw table key.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct RawKey<K: Key> {
key: Vec<u8>,
_phantom: std::marker::PhantomData<K>,