diff --git a/script/research/blockchain-storage-metrics/.gitignore b/script/research/blockchain-storage-metrics/.gitignore new file mode 100644 index 000000000..8bc55fd23 --- /dev/null +++ b/script/research/blockchain-storage-metrics/.gitignore @@ -0,0 +1,3 @@ +/target +Cargo.lock +rustfmt.toml diff --git a/script/research/blockchain-storage-metrics/Cargo.toml b/script/research/blockchain-storage-metrics/Cargo.toml new file mode 100644 index 000000000..72f6ddea9 --- /dev/null +++ b/script/research/blockchain-storage-metrics/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "blockchain-storage-metrics" +version = "0.4.1" +description = "Experimental script to grab blocks from darkfid and measure their usage." +authors = ["Dyne.org foundation "] +repository = "https://codeberg.org/darkrenaissance/darkfi" +license = "AGPL-3.0-only" +edition = "2021" + +[workspace] + +[dependencies] +# Darkfi +darkfi = {path = "../../../", features = ["async-daemonize", "validator", "rpc"]} +darkfi-serial = "0.4.2" + +# Misc +log = "0.4.22" + +# JSON-RPC +async-trait = "0.1.83" +tinyjson = "2.5.1" +url = "2.5.2" + +# Daemon +easy-parallel = "3.3.1" +signal-hook-async-std = "0.2.2" +signal-hook = "0.3.17" +simplelog = "0.12.2" +smol = "2.0.2" + +# Argument parsing +serde = {version = "1.0.210", features = ["derive"]} +structopt = "0.3.26" +structopt-toml = "0.5.1" + +[patch.crates-io] +halo2_proofs = {git="https://github.com/parazyd/halo2", branch="v4"} +halo2_gadgets = {git="https://github.com/parazyd/halo2", branch="v4"} diff --git a/script/research/blockchain-storage-metrics/blockchain_storage_metrics_config.toml b/script/research/blockchain-storage-metrics/blockchain_storage_metrics_config.toml new file mode 100644 index 000000000..2d4a85235 --- /dev/null +++ b/script/research/blockchain-storage-metrics/blockchain_storage_metrics_config.toml @@ -0,0 +1,13 @@ +## blockchain-explorer configuration file +## +## Please make sure you go through all the settings so you can configure +## your daemon properly. +## +## The default values are left commented. They can be overridden either by +## uncommenting, or by using the command-line. + +# darkfid JSON-RPC endpoint +endpoint = "tcp://127.0.0.1:8340" + +# Block height to measure until +height = 0 diff --git a/script/research/blockchain-storage-metrics/src/main.rs b/script/research/blockchain-storage-metrics/src/main.rs new file mode 100644 index 000000000..4c442d979 --- /dev/null +++ b/script/research/blockchain-storage-metrics/src/main.rs @@ -0,0 +1,194 @@ +/* This file is part of DarkFi (https://dark.fi) + * + * Copyright (C) 2020-2024 Dyne.org foundation + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +use std::{fmt, sync::Arc}; + +use log::info; +use smol::stream::StreamExt; +use structopt_toml::{serde::Deserialize, structopt::StructOpt, StructOptToml}; +use url::Url; + +use darkfi::{ + async_daemonize, + blockchain::BlockInfo, + cli_desc, + rpc::{client::RpcClient, jsonrpc::JsonRequest, util::JsonValue}, + util::encoding::base64, + Result, +}; +use darkfi_serial::{deserialize, serialize}; + +const CONFIG_FILE: &str = "blockchain_storage_metrics_config.toml"; +const CONFIG_FILE_CONTENTS: &str = include_str!("../blockchain_storage_metrics_config.toml"); + +#[derive(Clone, Debug, Deserialize, StructOpt, StructOptToml)] +#[serde(default)] +#[structopt(name = "blockchain-storage-metrics", about = cli_desc!())] +struct Args { + #[structopt(short, long)] + /// Configuration file to use + config: Option, + + #[structopt(short, long, default_value = "tcp://127.0.0.1:8340")] + /// darkfid JSON-RPC endpoint + endpoint: Url, + + #[structopt(short, long)] + /// Block height to measure until + height: Option, + + #[structopt(short, long)] + /// Set log file to output into + log: Option, + + #[structopt(short, parse(from_occurrences))] + /// Increase verbosity (-vvv supported) + verbose: u8, +} + +/// Structure representing block storage metrics. +/// Everything is measured in bytes. +struct BlockMetrics { + /// Header height + height: u32, + /// Header hash, + hash: String, + /// Header size + header_size: usize, + /// Transactions count + txs: usize, + /// Transactions size + txs_size: usize, + /// Block producer signature size + signature_size: usize, +} + +impl BlockMetrics { + fn new(block: &BlockInfo) -> Self { + let header_size = serialize(&block.header).len(); + let txs_size = serialize(&block.txs).len(); + let signature_size = serialize(&block.signature).len(); + Self { + height: block.header.height, + hash: block.hash().to_string(), + header_size, + txs: block.txs.len(), + txs_size, + signature_size, + } + } + + /// Compute Block total raw size, in bytes. + fn size(&self) -> usize { + self.header_size + self.txs_size + self.signature_size + } +} + +impl fmt::Display for BlockMetrics { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let s = format!( + "Block {} - {} metrics:\n\t{}: {}\n\t{}: {}\n\t{}: {}\n\t{}: {}\n\t{}: {}", + self.height, + self.hash, + "Header size", + self.header_size, + "Transactions count", + self.txs, + "Transactions size", + self.txs_size, + "Block producer signature size", + self.signature_size, + "Raw size", + self.size(), + ); + + write!(f, "{}", s) + } +} + +async_daemonize!(realmain); +async fn realmain(args: Args, ex: Arc>) -> Result<()> { + // DEV-NOTE: We store everything in memory so don't go crazy with it + info!(target: "blockchain-storage-metrics", "Initializing blockchain storage metrics script..."); + + // Initialize rpc client + let rpc_client = RpcClient::new(args.endpoint, ex).await?; + + // Grab all blocks up to configured height + let height = args.height.unwrap_or_default() + 1; + let mut blocks = Vec::with_capacity(height); + for h in 0..height { + info!(target: "blockchain-storage-metrics", "Requesting block for height: {h}"); + let req = JsonRequest::new( + "blockchain.get_block", + JsonValue::Array(vec![JsonValue::String(h.to_string())]), + ); + let rep = rpc_client.request(req).await?; + let encoded_block = rep.get::().unwrap(); + let bytes = base64::decode(encoded_block).unwrap(); + let block: BlockInfo = deserialize(&bytes)?; + info!(target: "blockchain-storage-metrics", "Retrieved block: {h} - {}", block.hash()); + blocks.push(block); + } + + // Stop rpc client + rpc_client.stop().await; + + // TODO: Create a dummy in memory validator to apply each block + + // Measure each block storage + let mut blocks_metrics = Vec::with_capacity(height); + for block in &blocks { + // TODO: Grab complete storage requirements from the validator + let block_metrics = BlockMetrics::new(block); + info!(target: "blockchain-storage-metrics", "{block_metrics}"); + blocks_metrics.push(block_metrics); + } + + // Measure total storage + let mut total_headers_size = 0_u64; + let mut total_txs = 0_u64; + let mut total_txs_size = 0_u64; + let mut total_signatures_size = 0_u64; + let mut total_size = 0_u64; + for block_metrics in blocks_metrics { + total_headers_size += block_metrics.header_size as u64; + total_txs += block_metrics.txs as u64; + total_txs_size += block_metrics.txs_size as u64; + total_signatures_size += block_metrics.signature_size as u64; + total_size += block_metrics.size() as u64; + } + let metrics = format!( + "Total metrics:\n\t{}: {}\n\t{}: {}\n\t{}: {}\n\t{}: {}\n\t{}: {}", + "Headers size", + total_headers_size, + "Transactions", + total_txs, + "Transactions size", + total_txs_size, + "Signatures size", + total_signatures_size, + "Raw size", + total_size + ); + info!(target: "blockchain-storage-metrics", "{metrics}"); + + // TODO: export metrics as a csv so we can use it to visualize stuff in charts + + Ok(()) +}