github/darkfi
1
1
Fork 0
mirror of https://github.com/darkrenaissance/darkfi.git synced 2026-01-09 14:48:08 -05:00
Code Issues Packages Projects Releases Wiki Activity

fud, fu: handle auto ipv6 external addr, improve fu get, improve fu watch

Browse Source
This commit is contained in:
epiphany
2025-04-11 17:58:05 +02:00
committed by epiphany1
parent 8d22f69cf8
commit b8331f71b9
6 changed files with 96 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
bin/fud/fu/src/main.rs
View File

@@ -149,7 +149,7 @@ impl Fu {
JsonValue::String(file_name.unwrap_or_default()), JsonValue::String(file_name.unwrap_or_default()),
]), ]),
); );
let _ = self.rpc_client.request(req).await; let _ = self.rpc_client.request(req).await?;
loop { loop {
match subscription.receive().await { match subscription.receive().await {
@@ -428,31 +428,12 @@ impl Fu {
let info = let info =
params.get("info").unwrap().get::<HashMap<String, JsonValue>>().unwrap(); params.get("info").unwrap().get::<HashMap<String, JsonValue>>().unwrap();
match params.get("event").unwrap().get::<String>().unwrap().as_str() { match params.get("event").unwrap().get::<String>().unwrap().as_str() {
"download_started" => { "download_started" |
let resource = info "file_download_completed" |
.get("resource") "chunk_download_completed" |
.unwrap() "download_completed" |
.get::<HashMap<String, JsonValue>>() "missing_chunks" |
.unwrap(); "file_not_found" => {
update_resource(resource).await;
}
"file_download_completed" => {
let resource = info
.get("resource")
.unwrap()
.get::<HashMap<String, JsonValue>>()
.unwrap();
update_resource(resource).await;
}
"chunk_download_completed" => {
let resource = info
.get("resource")
.unwrap()
.get::<HashMap<String, JsonValue>>()
.unwrap();
update_resource(resource).await;
}
"download_completed" => {
let resource = info let resource = info
.get("resource") .get("resource")
.unwrap() .unwrap()
@@ -478,14 +459,6 @@ impl Fu {
update_resource(resource).await; update_resource(resource).await;
} }
} }
"missing_chunks" => {
let resource = info
.get("resource")
.unwrap()
.get::<HashMap<String, JsonValue>>()
.unwrap();
update_resource(resource).await;
}
"download_error" => { "download_error" => {
// An error that caused the download to be unsuccessful // An error that caused the download to be unsuccessful
} }

51
bin/fud/fud/src/dht.rs
View File

@@ -87,11 +87,10 @@ impl From<DhtNode> for DhtRouterItem {
// TODO: Add a DhtSettings // TODO: Add a DhtSettings
pub struct Dht { pub struct Dht {
pub p2p: P2pPtr, /// Our own node id
pub executor: ExecutorPtr, pub node_id: blake3::Hash,
/// Are we bootstrapped?
pub bootstrapped: Arc<RwLock<bool>>, pub bootstrapped: Arc<RwLock<bool>>,
/// Vec of buckets /// Vec of buckets
pub buckets: Arc<RwLock<Vec<DhtBucket>>>, pub buckets: Arc<RwLock<Vec<DhtBucket>>>,
/// Number of parallel lookup requests /// Number of parallel lookup requests
@@ -106,10 +105,11 @@ pub struct Dht {
pub channel_cache: Arc<RwLock<HashMap<blake3::Hash, u32>>>, pub channel_cache: Arc<RwLock<HashMap<blake3::Hash, u32>>>,
/// Node ID -> Set of keys /// Node ID -> Set of keys
pub router_cache: Arc<RwLock<HashMap<blake3::Hash, HashSet<blake3::Hash>>>>, pub router_cache: Arc<RwLock<HashMap<blake3::Hash, HashSet<blake3::Hash>>>>,
/// Our own node
pub node: DhtNode,
/// Seconds /// Seconds
pub timeout: u64, pub timeout: u64,
pub p2p: P2pPtr,
pub executor: ExecutorPtr,
} }
impl Dht { impl Dht {
pub async fn new( pub async fn new(
@@ -127,7 +127,7 @@ impl Dht {
} }
Self { Self {
p2p: p2p.clone(), node_id: *node_id,
buckets: Arc::new(RwLock::new(buckets)), buckets: Arc::new(RwLock::new(buckets)),
bootstrapped: Arc::new(RwLock::new(false)), bootstrapped: Arc::new(RwLock::new(false)),
alpha: a, alpha: a,
@@ -136,9 +136,10 @@ impl Dht {
node_cache: Arc::new(RwLock::new(HashMap::new())), node_cache: Arc::new(RwLock::new(HashMap::new())),
channel_cache: Arc::new(RwLock::new(HashMap::new())), channel_cache: Arc::new(RwLock::new(HashMap::new())),
router_cache: Arc::new(RwLock::new(HashMap::new())), router_cache: Arc::new(RwLock::new(HashMap::new())),
executor: ex,
node: DhtNode { id: *node_id, addresses: p2p.clone().hosts().external_addrs().await },
timeout, timeout,
p2p: p2p.clone(),
executor: ex,
} }
} }
@@ -152,6 +153,23 @@ impl Dht {
*bootstrapped = true; *bootstrapped = true;
} }
/// Get own node
pub async fn node(&self) -> DhtNode {
DhtNode {
id: self.node_id,
addresses: self
.p2p
.clone()
.hosts()
.external_addrs()
.await
.iter()
.filter(|addr| !addr.to_string().contains("[::]"))
.cloned()
.collect(),
}
}
// Get the distance between `key_1` and `key_2` // Get the distance between `key_1` and `key_2`
pub fn distance(&self, key_1: &blake3::Hash, key_2: &blake3::Hash) -> [u8; 32] { pub fn distance(&self, key_1: &blake3::Hash, key_2: &blake3::Hash) -> [u8; 32] {
let bytes1 = key_1.as_bytes(); let bytes1 = key_1.as_bytes();
@@ -166,7 +184,7 @@ impl Dht {
result_bytes result_bytes
} }
// Sort `nodes` // Sort `nodes` by distance from `key`
pub fn sort_by_distance(&self, nodes: &mut [DhtNode], key: &blake3::Hash) { pub fn sort_by_distance(&self, nodes: &mut [DhtNode], key: &blake3::Hash) {
nodes.sort_by(|a, b| { nodes.sort_by(|a, b| {
let distance_a = BigUint::from_bytes_be(&self.distance(key, &a.id)); let distance_a = BigUint::from_bytes_be(&self.distance(key, &a.id));
@@ -177,10 +195,10 @@ impl Dht {
// key -> bucket index // key -> bucket index
pub async fn get_bucket_index(&self, key: &blake3::Hash) -> usize { pub async fn get_bucket_index(&self, key: &blake3::Hash) -> usize {
if key == &self.node.id { if key == &self.node_id {
return 0 return 0
} }
let distance = self.distance(&self.node.id, key); let distance = self.distance(&self.node_id, key);
let mut leading_zeros = 0; let mut leading_zeros = 0;
for &byte in &distance { for &byte in &distance {
@@ -261,11 +279,12 @@ pub trait DhtHandler {
message: &M, message: &M,
router: DhtRouterPtr, router: DhtRouterPtr,
) -> Result<()> { ) -> Result<()> {
if self.dht().node.addresses.is_empty() { let self_node = self.dht().node().await;
if self_node.addresses.is_empty() {
return Err(().into()); // TODO return Err(().into()); // TODO
} }
self.add_to_router(router.clone(), key, vec![self.dht().node.clone().into()]).await; self.add_to_router(router.clone(), key, vec![self_node.clone().into()]).await;
let nodes = self.lookup_nodes(key).await?; let nodes = self.lookup_nodes(key).await?;
for node in nodes { for node in nodes {
@@ -331,7 +350,7 @@ pub trait DhtHandler {
// Add a node in the correct bucket // Add a node in the correct bucket
async fn add_node(&self, node: DhtNode) { async fn add_node(&self, node: DhtNode) {
// Do not add ourselves to the buckets // Do not add ourselves to the buckets
if node.id == self.dht().node.id { if node.id == self.dht().node_id {
return; return;
} }
@@ -434,7 +453,7 @@ pub trait DhtHandler {
match value_result { match value_result {
Ok(mut nodes) => { Ok(mut nodes) => {
// Remove ourselves from the new nodes // Remove ourselves from the new nodes
nodes.retain(|node| node.id != self.dht().node.id); nodes.retain(|node| node.id != self.dht().node_id);
// Add each new node to our buckets // Add each new node to our buckets
for node in nodes.clone() { for node in nodes.clone() {

36
bin/fud/fud/src/main.rs
View File

@@ -23,16 +23,10 @@ use std::{
sync::Arc, sync::Arc,
}; };
use num_bigint::BigUint;
use resource::{Resource, ResourceStatus};
use rpc::{ChunkDownloadCompleted, ChunkNotFound};
use tasks::FetchReply;
use crate::rpc::FudEvent;
use async_trait::async_trait; use async_trait::async_trait;
use dht::{Dht, DhtHandler, DhtNode, DhtRouterItem, DhtRouterPtr};
use futures::{future::FutureExt, pin_mut, select}; use futures::{future::FutureExt, pin_mut, select};
use log::{debug, error, info, warn}; use log::{debug, error, info, warn};
use num_bigint::BigUint;
use rand::{prelude::IteratorRandom, rngs::OsRng, seq::SliceRandom, RngCore}; use rand::{prelude::IteratorRandom, rngs::OsRng, seq::SliceRandom, RngCore};
use smol::{ use smol::{
channel, channel,
@@ -44,10 +38,14 @@ use smol::{
}; };
use structopt_toml::{structopt::StructOpt, StructOptToml}; use structopt_toml::{structopt::StructOpt, StructOptToml};
use crate::rpc::FudEvent;
use darkfi::{ use darkfi::{
async_daemonize, cli_desc, async_daemonize, cli_desc,
geode::{hash_to_string, ChunkedFile, Geode}, geode::{hash_to_string, ChunkedFile, Geode},
net::{session::SESSION_DEFAULT, settings::SettingsOpt, ChannelPtr, P2p, P2pPtr}, net::{
session::SESSION_DEFAULT, settings::SettingsOpt, ChannelPtr, P2p, P2pPtr,
Settings as NetSettings,
},
rpc::{ rpc::{
jsonrpc::JsonSubscriber, jsonrpc::JsonSubscriber,
p2p_method::HandlerP2p, p2p_method::HandlerP2p,
@@ -58,6 +56,10 @@ use darkfi::{
util::path::expand_path, util::path::expand_path,
Error, Result, Error, Result,
}; };
use dht::{Dht, DhtHandler, DhtNode, DhtRouterItem, DhtRouterPtr};
use resource::{Resource, ResourceStatus};
use rpc::{ChunkDownloadCompleted, ChunkNotFound};
use tasks::FetchReply;
/// P2P protocols /// P2P protocols
mod proto; mod proto;
@@ -184,13 +186,12 @@ impl DhtHandler for Fud {
self.dht().set_bootstrapped().await; self.dht().set_bootstrapped().await;
// Lookup our own node id // Lookup our own node id
let self_node = self.dht().node.clone(); debug!(target: "fud::DhtHandler::on_new_node()", "DHT bootstrapping {}", hash_to_string(&self.dht().node_id));
debug!(target: "fud::DhtHandler::on_new_node()", "DHT bootstrapping {}", hash_to_string(&self_node.id)); let _ = self.lookup_nodes(&self.dht().node_id).await;
let _ = self.lookup_nodes(&self_node.id).await;
} }
// Send keys that are closer to this node than we are // Send keys that are closer to this node than we are
let self_id = self.dht().node.id; let self_id = self.dht().node_id;
let channel = self.get_channel(node).await?; let channel = self.get_channel(node).await?;
for (key, seeders) in self.seeders_router.read().await.iter() { for (key, seeders) in self.seeders_router.read().await.iter() {
let node_distance = BigUint::from_bytes_be(&self.dht().distance(key, &node.id)); let node_distance = BigUint::from_bytes_be(&self.dht().distance(key, &node.id));
@@ -250,12 +251,14 @@ impl Fud {
info!(target: "fud::init()", "Verifying resources..."); info!(target: "fud::init()", "Verifying resources...");
let resources = self.get_seeding_resources().await?; let resources = self.get_seeding_resources().await?;
if self.dht().node.clone().addresses.is_empty() { let self_node = self.dht().node().await;
if self_node.addresses.is_empty() {
return Ok(()); return Ok(());
} }
info!(target: "fud::init()", "Start seeding..."); info!(target: "fud::init()", "Start seeding...");
let self_router_items: Vec<DhtRouterItem> = vec![self.dht().node.clone().into()]; let self_router_items: Vec<DhtRouterItem> = vec![self_node.into()];
for resource in resources { for resource in resources {
self.add_to_router( self.add_to_router(
@@ -689,9 +692,10 @@ async fn realmain(args: Args, ex: Arc<Executor<'static>>) -> Result<()> {
let geode = Geode::new(&basedir).await?; let geode = Geode::new(&basedir).await?;
info!("Instantiating P2P network"); info!("Instantiating P2P network");
let p2p = P2p::new(args.net.into(), ex.clone()).await?; let net_settings: NetSettings = args.net.into();
let p2p = P2p::new(net_settings.clone(), ex.clone()).await?;
let external_addrs = p2p.hosts().external_addrs().await; let external_addrs = net_settings.external_addrs;
if external_addrs.is_empty() { if external_addrs.is_empty() {
warn!(target: "fud::realmain", "No external addresses, you won't be able to seed") warn!(target: "fud::realmain", "No external addresses, you won't be able to seed")

2
bin/fud/fud/src/proto.rs
View File

@@ -175,7 +175,7 @@ impl ProtocolFud {
}; };
info!(target: "fud::ProtocolFud::handle_fud_ping_request()", "Received PING"); info!(target: "fud::ProtocolFud::handle_fud_ping_request()", "Received PING");
let reply = FudPingReply { node: self.fud.dht.node.clone() }; let reply = FudPingReply { node: self.fud.dht.node().await };
match self.channel.send(&reply).await { match self.channel.send(&reply).await {
Ok(()) => continue, Ok(()) => continue,
Err(_e) => continue, Err(_e) => continue,

42
bin/fud/fud/src/rpc.rs
View File

@@ -82,7 +82,9 @@ impl Fud {
// --> {"jsonrpc": "2.0", "method": "put", "params": ["/foo.txt"], "id": 42} // --> {"jsonrpc": "2.0", "method": "put", "params": ["/foo.txt"], "id": 42}
// <-- {"jsonrpc": "2.0", "result: "df4...3db7", "id": 42} // <-- {"jsonrpc": "2.0", "result: "df4...3db7", "id": 42}
async fn put(&self, id: u16, params: JsonValue) -> JsonResult { async fn put(&self, id: u16, params: JsonValue) -> JsonResult {
if self.dht().node.addresses.is_empty() { let self_node = self.dht.node().await;
if self_node.addresses.is_empty() {
error!(target: "fud::put()", "Cannot put file, you don't have any external address"); error!(target: "fud::put()", "Cannot put file, you don't have any external address");
return JsonError::new( return JsonError::new(
ErrorCode::InternalError, ErrorCode::InternalError,
@@ -123,8 +125,7 @@ impl Fud {
}; };
// Announce file // Announce file
let self_node = self.dht.node.clone(); let fud_announce = FudAnnounce { key: file_hash, seeders: vec![self_node.into()] };
let fud_announce = FudAnnounce { key: file_hash, seeders: vec![self_node.clone().into()] };
let _ = self.announce(&file_hash, &fud_announce, self.seeders_router.clone()).await; let _ = self.announce(&file_hash, &fud_announce, self.seeders_router.clone()).await;
JsonResponse::new(JsonValue::String(hash_to_string(&file_hash)), id).into() JsonResponse::new(JsonValue::String(hash_to_string(&file_hash)), id).into()
@@ -142,13 +143,20 @@ impl Fud {
return JsonError::new(ErrorCode::InvalidParams, None, id).into() return JsonError::new(ErrorCode::InvalidParams, None, id).into()
} }
let mut hash_buf = [0u8; 32]; let mut hash_buf = vec![];
match bs58::decode(params[0].get::<String>().unwrap().as_str()).onto(&mut hash_buf) { match bs58::decode(params[0].get::<String>().unwrap().as_str()).onto(&mut hash_buf) {
Ok(_) => {} Ok(_) => {}
Err(_) => return JsonError::new(ErrorCode::InvalidParams, None, id).into(), Err(_) => return JsonError::new(ErrorCode::InvalidParams, None, id).into(),
} }
let file_hash = blake3::Hash::from_bytes(hash_buf); if hash_buf.len() != 32 {
return JsonError::new(ErrorCode::InvalidParams, None, id).into()
}
let mut hash_buf_arr = [0u8; 32];
hash_buf_arr.copy_from_slice(&hash_buf);
let file_hash = blake3::Hash::from_bytes(hash_buf_arr);
let file_hash_str = hash_to_string(&file_hash); let file_hash_str = hash_to_string(&file_hash);
let file_path = match params[1].get::<String>() { let file_path = match params[1].get::<String>() {
@@ -359,6 +367,7 @@ pub struct ChunkNotFound {
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct FileNotFound { pub struct FileNotFound {
pub file_hash: blake3::Hash, pub file_hash: blake3::Hash,
pub resource: Resource,
} }
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct MissingChunks { pub struct MissingChunks {
@@ -434,7 +443,10 @@ impl From<ChunkNotFound> for JsonValue {
} }
impl From<FileNotFound> for JsonValue { impl From<FileNotFound> for JsonValue {
fn from(info: FileNotFound) -> JsonValue { fn from(info: FileNotFound) -> JsonValue {
json_map([("file_hash", JsonValue::String(hash_to_string(&info.file_hash)))]) json_map([
("file_hash", JsonValue::String(hash_to_string(&info.file_hash))),
("resource", info.resource.into()),
])
} }
} }
impl From<MissingChunks> for JsonValue { impl From<MissingChunks> for JsonValue {
@@ -490,7 +502,7 @@ impl From<FudEvent> for JsonValue {
impl Fud { impl Fud {
/// Handle `get` RPC request /// Handle `get` RPC request
pub async fn handle_get(&self, file_hash: &blake3::Hash, file_path: &PathBuf) -> Result<()> { pub async fn handle_get(&self, file_hash: &blake3::Hash, file_path: &PathBuf) -> Result<()> {
let self_node = self.dht().node.clone(); let self_node = self.dht().node().await;
// Add resource to `self.resources` // Add resource to `self.resources`
let resource = Resource { let resource = Resource {
@@ -523,9 +535,19 @@ impl Fud {
Ok(()) => self.geode.get(&i_file_hash).await.unwrap(), Ok(()) => self.geode.get(&i_file_hash).await.unwrap(),
Err(Error::GeodeFileRouteNotFound) => { Err(Error::GeodeFileRouteNotFound) => {
self.event_publisher // Set resource status to `Incomplete` and send FudEvent::FileNotFound
.notify(FudEvent::FileNotFound(FileNotFound { file_hash: *file_hash })) let mut resources_write = self.resources.write().await;
.await; if let Some(resource) = resources_write.get_mut(file_hash) {
resource.status = ResourceStatus::Incomplete;
self.event_publisher
.notify(FudEvent::FileNotFound(FileNotFound {
file_hash: *file_hash,
resource: resource.clone(),
}))
.await;
}
drop(resources_write);
return Err(Error::GeodeFileRouteNotFound); return Err(Error::GeodeFileRouteNotFound);
} }

2
bin/fud/fud/src/tasks.rs
View File

@@ -102,7 +102,7 @@ pub async fn announce_seed_task(fud: Arc<Fud>) -> Result<()> {
loop { loop {
sleep(interval).await; sleep(interval).await;
let seeders = vec![fud.dht().node.clone().into()]; let seeders = vec![fud.dht().node().await.into()];
info!(target: "fud::announce_task()", "Verifying seeds..."); info!(target: "fud::announce_task()", "Verifying seeds...");
let seeding_resources = match fud.get_seeding_resources().await { let seeding_resources = match fud.get_seeding_resources().await {