diff --git a/Cargo.toml b/Cargo.toml index d9fcf99ec..649b8b49e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,7 +37,9 @@ members = [ "bin/darkwiki/darkwiki-cli", "bin/vanityaddr", "bin/lilith", + "src/sdk", + "src/serial", "src/serial/derive", "src/serial/derive-internal", @@ -79,6 +81,7 @@ toml = {version = "0.5.9", optional = true} # Utilities chrono = {version = "0.4.22", optional = true} +darkfi-serial = {path = "src/serial", optional = true} darkfi-derive = {path = "src/serial/derive", optional = true} darkfi-derive-internal = {path = "src/serial/derive-internal", optional = true} dirs = {version = "4.0.0", optional = true} @@ -166,8 +169,8 @@ blockchain = [ "async-runtime", "crypto", + "darkfi-serial", "net", - "serial", "tx", "node", "util", @@ -194,20 +197,22 @@ crypto = [ "sha2", "subtle", + "darkfi-serial", + "darkfi-serial/crypto", "util", - "serial", "zkas", ] dht = [ "blake3", "chrono", - "fxhash", "rand", "async-runtime", + "darkfi-serial", + "darkfi-serial/collections", + "darkfi-serial/hash", "net", - "serial", ] net = [ @@ -229,7 +234,9 @@ net = [ "url", "async-runtime", - "serial", + "darkfi-serial", + "darkfi-serial/async", + "darkfi-serial/url", "system", "util", ] @@ -243,7 +250,7 @@ node = [ "async-runtime", "blockchain", "crypto", - "serial", + "darkfi-serial", "tx", "wallet", ] @@ -255,8 +262,8 @@ raft = [ "sled", "async-runtime", + "darkfi-serial", "net", - "serial", "util", ] @@ -268,12 +275,8 @@ rpc = [ "url", "async-runtime", + "darkfi-serial", "net", - "serial", -] - -serial = [ - "darkfi-derive", ] system = [ @@ -288,7 +291,7 @@ tx = [ "incrementalmerkletree", "rand", - "serial", + "darkfi-serial", "crypto", ] @@ -304,7 +307,7 @@ util = [ "toml", "url", - "serial", + "darkfi-serial", ] wallet = [ @@ -318,7 +321,7 @@ wallet = [ "libsqlite3-sys", "crypto", - "serial", + "darkfi-serial", "util", ] @@ -341,7 +344,7 @@ zkas = [ "indexmap", "itertools", - "serial", + "darkfi-serial", ] # -----END LIBRARY FEATURES----- diff --git a/src/lib.rs b/src/lib.rs index 4046bca86..c79061136 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,9 +28,6 @@ pub mod raft; #[cfg(feature = "rpc")] pub mod rpc; -#[cfg(feature = "serial")] -pub mod serial; - #[cfg(feature = "system")] pub mod system; diff --git a/src/serial/Cargo.toml b/src/serial/Cargo.toml new file mode 100644 index 000000000..fbecac28b --- /dev/null +++ b/src/serial/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "darkfi-serial" +version = "0.3.0" +homepage = "https://dark.fi" +description = "DarkFi serialization library" +authors = ["darkfi "] +repository = "https://github.com/darkrenaissance/darkfi" +license = "AGPL-3.0-only" +edition = "2021" + +[dependencies] +darkfi-derive = {path = "./derive", optional = true} +futures-lite = {version = "1.12.0", optional = true} + +# Supported types for encoding +blake3 = {version = "1.3.1", optional = true} +fxhash = {version = "0.2.1", optional = true} +incrementalmerkletree = {version = "0.3.0", optional = true} +pasta_curves = {version = "0.4.0", optional = true} +url = {version = "2.3.1", optional = true} + +[features] +default = ["derive"] +derive = ["darkfi-derive"] + +async = ["futures-lite"] +collections = ["fxhash"] +crypto = ["collections", "hash", "incrementalmerkletree", "pasta_curves"] +hash = ["blake3"] diff --git a/src/serial/derive-internal/src/lib.rs b/src/serial/derive-internal/src/lib.rs index bf25458eb..a897eb2f2 100644 --- a/src/serial/derive-internal/src/lib.rs +++ b/src/serial/derive-internal/src/lib.rs @@ -32,7 +32,7 @@ pub fn enum_ser(input: &ItemEnum, cratename: Ident) -> syn::Result let field_type = &field.ty; where_clause.predicates.push( syn::parse2(quote! { - #field_type: #cratename::serial::Encodable + #field_type: #cratename::Encodable }) .unwrap(), ); @@ -60,7 +60,7 @@ pub fn enum_ser(input: &ItemEnum, cratename: Ident) -> syn::Result let field_type = &field.ty; where_clause.predicates.push( syn::parse2(quote! { - #field_type: #cratename::serial::Encodable + #field_type: #cratename::Encodable }) .unwrap(), ); @@ -92,7 +92,7 @@ pub fn enum_ser(input: &ItemEnum, cratename: Ident) -> syn::Result } Ok(quote! { - impl #impl_generics #cratename::serial::Encodable for #name #ty_generics #where_clause { + impl #impl_generics #cratename::Encodable for #name #ty_generics #where_clause { fn encode(&self, mut s: S) -> ::core::result::Result { let variant_idx: u8 = match self { #variant_idx_body @@ -136,13 +136,13 @@ pub fn enum_de(input: &ItemEnum, cratename: Ident) -> syn::Result let field_type = &field.ty; where_clause.predicates.push( syn::parse2(quote! { - #field_type: #cratename::serial::Decodable + #field_type: #cratename::Decodable }) .unwrap(), ); variant_header.extend(quote! { - #field_name: #cratename::serial::Decodable::decode(&mut d)?, + #field_name: #cratename::Decodable::decode(&mut d)?, }); } } @@ -156,13 +156,12 @@ pub fn enum_de(input: &ItemEnum, cratename: Ident) -> syn::Result let field_type = &field.ty; where_clause.predicates.push( syn::parse2(quote! { - #field_type: #cratename::serial::Decodable + #field_type: #cratename::Decodable }) .unwrap(), ); - variant_header - .extend(quote! { #cratename::serial::Decodable::decode(&mut d)?, }); + variant_header.extend(quote! { #cratename::Decodable::decode(&mut d)?, }); } } variant_header = quote! { ( #variant_header ) }; @@ -176,11 +175,11 @@ pub fn enum_de(input: &ItemEnum, cratename: Ident) -> syn::Result } let variant_idx = quote! { - let variant_idx: u8 = #cratename::serial::Decodable::decode(&mut d)?; + let variant_idx: u8 = #cratename::Decodable::decode(&mut d)?; }; Ok(quote! { - impl #impl_generics #cratename::serial::Decodable for #name #ty_generics #where_clause { + impl #impl_generics #cratename::Decodable for #name #ty_generics #where_clause { fn decode(mut d: D) -> ::core::result::Result { #variant_idx @@ -223,7 +222,7 @@ pub fn struct_ser(input: &ItemStruct, cratename: Ident) -> syn::Result syn::Result(&self, mut s: S) -> ::core::result::Result { let mut len = 0; #body @@ -277,13 +276,13 @@ pub fn struct_de(input: &ItemStruct, cratename: Ident) -> syn::Result syn::Result syn::Result(mut d: D) -> ::core::result::Result { Ok(#return_value) } diff --git a/src/serial/derive/src/lib.rs b/src/serial/derive/src/lib.rs index 42bb1991f..33cdd6070 100644 --- a/src/serial/derive/src/lib.rs +++ b/src/serial/derive/src/lib.rs @@ -8,7 +8,7 @@ use darkfi_derive_internal::{enum_de, enum_ser, struct_de, struct_ser}; #[proc_macro_derive(SerialEncodable, attributes(skip_serialize))] pub fn darkfi_serialize(input: TokenStream) -> TokenStream { - let found_crate = crate_name("darkfi").expect("darkfi is found in Cargo.toml"); + let found_crate = crate_name("darkfi-serial").expect("darkfi-serial is found in Cargo.toml"); let found_crate = match found_crate { FoundCrate::Name(name) => name, @@ -36,7 +36,7 @@ pub fn darkfi_serialize(input: TokenStream) -> TokenStream { #[proc_macro_derive(SerialDecodable, attributes(skip_serialize))] pub fn darkfi_deserialize(input: TokenStream) -> TokenStream { - let found_crate = crate_name("darkfi").expect("darkfi is found in Cargo.toml"); + let found_crate = crate_name("darkfi-serial").expect("darkfi-serial is found in Cargo.toml"); let found_crate = match found_crate { FoundCrate::Name(name) => name, diff --git a/src/serial/async_serial.rs b/src/serial/src/async_serial.rs similarity index 96% rename from src/serial/async_serial.rs rename to src/serial/src/async_serial.rs index 1e2f8a15c..3f1940363 100644 --- a/src/serial/async_serial.rs +++ b/src/serial/src/async_serial.rs @@ -1,6 +1,9 @@ -use futures::prelude::*; use std::io::{Error, ErrorKind}; +use futures_lite::{ + AsyncRead, AsyncReadExt as AsyncReadExtFut, AsyncWrite, AsyncWriteExt as AsyncWriteExtFut, +}; + use super::{endian, VarInt}; pub struct AsyncReadExt; diff --git a/src/serial/endian.rs b/src/serial/src/endian.rs similarity index 100% rename from src/serial/endian.rs rename to src/serial/src/endian.rs diff --git a/src/serial/mod.rs b/src/serial/src/lib.rs similarity index 99% rename from src/serial/mod.rs rename to src/serial/src/lib.rs index 6ecaaeaa5..980c2e57a 100644 --- a/src/serial/mod.rs +++ b/src/serial/src/lib.rs @@ -1,12 +1,13 @@ use std::io::{Cursor, Error, ErrorKind, Read, Write}; +#[cfg(feature = "derive")] pub use darkfi_derive::{SerialDecodable, SerialEncodable}; -#[cfg(feature = "async-runtime")] +#[cfg(feature = "async")] mod async_serial; -mod encoding_types; mod endian; +mod types; /// Data which can be encoded in a consensus-consistent way. pub trait Encodable { diff --git a/src/serial/src/types.rs b/src/serial/src/types.rs new file mode 100644 index 000000000..19528319f --- /dev/null +++ b/src/serial/src/types.rs @@ -0,0 +1,16 @@ +//! Encodings for external crates + +#[cfg(feature = "collections")] +mod collections; + +#[cfg(feature = "hash")] +mod hash; + +#[cfg(feature = "incrementalmerkletree")] +mod incrementalmerkletree; + +#[cfg(feature = "pasta_curves")] +mod pasta; + +#[cfg(feature = "url")] +mod url; diff --git a/src/serial/encoding_types.rs b/src/serial/src/types/collections.rs similarity index 80% rename from src/serial/encoding_types.rs rename to src/serial/src/types/collections.rs index f5461d4d7..27d24e860 100644 --- a/src/serial/encoding_types.rs +++ b/src/serial/src/types/collections.rs @@ -1,11 +1,10 @@ -//! Encodings for external crates +//! Serialization of collections use std::{ - collections::{BTreeMap, BTreeSet, HashSet}, + collections::{BTreeMap, BTreeSet, HashMap, HashSet}, io::{Error, Read, Write}, }; -#[allow(unused_imports)] -use super::{Decodable, Encodable, ReadExt, VarInt, WriteExt}; +use crate::{Decodable, Encodable, VarInt}; impl Encodable for HashSet { fn encode(&self, mut s: S) -> Result { @@ -78,23 +77,6 @@ impl Decodable for BTreeSet { } } -#[cfg(feature = "blake3")] -impl Encodable for blake3::Hash { - fn encode(&self, mut s: S) -> Result { - s.write_slice(self.as_bytes())?; - Ok(32) - } -} - -#[cfg(feature = "blake3")] -impl Decodable for blake3::Hash { - fn decode(mut d: D) -> Result { - let mut bytes = [0u8; 32]; - d.read_slice(&mut bytes)?; - Ok(bytes.into()) - } -} - #[cfg(feature = "fxhash")] impl Encodable for fxhash::FxHashMap { fn encode(&self, mut s: S) -> Result { @@ -108,6 +90,31 @@ impl Encodable for fxhash::FxHashMap { } } +impl Decodable for HashMap { + fn decode(mut d: D) -> Result { + let len = VarInt::decode(&mut d)?.0; + let mut ret = HashMap::new(); + for _ in 0..len { + let key: T = Decodable::decode(&mut d)?; + let entry: U = Decodable::decode(&mut d)?; + ret.insert(key, entry); + } + Ok(ret) + } +} + +impl Encodable for HashMap { + fn encode(&self, mut s: S) -> Result { + let mut len = 0; + len += VarInt(self.len() as u64).encode(&mut s)?; + for c in self.iter() { + len += c.0.encode(&mut s)?; + len += c.1.encode(&mut s)?; + } + Ok(len) + } +} + #[cfg(feature = "fxhash")] impl Decodable for fxhash::FxHashMap diff --git a/src/serial/src/types/hash.rs b/src/serial/src/types/hash.rs new file mode 100644 index 000000000..fd01a4d52 --- /dev/null +++ b/src/serial/src/types/hash.rs @@ -0,0 +1,20 @@ +use std::io::{Error, Read, Write}; + +use crate::{Decodable, Encodable, ReadExt, WriteExt}; + +#[cfg(feature = "blake3")] +impl Encodable for blake3::Hash { + fn encode(&self, mut s: S) -> Result { + s.write_slice(self.as_bytes())?; + Ok(32) + } +} + +#[cfg(feature = "blake3")] +impl Decodable for blake3::Hash { + fn decode(mut d: D) -> Result { + let mut bytes = [0u8; 32]; + d.read_slice(&mut bytes)?; + Ok(bytes.into()) + } +} diff --git a/src/serial/src/types/incrementalmerkletree.rs b/src/serial/src/types/incrementalmerkletree.rs new file mode 100644 index 000000000..94f51fc23 --- /dev/null +++ b/src/serial/src/types/incrementalmerkletree.rs @@ -0,0 +1,173 @@ +use std::io::{Error, ErrorKind, Read, Write}; + +use incrementalmerkletree::Hashable; + +use crate::{Decodable, Encodable}; + +impl Encodable for incrementalmerkletree::Position { + fn encode(&self, mut s: S) -> Result { + u64::from(*self).encode(&mut s) + } +} + +impl Decodable for incrementalmerkletree::Position { + fn decode(mut d: D) -> Result { + let dec: u64 = Decodable::decode(&mut d)?; + Ok(Self::try_from(dec).unwrap()) + } +} + +impl Encodable for incrementalmerkletree::bridgetree::Leaf { + fn encode(&self, mut s: S) -> Result { + let mut len = 0; + + match self { + incrementalmerkletree::bridgetree::Leaf::Left(a) => { + len += false.encode(&mut s)?; + len += a.encode(&mut s)?; + } + + incrementalmerkletree::bridgetree::Leaf::Right(a, b) => { + len += true.encode(&mut s)?; + len += a.encode(&mut s)?; + len += b.encode(&mut s)?; + } + } + + Ok(len) + } +} + +impl Decodable for incrementalmerkletree::bridgetree::Leaf { + fn decode(mut d: D) -> Result { + let side: bool = Decodable::decode(&mut d)?; + + match side { + false => { + let a: T = Decodable::decode(&mut d)?; + Ok(Self::Left(a)) + } + true => { + let a: T = Decodable::decode(&mut d)?; + let b: T = Decodable::decode(&mut d)?; + Ok(Self::Right(a, b)) + } + } + } +} + +impl Encodable for incrementalmerkletree::bridgetree::Checkpoint { + fn encode(&self, mut s: S) -> Result { + let mut len = 0; + len += self.bridges_len().encode(&mut s)?; + len += self.is_witnessed().encode(&mut s)?; + len += self.witnessed().encode(&mut s)?; + len += self.forgotten().encode(&mut s)?; + Ok(len) + } +} + +impl Decodable for incrementalmerkletree::bridgetree::Checkpoint { + fn decode(mut d: D) -> Result { + let bridges_len = Decodable::decode(&mut d)?; + let is_witnessed = Decodable::decode(&mut d)?; + let witnessed = Decodable::decode(&mut d)?; + let forgotten = Decodable::decode(&mut d)?; + Ok(Self::from_parts(bridges_len, is_witnessed, witnessed, forgotten)) + } +} + +impl Encodable + for incrementalmerkletree::bridgetree::NonEmptyFrontier +{ + fn encode(&self, mut s: S) -> Result { + let mut len = 0; + len += self.position().encode(&mut s)?; + len += self.leaf().encode(&mut s)?; + len += self.ommers().to_vec().encode(&mut s)?; + Ok(len) + } +} + +impl Decodable + for incrementalmerkletree::bridgetree::NonEmptyFrontier +{ + fn decode(mut d: D) -> Result { + let position = Decodable::decode(&mut d)?; + let leaf = Decodable::decode(&mut d)?; + let ommers = Decodable::decode(&mut d)?; + + match Self::from_parts(position, leaf, ommers) { + Ok(v) => Ok(v), + Err(_) => Err(Error::new(ErrorKind::Other, "FrontierError")), + } + } +} + +impl Encodable for incrementalmerkletree::bridgetree::AuthFragment { + fn encode(&self, mut s: S) -> Result { + let mut len = 0; + len += self.position().encode(&mut s)?; + len += self.altitudes_observed().encode(&mut s)?; + len += self.values().to_vec().encode(&mut s)?; + Ok(len) + } +} + +impl Decodable for incrementalmerkletree::bridgetree::AuthFragment { + fn decode(mut d: D) -> Result { + let position = Decodable::decode(&mut d)?; + let altitudes_observed = Decodable::decode(&mut d)?; + let values = Decodable::decode(&mut d)?; + Ok(Self::from_parts(position, altitudes_observed, values)) + } +} + +impl Encodable for incrementalmerkletree::bridgetree::MerkleBridge { + fn encode(&self, mut s: S) -> Result { + let mut len = 0; + len += self.prior_position().encode(&mut s)?; + len += self.auth_fragments().encode(&mut s)?; + len += self.frontier().encode(&mut s)?; + Ok(len) + } +} + +impl Decodable for incrementalmerkletree::bridgetree::MerkleBridge { + fn decode(mut d: D) -> Result { + let prior_position = Decodable::decode(&mut d)?; + let auth_fragments = Decodable::decode(&mut d)?; + let frontier = Decodable::decode(&mut d)?; + Ok(Self::from_parts(prior_position, auth_fragments, frontier)) + } +} + +impl Encodable + for incrementalmerkletree::bridgetree::BridgeTree +{ + fn encode(&self, mut s: S) -> Result { + let mut len = 0; + len += self.prior_bridges().to_vec().encode(&mut s)?; + len += self.current_bridge().encode(&mut s)?; + len += self.witnessed_indices().encode(&mut s)?; + len += self.checkpoints().to_vec().encode(&mut s)?; + len += self.max_checkpoints().encode(&mut s)?; + Ok(len) + } +} + +impl Decodable + for incrementalmerkletree::bridgetree::BridgeTree +{ + fn decode(mut d: D) -> Result { + let prior_bridges = Decodable::decode(&mut d)?; + let current_bridge = Decodable::decode(&mut d)?; + let saved = Decodable::decode(&mut d)?; + let checkpoints = Decodable::decode(&mut d)?; + let max_checkpoints = Decodable::decode(&mut d)?; + match Self::from_parts(prior_bridges, current_bridge, saved, checkpoints, max_checkpoints) { + Ok(v) => Ok(v), + Err(_) => Err(Error::new(ErrorKind::Other, "BridgeTreeError")), + } + } +} diff --git a/src/serial/src/types/pasta.rs b/src/serial/src/types/pasta.rs new file mode 100644 index 000000000..80f9486e6 --- /dev/null +++ b/src/serial/src/types/pasta.rs @@ -0,0 +1,89 @@ +//! Implementations for pasta curves +use std::io::{Error, ErrorKind, Read, Write}; + +use pasta_curves::{ + group::{ff::PrimeField, GroupEncoding}, + Ep, Eq, Fp, Fq, +}; + +use crate::{Decodable, Encodable, ReadExt, WriteExt}; + +impl Encodable for Fp { + #[inline] + fn encode(&self, mut s: S) -> Result { + s.write_slice(&self.to_repr())?; + Ok(32) + } +} + +impl Decodable for Fp { + #[inline] + fn decode(mut d: D) -> Result { + let mut bytes = [0u8; 32]; + d.read_slice(&mut bytes)?; + match Self::from_repr(bytes).into() { + Some(v) => Ok(v), + None => Err(Error::new(ErrorKind::Other, "Noncanonical bytes for pallas::Base")), + } + } +} + +impl Encodable for Fq { + #[inline] + fn encode(&self, mut s: S) -> Result { + s.write_slice(&self.to_repr())?; + Ok(32) + } +} + +impl Decodable for Fq { + #[inline] + fn decode(mut d: D) -> Result { + let mut bytes = [0u8; 32]; + d.read_slice(&mut bytes)?; + match Self::from_repr(bytes).into() { + Some(v) => Ok(v), + None => Err(Error::new(ErrorKind::Other, "Noncanonical bytes for pallas::Scalar")), + } + } +} + +impl Encodable for Ep { + #[inline] + fn encode(&self, mut s: S) -> Result { + s.write_slice(&self.to_bytes())?; + Ok(32) + } +} + +impl Decodable for Ep { + #[inline] + fn decode(mut d: D) -> Result { + let mut bytes = [0u8; 32]; + d.read_slice(&mut bytes)?; + match Self::from_bytes(&bytes).into() { + Some(v) => Ok(v), + None => Err(Error::new(ErrorKind::Other, "Noncanonical bytes for pallas::Point")), + } + } +} + +impl Encodable for Eq { + #[inline] + fn encode(&self, mut s: S) -> Result { + s.write_slice(&self.to_bytes())?; + Ok(32) + } +} + +impl Decodable for Eq { + #[inline] + fn decode(mut d: D) -> Result { + let mut bytes = [0u8; 32]; + d.read_slice(&mut bytes)?; + match Self::from_bytes(&bytes).into() { + Some(v) => Ok(v), + None => Err(Error::new(ErrorKind::Other, "Noncanonical bytes for vesta::Point")), + } + } +} diff --git a/src/serial/src/types/url.rs b/src/serial/src/types/url.rs new file mode 100644 index 000000000..46bc1775c --- /dev/null +++ b/src/serial/src/types/url.rs @@ -0,0 +1,23 @@ +use std::io::{Error, ErrorKind, Read, Write}; + +use url::Url; + +use crate::{Decodable, Encodable}; + +impl Encodable for Url { + #[inline] + fn encode(&self, s: S) -> Result { + self.as_str().to_string().encode(s) + } +} + +impl Decodable for Url { + #[inline] + fn decode(mut d: D) -> Result { + let s: String = Decodable::decode(&mut d)?; + match Url::parse(&s) { + Ok(v) => Ok(v), + Err(e) => Err(Error::new(ErrorKind::Other, e)), + } + } +}