serial: Move module into a separate crate.

This commit is contained in:
Luther Blissett
2022-10-15 19:32:11 +02:00
parent e5bfd44c74
commit 651a1dd3ce
14 changed files with 421 additions and 61 deletions

View File

@@ -37,7 +37,9 @@ members = [
"bin/darkwiki/darkwiki-cli",
"bin/vanityaddr",
"bin/lilith",
"src/sdk",
"src/serial",
"src/serial/derive",
"src/serial/derive-internal",
@@ -79,6 +81,7 @@ toml = {version = "0.5.9", optional = true}
# Utilities
chrono = {version = "0.4.22", optional = true}
darkfi-serial = {path = "src/serial", optional = true}
darkfi-derive = {path = "src/serial/derive", optional = true}
darkfi-derive-internal = {path = "src/serial/derive-internal", optional = true}
dirs = {version = "4.0.0", optional = true}
@@ -166,8 +169,8 @@ blockchain = [
"async-runtime",
"crypto",
"darkfi-serial",
"net",
"serial",
"tx",
"node",
"util",
@@ -194,20 +197,22 @@ crypto = [
"sha2",
"subtle",
"darkfi-serial",
"darkfi-serial/crypto",
"util",
"serial",
"zkas",
]
dht = [
"blake3",
"chrono",
"fxhash",
"rand",
"async-runtime",
"darkfi-serial",
"darkfi-serial/collections",
"darkfi-serial/hash",
"net",
"serial",
]
net = [
@@ -229,7 +234,9 @@ net = [
"url",
"async-runtime",
"serial",
"darkfi-serial",
"darkfi-serial/async",
"darkfi-serial/url",
"system",
"util",
]
@@ -243,7 +250,7 @@ node = [
"async-runtime",
"blockchain",
"crypto",
"serial",
"darkfi-serial",
"tx",
"wallet",
]
@@ -255,8 +262,8 @@ raft = [
"sled",
"async-runtime",
"darkfi-serial",
"net",
"serial",
"util",
]
@@ -268,12 +275,8 @@ rpc = [
"url",
"async-runtime",
"darkfi-serial",
"net",
"serial",
]
serial = [
"darkfi-derive",
]
system = [
@@ -288,7 +291,7 @@ tx = [
"incrementalmerkletree",
"rand",
"serial",
"darkfi-serial",
"crypto",
]
@@ -304,7 +307,7 @@ util = [
"toml",
"url",
"serial",
"darkfi-serial",
]
wallet = [
@@ -318,7 +321,7 @@ wallet = [
"libsqlite3-sys",
"crypto",
"serial",
"darkfi-serial",
"util",
]
@@ -341,7 +344,7 @@ zkas = [
"indexmap",
"itertools",
"serial",
"darkfi-serial",
]
# -----END LIBRARY FEATURES-----

View File

@@ -28,9 +28,6 @@ pub mod raft;
#[cfg(feature = "rpc")]
pub mod rpc;
#[cfg(feature = "serial")]
pub mod serial;
#[cfg(feature = "system")]
pub mod system;

29
src/serial/Cargo.toml Normal file
View File

@@ -0,0 +1,29 @@
[package]
name = "darkfi-serial"
version = "0.3.0"
homepage = "https://dark.fi"
description = "DarkFi serialization library"
authors = ["darkfi <dev@dark.fi>"]
repository = "https://github.com/darkrenaissance/darkfi"
license = "AGPL-3.0-only"
edition = "2021"
[dependencies]
darkfi-derive = {path = "./derive", optional = true}
futures-lite = {version = "1.12.0", optional = true}
# Supported types for encoding
blake3 = {version = "1.3.1", optional = true}
fxhash = {version = "0.2.1", optional = true}
incrementalmerkletree = {version = "0.3.0", optional = true}
pasta_curves = {version = "0.4.0", optional = true}
url = {version = "2.3.1", optional = true}
[features]
default = ["derive"]
derive = ["darkfi-derive"]
async = ["futures-lite"]
collections = ["fxhash"]
crypto = ["collections", "hash", "incrementalmerkletree", "pasta_curves"]
hash = ["blake3"]

View File

@@ -32,7 +32,7 @@ pub fn enum_ser(input: &ItemEnum, cratename: Ident) -> syn::Result<TokenStream2>
let field_type = &field.ty;
where_clause.predicates.push(
syn::parse2(quote! {
#field_type: #cratename::serial::Encodable
#field_type: #cratename::Encodable
})
.unwrap(),
);
@@ -60,7 +60,7 @@ pub fn enum_ser(input: &ItemEnum, cratename: Ident) -> syn::Result<TokenStream2>
let field_type = &field.ty;
where_clause.predicates.push(
syn::parse2(quote! {
#field_type: #cratename::serial::Encodable
#field_type: #cratename::Encodable
})
.unwrap(),
);
@@ -92,7 +92,7 @@ pub fn enum_ser(input: &ItemEnum, cratename: Ident) -> syn::Result<TokenStream2>
}
Ok(quote! {
impl #impl_generics #cratename::serial::Encodable for #name #ty_generics #where_clause {
impl #impl_generics #cratename::Encodable for #name #ty_generics #where_clause {
fn encode<S: std::io::Write>(&self, mut s: S) -> ::core::result::Result<usize, std::io::Error> {
let variant_idx: u8 = match self {
#variant_idx_body
@@ -136,13 +136,13 @@ pub fn enum_de(input: &ItemEnum, cratename: Ident) -> syn::Result<TokenStream2>
let field_type = &field.ty;
where_clause.predicates.push(
syn::parse2(quote! {
#field_type: #cratename::serial::Decodable
#field_type: #cratename::Decodable
})
.unwrap(),
);
variant_header.extend(quote! {
#field_name: #cratename::serial::Decodable::decode(&mut d)?,
#field_name: #cratename::Decodable::decode(&mut d)?,
});
}
}
@@ -156,13 +156,12 @@ pub fn enum_de(input: &ItemEnum, cratename: Ident) -> syn::Result<TokenStream2>
let field_type = &field.ty;
where_clause.predicates.push(
syn::parse2(quote! {
#field_type: #cratename::serial::Decodable
#field_type: #cratename::Decodable
})
.unwrap(),
);
variant_header
.extend(quote! { #cratename::serial::Decodable::decode(&mut d)?, });
variant_header.extend(quote! { #cratename::Decodable::decode(&mut d)?, });
}
}
variant_header = quote! { ( #variant_header ) };
@@ -176,11 +175,11 @@ pub fn enum_de(input: &ItemEnum, cratename: Ident) -> syn::Result<TokenStream2>
}
let variant_idx = quote! {
let variant_idx: u8 = #cratename::serial::Decodable::decode(&mut d)?;
let variant_idx: u8 = #cratename::Decodable::decode(&mut d)?;
};
Ok(quote! {
impl #impl_generics #cratename::serial::Decodable for #name #ty_generics #where_clause {
impl #impl_generics #cratename::Decodable for #name #ty_generics #where_clause {
fn decode<D: std::io::Read>(mut d: D) -> ::core::result::Result<Self, std::io::Error> {
#variant_idx
@@ -223,7 +222,7 @@ pub fn struct_ser(input: &ItemStruct, cratename: Ident) -> syn::Result<TokenStre
let field_type = &field.ty;
where_clause.predicates.push(
syn::parse2(quote! {
#field_type: #cratename::serial::Encodable
#field_type: #cratename::Encodable
})
.unwrap(),
);
@@ -245,7 +244,7 @@ pub fn struct_ser(input: &ItemStruct, cratename: Ident) -> syn::Result<TokenStre
}
Ok(quote! {
impl #impl_generics #cratename::serial::Encodable for #name #ty_generics #where_clause {
impl #impl_generics #cratename::Encodable for #name #ty_generics #where_clause {
fn encode<S: std::io::Write>(&self, mut s: S) -> ::core::result::Result<usize, std::io::Error> {
let mut len = 0;
#body
@@ -277,13 +276,13 @@ pub fn struct_de(input: &ItemStruct, cratename: Ident) -> syn::Result<TokenStrea
let field_type = &field.ty;
where_clause.predicates.push(
syn::parse2(quote! {
#field_type: #cratename::serial::Decodable
#field_type: #cratename::Decodable
})
.unwrap(),
);
quote! {
#field_name: #cratename::serial::Decodable::decode(&mut d)?,
#field_name: #cratename::Decodable::decode(&mut d)?,
}
};
body.extend(delta);
@@ -296,7 +295,7 @@ pub fn struct_de(input: &ItemStruct, cratename: Ident) -> syn::Result<TokenStrea
let mut body = TokenStream2::new();
for _ in 0..fields.unnamed.len() {
let delta = quote! {
#cratename::serial::Decodable::decode(&mut d)?,
#cratename::Decodable::decode(&mut d)?,
};
body.extend(delta);
}
@@ -312,7 +311,7 @@ pub fn struct_de(input: &ItemStruct, cratename: Ident) -> syn::Result<TokenStrea
};
Ok(quote! {
impl #impl_generics #cratename::serial::Decodable for #name #ty_generics #where_clause {
impl #impl_generics #cratename::Decodable for #name #ty_generics #where_clause {
fn decode<D: std::io::Read>(mut d: D) -> ::core::result::Result<Self, std::io::Error> {
Ok(#return_value)
}

View File

@@ -8,7 +8,7 @@ use darkfi_derive_internal::{enum_de, enum_ser, struct_de, struct_ser};
#[proc_macro_derive(SerialEncodable, attributes(skip_serialize))]
pub fn darkfi_serialize(input: TokenStream) -> TokenStream {
let found_crate = crate_name("darkfi").expect("darkfi is found in Cargo.toml");
let found_crate = crate_name("darkfi-serial").expect("darkfi-serial is found in Cargo.toml");
let found_crate = match found_crate {
FoundCrate::Name(name) => name,
@@ -36,7 +36,7 @@ pub fn darkfi_serialize(input: TokenStream) -> TokenStream {
#[proc_macro_derive(SerialDecodable, attributes(skip_serialize))]
pub fn darkfi_deserialize(input: TokenStream) -> TokenStream {
let found_crate = crate_name("darkfi").expect("darkfi is found in Cargo.toml");
let found_crate = crate_name("darkfi-serial").expect("darkfi-serial is found in Cargo.toml");
let found_crate = match found_crate {
FoundCrate::Name(name) => name,

View File

@@ -1,6 +1,9 @@
use futures::prelude::*;
use std::io::{Error, ErrorKind};
use futures_lite::{
AsyncRead, AsyncReadExt as AsyncReadExtFut, AsyncWrite, AsyncWriteExt as AsyncWriteExtFut,
};
use super::{endian, VarInt};
pub struct AsyncReadExt;

View File

@@ -1,12 +1,13 @@
use std::io::{Cursor, Error, ErrorKind, Read, Write};
#[cfg(feature = "derive")]
pub use darkfi_derive::{SerialDecodable, SerialEncodable};
#[cfg(feature = "async-runtime")]
#[cfg(feature = "async")]
mod async_serial;
mod encoding_types;
mod endian;
mod types;
/// Data which can be encoded in a consensus-consistent way.
pub trait Encodable {

16
src/serial/src/types.rs Normal file
View File

@@ -0,0 +1,16 @@
//! Encodings for external crates
#[cfg(feature = "collections")]
mod collections;
#[cfg(feature = "hash")]
mod hash;
#[cfg(feature = "incrementalmerkletree")]
mod incrementalmerkletree;
#[cfg(feature = "pasta_curves")]
mod pasta;
#[cfg(feature = "url")]
mod url;

View File

@@ -1,11 +1,10 @@
//! Encodings for external crates
//! Serialization of collections
use std::{
collections::{BTreeMap, BTreeSet, HashSet},
collections::{BTreeMap, BTreeSet, HashMap, HashSet},
io::{Error, Read, Write},
};
#[allow(unused_imports)]
use super::{Decodable, Encodable, ReadExt, VarInt, WriteExt};
use crate::{Decodable, Encodable, VarInt};
impl<T: Encodable> Encodable for HashSet<T> {
fn encode<S: Write>(&self, mut s: S) -> Result<usize, Error> {
@@ -78,23 +77,6 @@ impl<T: Decodable + std::cmp::Ord> Decodable for BTreeSet<T> {
}
}
#[cfg(feature = "blake3")]
impl Encodable for blake3::Hash {
fn encode<S: Write>(&self, mut s: S) -> Result<usize, Error> {
s.write_slice(self.as_bytes())?;
Ok(32)
}
}
#[cfg(feature = "blake3")]
impl Decodable for blake3::Hash {
fn decode<D: Read>(mut d: D) -> Result<Self, Error> {
let mut bytes = [0u8; 32];
d.read_slice(&mut bytes)?;
Ok(bytes.into())
}
}
#[cfg(feature = "fxhash")]
impl<T: Encodable, U: Encodable> Encodable for fxhash::FxHashMap<T, U> {
fn encode<S: Write>(&self, mut s: S) -> Result<usize, Error> {
@@ -108,6 +90,31 @@ impl<T: Encodable, U: Encodable> Encodable for fxhash::FxHashMap<T, U> {
}
}
impl<T: Decodable + std::cmp::Eq + std::hash::Hash, U: Decodable> Decodable for HashMap<T, U> {
fn decode<D: Read>(mut d: D) -> Result<Self, Error> {
let len = VarInt::decode(&mut d)?.0;
let mut ret = HashMap::new();
for _ in 0..len {
let key: T = Decodable::decode(&mut d)?;
let entry: U = Decodable::decode(&mut d)?;
ret.insert(key, entry);
}
Ok(ret)
}
}
impl<T: Encodable, U: Encodable> Encodable for HashMap<T, U> {
fn encode<S: Write>(&self, mut s: S) -> Result<usize, Error> {
let mut len = 0;
len += VarInt(self.len() as u64).encode(&mut s)?;
for c in self.iter() {
len += c.0.encode(&mut s)?;
len += c.1.encode(&mut s)?;
}
Ok(len)
}
}
#[cfg(feature = "fxhash")]
impl<T: Decodable + std::cmp::Eq + std::hash::Hash, U: Decodable> Decodable
for fxhash::FxHashMap<T, U>

View File

@@ -0,0 +1,20 @@
use std::io::{Error, Read, Write};
use crate::{Decodable, Encodable, ReadExt, WriteExt};
#[cfg(feature = "blake3")]
impl Encodable for blake3::Hash {
fn encode<S: Write>(&self, mut s: S) -> Result<usize, Error> {
s.write_slice(self.as_bytes())?;
Ok(32)
}
}
#[cfg(feature = "blake3")]
impl Decodable for blake3::Hash {
fn decode<D: Read>(mut d: D) -> Result<Self, Error> {
let mut bytes = [0u8; 32];
d.read_slice(&mut bytes)?;
Ok(bytes.into())
}
}

View File

@@ -0,0 +1,173 @@
use std::io::{Error, ErrorKind, Read, Write};
use incrementalmerkletree::Hashable;
use crate::{Decodable, Encodable};
impl Encodable for incrementalmerkletree::Position {
fn encode<S: Write>(&self, mut s: S) -> Result<usize, Error> {
u64::from(*self).encode(&mut s)
}
}
impl Decodable for incrementalmerkletree::Position {
fn decode<D: Read>(mut d: D) -> Result<Self, Error> {
let dec: u64 = Decodable::decode(&mut d)?;
Ok(Self::try_from(dec).unwrap())
}
}
impl<T: Encodable + Ord> Encodable for incrementalmerkletree::bridgetree::Leaf<T> {
fn encode<S: Write>(&self, mut s: S) -> Result<usize, Error> {
let mut len = 0;
match self {
incrementalmerkletree::bridgetree::Leaf::Left(a) => {
len += false.encode(&mut s)?;
len += a.encode(&mut s)?;
}
incrementalmerkletree::bridgetree::Leaf::Right(a, b) => {
len += true.encode(&mut s)?;
len += a.encode(&mut s)?;
len += b.encode(&mut s)?;
}
}
Ok(len)
}
}
impl<T: Decodable + Ord> Decodable for incrementalmerkletree::bridgetree::Leaf<T> {
fn decode<D: Read>(mut d: D) -> Result<Self, Error> {
let side: bool = Decodable::decode(&mut d)?;
match side {
false => {
let a: T = Decodable::decode(&mut d)?;
Ok(Self::Left(a))
}
true => {
let a: T = Decodable::decode(&mut d)?;
let b: T = Decodable::decode(&mut d)?;
Ok(Self::Right(a, b))
}
}
}
}
impl Encodable for incrementalmerkletree::bridgetree::Checkpoint {
fn encode<S: Write>(&self, mut s: S) -> Result<usize, Error> {
let mut len = 0;
len += self.bridges_len().encode(&mut s)?;
len += self.is_witnessed().encode(&mut s)?;
len += self.witnessed().encode(&mut s)?;
len += self.forgotten().encode(&mut s)?;
Ok(len)
}
}
impl Decodable for incrementalmerkletree::bridgetree::Checkpoint {
fn decode<D: Read>(mut d: D) -> Result<Self, Error> {
let bridges_len = Decodable::decode(&mut d)?;
let is_witnessed = Decodable::decode(&mut d)?;
let witnessed = Decodable::decode(&mut d)?;
let forgotten = Decodable::decode(&mut d)?;
Ok(Self::from_parts(bridges_len, is_witnessed, witnessed, forgotten))
}
}
impl<T: Encodable + Ord + Clone> Encodable
for incrementalmerkletree::bridgetree::NonEmptyFrontier<T>
{
fn encode<S: Write>(&self, mut s: S) -> Result<usize, Error> {
let mut len = 0;
len += self.position().encode(&mut s)?;
len += self.leaf().encode(&mut s)?;
len += self.ommers().to_vec().encode(&mut s)?;
Ok(len)
}
}
impl<T: Decodable + Ord + Clone> Decodable
for incrementalmerkletree::bridgetree::NonEmptyFrontier<T>
{
fn decode<D: Read>(mut d: D) -> Result<Self, Error> {
let position = Decodable::decode(&mut d)?;
let leaf = Decodable::decode(&mut d)?;
let ommers = Decodable::decode(&mut d)?;
match Self::from_parts(position, leaf, ommers) {
Ok(v) => Ok(v),
Err(_) => Err(Error::new(ErrorKind::Other, "FrontierError")),
}
}
}
impl<T: Encodable + Ord + Clone> Encodable for incrementalmerkletree::bridgetree::AuthFragment<T> {
fn encode<S: Write>(&self, mut s: S) -> Result<usize, Error> {
let mut len = 0;
len += self.position().encode(&mut s)?;
len += self.altitudes_observed().encode(&mut s)?;
len += self.values().to_vec().encode(&mut s)?;
Ok(len)
}
}
impl<T: Decodable + Ord + Clone> Decodable for incrementalmerkletree::bridgetree::AuthFragment<T> {
fn decode<D: Read>(mut d: D) -> Result<Self, Error> {
let position = Decodable::decode(&mut d)?;
let altitudes_observed = Decodable::decode(&mut d)?;
let values = Decodable::decode(&mut d)?;
Ok(Self::from_parts(position, altitudes_observed, values))
}
}
impl<T: Encodable + Ord + Clone> Encodable for incrementalmerkletree::bridgetree::MerkleBridge<T> {
fn encode<S: Write>(&self, mut s: S) -> Result<usize, Error> {
let mut len = 0;
len += self.prior_position().encode(&mut s)?;
len += self.auth_fragments().encode(&mut s)?;
len += self.frontier().encode(&mut s)?;
Ok(len)
}
}
impl<T: Decodable + Ord + Clone> Decodable for incrementalmerkletree::bridgetree::MerkleBridge<T> {
fn decode<D: Read>(mut d: D) -> Result<Self, Error> {
let prior_position = Decodable::decode(&mut d)?;
let auth_fragments = Decodable::decode(&mut d)?;
let frontier = Decodable::decode(&mut d)?;
Ok(Self::from_parts(prior_position, auth_fragments, frontier))
}
}
impl<T: Encodable + Ord + Clone, const V: u8> Encodable
for incrementalmerkletree::bridgetree::BridgeTree<T, V>
{
fn encode<S: Write>(&self, mut s: S) -> Result<usize, Error> {
let mut len = 0;
len += self.prior_bridges().to_vec().encode(&mut s)?;
len += self.current_bridge().encode(&mut s)?;
len += self.witnessed_indices().encode(&mut s)?;
len += self.checkpoints().to_vec().encode(&mut s)?;
len += self.max_checkpoints().encode(&mut s)?;
Ok(len)
}
}
impl<T: Decodable + Ord + Clone + Hashable, const V: u8> Decodable
for incrementalmerkletree::bridgetree::BridgeTree<T, V>
{
fn decode<D: Read>(mut d: D) -> Result<Self, Error> {
let prior_bridges = Decodable::decode(&mut d)?;
let current_bridge = Decodable::decode(&mut d)?;
let saved = Decodable::decode(&mut d)?;
let checkpoints = Decodable::decode(&mut d)?;
let max_checkpoints = Decodable::decode(&mut d)?;
match Self::from_parts(prior_bridges, current_bridge, saved, checkpoints, max_checkpoints) {
Ok(v) => Ok(v),
Err(_) => Err(Error::new(ErrorKind::Other, "BridgeTreeError")),
}
}
}

View File

@@ -0,0 +1,89 @@
//! Implementations for pasta curves
use std::io::{Error, ErrorKind, Read, Write};
use pasta_curves::{
group::{ff::PrimeField, GroupEncoding},
Ep, Eq, Fp, Fq,
};
use crate::{Decodable, Encodable, ReadExt, WriteExt};
impl Encodable for Fp {
#[inline]
fn encode<S: Write>(&self, mut s: S) -> Result<usize, Error> {
s.write_slice(&self.to_repr())?;
Ok(32)
}
}
impl Decodable for Fp {
#[inline]
fn decode<D: Read>(mut d: D) -> Result<Self, Error> {
let mut bytes = [0u8; 32];
d.read_slice(&mut bytes)?;
match Self::from_repr(bytes).into() {
Some(v) => Ok(v),
None => Err(Error::new(ErrorKind::Other, "Noncanonical bytes for pallas::Base")),
}
}
}
impl Encodable for Fq {
#[inline]
fn encode<S: Write>(&self, mut s: S) -> Result<usize, Error> {
s.write_slice(&self.to_repr())?;
Ok(32)
}
}
impl Decodable for Fq {
#[inline]
fn decode<D: Read>(mut d: D) -> Result<Self, Error> {
let mut bytes = [0u8; 32];
d.read_slice(&mut bytes)?;
match Self::from_repr(bytes).into() {
Some(v) => Ok(v),
None => Err(Error::new(ErrorKind::Other, "Noncanonical bytes for pallas::Scalar")),
}
}
}
impl Encodable for Ep {
#[inline]
fn encode<S: Write>(&self, mut s: S) -> Result<usize, Error> {
s.write_slice(&self.to_bytes())?;
Ok(32)
}
}
impl Decodable for Ep {
#[inline]
fn decode<D: Read>(mut d: D) -> Result<Self, Error> {
let mut bytes = [0u8; 32];
d.read_slice(&mut bytes)?;
match Self::from_bytes(&bytes).into() {
Some(v) => Ok(v),
None => Err(Error::new(ErrorKind::Other, "Noncanonical bytes for pallas::Point")),
}
}
}
impl Encodable for Eq {
#[inline]
fn encode<S: Write>(&self, mut s: S) -> Result<usize, Error> {
s.write_slice(&self.to_bytes())?;
Ok(32)
}
}
impl Decodable for Eq {
#[inline]
fn decode<D: Read>(mut d: D) -> Result<Self, Error> {
let mut bytes = [0u8; 32];
d.read_slice(&mut bytes)?;
match Self::from_bytes(&bytes).into() {
Some(v) => Ok(v),
None => Err(Error::new(ErrorKind::Other, "Noncanonical bytes for vesta::Point")),
}
}
}

View File

@@ -0,0 +1,23 @@
use std::io::{Error, ErrorKind, Read, Write};
use url::Url;
use crate::{Decodable, Encodable};
impl Encodable for Url {
#[inline]
fn encode<S: Write>(&self, s: S) -> Result<usize, Error> {
self.as_str().to_string().encode(s)
}
}
impl Decodable for Url {
#[inline]
fn decode<D: Read>(mut d: D) -> Result<Self, Error> {
let s: String = Decodable::decode(&mut d)?;
match Url::parse(&s) {
Ok(v) => Ok(v),
Err(e) => Err(Error::new(ErrorKind::Other, e)),
}
}
}