github/extism
1
1
Fork 0
mirror of https://github.com/extism/extism.git synced 2026-01-08 05:23:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat(convert): add derive macros for To and FromBytes (#667)

Browse Source 
closes #661.

- [x] docs
- [x] tests
- [x] depend on `extism-convert/extism-pdk-path` feature in
https://github.com/extism/rust-pdk
https://github.com/extism/rust-pdk/pull/47
This commit is contained in:
Roland Fredenhagen
2024-02-06 00:59:55 +01:00
committed by GitHub
parent efa69d3668
commit 1a083f612a
13 changed files with 396 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
Cargo.toml
View File

@@ -1,6 +1,6 @@
[workspace]
resolver = "2"
members = ["extism-maturin", "manifest", "runtime", "libextism", "convert"]
members = ["extism-maturin", "manifest", "runtime", "libextism", "convert", "convert-macros"]
exclude = ["kernel"]
[workspace.package]
@@ -14,4 +14,5 @@ version = "0.0.0+replaced-by-ci"
[workspace.dependencies]
extism = { path = "./runtime", version = "0.0.0+replaced-by-ci" }
extism-convert = { path = "./convert", version = "0.0.0+replaced-by-ci" }
extism-convert-macros = { path = "./convert-macros", version = "0.0.0+replaced-by-ci" }
extism-manifest = { path = "./manifest", version = "0.0.0+replaced-by-ci" }

27
convert-macros/Cargo.toml Normal file
View File

@@ -0,0 +1,27 @@
[package]
name = "extism-convert-macros"
readme = "./README.md"
edition.workspace = true
authors.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
version.workspace = true
description = "Macros to remove boilerplate with Extism"
[lib]
proc-macro = true
[features]
extism-path = []
extism-pdk-path = []
[dependencies]
manyhow.version = "0.11.0"
proc-macro-crate = "3.1.0"
proc-macro2 = "1.0.78"
quote = "1.0.35"
syn = { version = "2.0.48", features = ["derive"] }
[dev-dependencies]
trybuild = "1.0.89"

108
convert-macros/src/lib.rs Normal file
View File

@@ -0,0 +1,108 @@
use std::iter;
use manyhow::{ensure, error_message, manyhow, Result};
use proc_macro_crate::{crate_name, FoundCrate};
use quote::{format_ident, quote, ToTokens};
use syn::{parse_quote, Attribute, DeriveInput, Path};
/// Tries to resolve the path to `extism_convert` dynamically, falling back to feature flags when unsuccessful.
fn convert_path() -> Path {
match (
crate_name("extism"),
crate_name("extism-convert"),
crate_name("extism-pdk"),
) {
(Ok(FoundCrate::Name(name)), ..) => {
let ident = format_ident!("{name}");
parse_quote!(::#ident::convert)
}
(_, Ok(FoundCrate::Name(name)), ..) | (.., Ok(FoundCrate::Name(name))) => {
let ident = format_ident!("{name}");
parse_quote!(::#ident)
}
(Ok(FoundCrate::Itself), ..) => parse_quote!(::extism::convert),
(_, Ok(FoundCrate::Itself), ..) => parse_quote!(::extism_convert),
(.., Ok(FoundCrate::Itself)) => parse_quote!(::extism_pdk),
_ if cfg!(feature = "extism-path") => parse_quote!(::extism::convert),
_ if cfg!(feature = "extism-pdk-path") => parse_quote!(::extism_pdk),
_ => parse_quote!(::extism_convert),
}
}
fn extract_encoding(attrs: &[Attribute]) -> Result<Path> {
let encodings: Vec<_> = attrs
.iter()
.filter(|attr| attr.path().is_ident("encoding"))
.collect();
ensure!(!encodings.is_empty(), "encoding needs to be specified"; try = "`#[encoding(ToJson)]`");
ensure!(encodings.len() < 2, encodings[1], "only one encoding can be specified"; try = "remove `{}`", encodings[1].to_token_stream());
Ok(encodings[0].parse_args().map_err(
|e| error_message!(e.span(), "{e}"; note= "expects a path"; try = "`#[encoding(ToJson)]`"),
)?)
}
#[manyhow]
#[proc_macro_derive(ToBytes, attributes(encoding))]
pub fn to_bytes(
DeriveInput {
attrs,
ident,
generics,
..
}: DeriveInput,
) -> Result {
let encoding = extract_encoding(&attrs)?;
let convert = convert_path();
let (_, type_generics, _) = generics.split_for_impl();
let mut generics = generics.clone();
generics.make_where_clause().predicates.push(
parse_quote!(for<'__to_bytes_b> #encoding<&'__to_bytes_b Self>: #convert::ToBytes<'__to_bytes_b>)
);
generics.params = iter::once(parse_quote!('__to_bytes_a))
.chain(generics.params)
.collect();
let (impl_generics, _, where_clause) = generics.split_for_impl();
Ok(quote! {
impl #impl_generics #convert::ToBytes<'__to_bytes_a> for #ident #type_generics #where_clause
{
type Bytes = ::std::vec::Vec<u8>;
fn to_bytes(&self) -> Result<Self::Bytes, #convert::Error> {
#convert::ToBytes::to_bytes(&#encoding(self)).map(|__bytes| __bytes.as_ref().to_vec())
}
}
})
}
#[manyhow]
#[proc_macro_derive(FromBytes, attributes(encoding))]
pub fn from_bytes(
DeriveInput {
attrs,
ident,
mut generics,
..
}: DeriveInput,
) -> Result {
let encoding = extract_encoding(&attrs)?;
let convert = convert_path();
generics
.make_where_clause()
.predicates
.push(parse_quote!(#encoding<Self>: #convert::FromBytesOwned));
let (impl_generics, type_generics, where_clause) = generics.split_for_impl();
Ok(quote! {
impl #impl_generics #convert::FromBytesOwned for #ident #type_generics #where_clause
{
fn from_bytes_owned(__data: &[u8]) -> Result<Self, #convert::Error> {
<#encoding<Self> as #convert::FromBytesOwned>::from_bytes_owned(__data).map(|__encoding| __encoding.0)
}
}
})
}

5
convert-macros/tests/ui.rs Normal file
View File

@@ -0,0 +1,5 @@
#[test]
fn ui() {
let t = trybuild::TestCases::new();
t.compile_fail("tests/ui/*.rs");
}

23
convert-macros/tests/ui/invalid-encoding.rs Normal file
View File

@@ -0,0 +1,23 @@
use extism_convert_macros::ToBytes;
#[derive(ToBytes)]
struct MissingEncoding;
#[derive(ToBytes)]
#[encoding]
struct EmptyAttr;
#[derive(ToBytes)]
#[encoding = "string"]
struct EqNoParen;
#[derive(ToBytes)]
#[encoding(something, else)]
struct NotAPath;
#[derive(ToBytes)]
#[encoding(Multiple)]
#[encoding(Encodings)]
struct MultipleEncodings;
fn main() {}

44
convert-macros/tests/ui/invalid-encoding.stderr Normal file
View File

@@ -0,0 +1,44 @@
error: encoding needs to be specified
= try: `#[encoding(ToJson)]`
--> tests/ui/invalid-encoding.rs:3:10
|
3 | #[derive(ToBytes)]
| ^^^^^^^
|
= note: this error originates in the derive macro `ToBytes` (in Nightly builds, run with -Z macro-backtrace for more info)
error: expected attribute arguments in parentheses: #[encoding(...)]
= note: expects a path
= try: `#[encoding(ToJson)]`
--> tests/ui/invalid-encoding.rs:7:3
|
7 | #[encoding]
| ^^^^^^^^
error: expected parentheses: #[encoding(...)]
= note: expects a path
= try: `#[encoding(ToJson)]`
--> tests/ui/invalid-encoding.rs:11:12
|
11 | #[encoding = "string"]
| ^
error: unexpected token
= note: expects a path
= try: `#[encoding(ToJson)]`
--> tests/ui/invalid-encoding.rs:15:21
|
15 | #[encoding(something, else)]
| ^
error: only one encoding can be specified
= try: remove `#[encoding(Encodings)]`
--> tests/ui/invalid-encoding.rs:20:1
|
20 | #[encoding(Encodings)]
| ^^^^^^^^^^^^^^^^^^^^^^

5
convert/Cargo.toml
View File

@@ -18,11 +18,16 @@ protobuf = { version = "3.2.0", optional = true }
rmp-serde = { version = "1.1.2", optional = true }
serde = "1.0.186"
serde_json = "1.0.105"
extism-convert-macros.workspace = true
[dev-dependencies]
# Only required for tests run from the workspace root, as that enables the `extism-path` feature.
extism.workspace = true
serde = { version = "1.0.186", features = ["derive"] }
[features]
default = ["msgpack", "prost", "raw"]
msgpack = ["rmp-serde"]
raw = ["bytemuck"]
extism-path = ["extism-convert-macros/extism-path"]
extism-pdk-path = ["extism-convert-macros/extism-pdk-path"]

17
convert/src/encoding.rs
View File

@@ -8,11 +8,22 @@ use base64::Engine;
/// For example, the following line creates a new JSON encoding using serde_json:
///
/// ```
/// extism_convert::encoding!(MyJson, serde_json::to_vec, serde_json::from_slice);
#[cfg_attr(
feature = "extism-path",
doc = "extism::convert::encoding!(MyJson, serde_json::to_vec, serde_json::from_slice);"
)]
#[cfg_attr(
all(feature = "extism-pdk-path", not(feature = "extism-path")),
doc = "extism_pdk::encoding!(MyJson, serde_json::to_vec, serde_json::from_slice);"
)]
#[cfg_attr(
not(any(feature = "extism-path", feature = "extism-pdk-path")),
doc = "extism_convert::encoding!(MyJson, serde_json::to_vec, serde_json::from_slice);"
)]
/// ```
///
/// This will create a struct `struct MyJson<T>(pub T)` and implement `ToBytes` using `serde_json::to_vec`
/// and `FromBytesOwned` using `serde_json::from_vec`
/// This will create a struct `struct MyJson<T>(pub T)` and implement [`ToBytes`] using [`serde_json::to_vec`]
/// and [`FromBytesOwned`] using [`serde_json::from_slice`]
#[macro_export]
macro_rules! encoding {
($pub:vis $name:ident, $to_vec:expr, $from_slice:expr) => {

80
convert/src/from_bytes.rs
View File

@@ -1,14 +1,90 @@
use crate::*;
pub use extism_convert_macros::FromBytes;
/// `FromBytes` is used to define how a type should be decoded when working with
/// Extism memory. It is used for plugin output and host function input.
///
/// `FromBytes` can be derived by delegating encoding to generic type implementing
/// `FromBytes`, e.g., [`Json`], [`Msgpack`].
///
/// ```
#[cfg_attr(
feature = "extism-path",
doc = "use extism::convert::{Json, FromBytes};"
)]
#[cfg_attr(
all(feature = "extism-pdk-path", not(feature = "extism-path")),
doc = "use extism_pdk::{Json, FromBytes};"
)]
#[cfg_attr(
not(any(feature = "extism-path", feature = "extism-pdk-path")),
doc = "use extism_convert::{Json, FromBytes};"
)]
/// use serde::Deserialize;
///
/// #[derive(FromBytes, Deserialize, PartialEq, Debug)]
/// #[encoding(Json)]
/// struct Struct {
/// hello: String,
/// }
///
/// assert_eq!(Struct::from_bytes(br#"{"hello":"hi"}"#)?, Struct { hello: "hi".into() });
/// # Ok::<(), extism_convert::Error>(())
/// ```
///
/// Custom encodings can also be used, through new-types with a single generic
/// argument, i.e., `Type<T>(T)`, that implement `FromBytesOwned` for the struct.
///
/// ```
/// use std::str::{self, FromStr};
/// use std::convert::Infallible;
#[cfg_attr(
feature = "extism-path",
doc = "use extism::convert::{Error, FromBytes, FromBytesOwned};"
)]
#[cfg_attr(
all(feature = "extism-pdk-path", not(feature = "extism-path")),
doc = "use extism_pdk::{Error, FromBytes, FromBytesOwned};"
)]
#[cfg_attr(
not(any(feature = "extism-path", feature = "extism-pdk-path")),
doc = "use extism_convert::{Error, FromBytes, FromBytesOwned};"
)]
///
/// // Custom serialization using `FromStr`
/// struct StringEnc<T>(T);
/// impl<T: FromStr> FromBytesOwned for StringEnc<T> where Error: From<<T as FromStr>::Err> {
/// fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
/// Ok(Self(str::from_utf8(data)?.parse()?))
/// }
/// }
///
/// #[derive(FromBytes, PartialEq, Debug)]
/// #[encoding(StringEnc)]
/// struct Struct {
/// hello: String,
/// }
///
/// impl FromStr for Struct {
/// type Err = Infallible;
/// fn from_str(s: &str) -> Result<Self, Infallible> {
/// Ok(Self { hello: s.to_owned() })
/// }
/// }
///
/// assert_eq!(Struct::from_bytes(b"hi")?, Struct { hello: "hi".into() });
/// # Ok::<(), extism_convert::Error>(())
/// ```
pub trait FromBytes<'a>: Sized {
/// Decode a value from a slice of bytes
fn from_bytes(data: &'a [u8]) -> Result<Self, Error>;
}
/// `FromBytesOwned` is similar to `FromBytes` but it doesn't borrow from the input slice.
/// `FromBytes` is automatically implemented for all types that implement `FromBytesOwned`
/// `FromBytesOwned` is similar to [`FromBytes`] but it doesn't borrow from the input slice.
/// [`FromBytes`] is automatically implemented for all types that implement `FromBytesOwned`.
///
/// `FromBytesOwned` can be derived through [`#[derive(FromBytes)]`](FromBytes).
pub trait FromBytesOwned: Sized {
/// Decode a value from a slice of bytes, the resulting value should not borrow the input
/// data.

3
convert/src/lib.rs
View File

@@ -5,6 +5,9 @@
//! similar to [axum extractors](https://docs.rs/axum/latest/axum/extract/index.html#intro) - they are
//! implemented as a tuple struct with a single field that is meant to be extracted using pattern matching.
// Makes proc-macros able to resolve `::extism_convert` correctly
extern crate self as extism_convert;
pub use anyhow::Error;
mod encoding;

82
convert/src/to_bytes.rs
View File

@@ -1,7 +1,77 @@
use crate::*;
pub use extism_convert_macros::ToBytes;
/// `ToBytes` is used to define how a type should be encoded when working with
/// Extism memory. It is used for plugin input and host function output.
///
/// `ToBytes` can be derived by delegating encoding to generic type implementing
/// `ToBytes`, e.g., [`Json`], [`Msgpack`].
///
/// ```
#[cfg_attr(feature = "extism-path", doc = "use extism::convert::{Json, ToBytes};")]
#[cfg_attr(
all(feature = "extism-pdk-path", not(feature = "extism-path")),
doc = "use extism_pdk::{Json, ToBytes};"
)]
#[cfg_attr(
not(any(feature = "extism-path", feature = "extism-pdk-path")),
doc = "use extism_convert::{Json, ToBytes};"
)]
/// use serde::Serialize;
///
/// #[derive(ToBytes, Serialize)]
/// #[encoding(Json)]
/// struct Struct {
/// hello: String,
/// }
///
/// assert_eq!(Struct { hello: "hi".into() }.to_bytes()?, br#"{"hello":"hi"}"#);
/// # Ok::<(), extism_convert::Error>(())
/// ```
///
/// But custom types can also be used, as long as they are new-types with a single
/// generic argument, i.e., `Type<T>(T)`, that implement `ToBytes` for the struct.
///
/// ```
#[cfg_attr(
feature = "extism-path",
doc = "use extism::convert::{Error, ToBytes};"
)]
#[cfg_attr(
all(feature = "extism-pdk-path", not(feature = "extism-path")),
doc = "use extism_pdk::{Error, ToBytes};"
)]
#[cfg_attr(
not(any(feature = "extism-path", feature = "extism-pdk-path")),
doc = "use extism_convert::{Error, ToBytes};"
)]
///
/// // Custom serialization using `ToString`
/// struct StringEnc<T>(T);
/// impl<T: ToString> ToBytes<'_> for StringEnc<&T> {
/// type Bytes = String;
///
/// fn to_bytes(&self) -> Result<String, Error> {
/// Ok(self.0.to_string())
/// }
/// }
///
/// #[derive(ToBytes)]
/// #[encoding(StringEnc)]
/// struct Struct {
/// hello: String,
/// }
///
/// impl ToString for Struct {
/// fn to_string(&self) -> String {
/// self.hello.clone()
/// }
/// }
///
/// assert_eq!(Struct { hello: "hi".into() }.to_bytes()?, b"hi");
/// # Ok::<(), Error>(())
/// ```
pub trait ToBytes<'a> {
/// A configurable byte slice representation, allows any type that implements `AsRef<[u8]>`
type Bytes: AsRef<[u8]>;
@@ -111,3 +181,15 @@ impl<'a, T: ToBytes<'a>> ToBytes<'a> for Option<T> {
}
}
}
#[test]
fn test() {
use extism_convert::{Json, ToBytes};
use serde::Serialize;
#[derive(ToBytes, Serialize)]
#[encoding(Json)]
struct Struct {
hello: String,
}
}

2
runtime/Cargo.toml
View File

@@ -22,7 +22,7 @@ url = "2"
glob = "0.3"
ureq = {version = "2.5", optional=true}
extism-manifest = { workspace = true }
extism-convert = { workspace = true }
extism-convert = { workspace = true, features = ["extism-path"] }
uuid = { version = "1", features = ["v4"] }
libc = "0.2"

4
runtime/src/lib.rs
View File

@@ -1,8 +1,12 @@
// Makes proc-macros able to resolve `::extism` correctly
extern crate self as extism;
pub(crate) use extism_convert::*;
pub(crate) use std::collections::BTreeMap;
use std::str::FromStr;
pub(crate) use wasmtime::*;
#[doc(inline)]
pub use extism_convert as convert;
pub use anyhow::Error;