From 9cf54d5f1fbcba13272d9474d48928a1e080adaa Mon Sep 17 00:00:00 2001 From: zach Date: Mon, 5 Dec 2022 15:09:50 -0800 Subject: [PATCH] feat: Improve usability of manifest types (#124) - Adds some helper functions for creating a manifest, mostly helpful for the Rust SDK - Renames `ManifestWasm` to `Wasm` - Renames `ManifestMemory` to `MemoryOptions` - `ManifestWasm` and `ManifestMemory` have been marked as deprecated - Adds an alias from `MemoryOptions::max_pages` to `MemoryOptions::max` in serde specification Co-authored-by: Steve Manuel --- extism.go | 12 +-- haskell/manifest/Extism/Manifest.hs | 8 +- manifest/schema.json | 14 ++-- manifest/src/lib.rs | 120 ++++++++++++++++++++++++---- node/src/index.ts | 2 +- ocaml/lib/extism.ml | 6 +- ocaml/lib/extism.mli | 6 +- python/tests/test_extism.py | 2 +- runtime/src/manifest.rs | 36 ++++----- runtime/src/pdk.rs | 2 +- runtime/src/plugin.rs | 5 +- 11 files changed, 152 insertions(+), 61 deletions(-) diff --git a/extism.go b/extism.go index c097cdd..a71947b 100644 --- a/extism.go +++ b/extism.go @@ -53,11 +53,11 @@ type WasmFile struct { } type WasmUrl struct { - Url string `json:"url"` - Hash string `json:"hash,omitempty"` - Header map[string]string `json:"header,omitempty"` - Name string `json:"name,omitempty"` - Method string `json:"method,omitempty"` + Url string `json:"url"` + Hash string `json:"hash,omitempty"` + Headers map[string]string `json:"headers,omitempty"` + Name string `json:"name,omitempty"` + Method string `json:"method,omitempty"` } type Wasm interface{} @@ -65,7 +65,7 @@ type Wasm interface{} type Manifest struct { Wasm []Wasm `json:"wasm"` Memory struct { - Max uint32 `json:"max,omitempty"` + MaxPages uint32 `json:"max_pages,omitempty"` } `json:"memory,omitempty"` Config map[string]string `json:"config,omitempty"` AllowedHosts []string `json:"allowed_hosts,omitempty"` diff --git a/haskell/manifest/Extism/Manifest.hs b/haskell/manifest/Extism/Manifest.hs index a302914..6f1f831 100644 --- a/haskell/manifest/Extism/Manifest.hs +++ b/haskell/manifest/Extism/Manifest.hs @@ -43,14 +43,14 @@ instance JSONValue Memory where data HTTPRequest = HTTPRequest { url :: String - , header :: Maybe [(String, String)] + , headers :: Maybe [(String, String)] , method :: Maybe String } -requestObj (HTTPRequest url header method) = +requestObj (HTTPRequest url headers method) = [ "url" .= url , - "header" .= header, + "headers" .= headers, "method" .= method ] @@ -118,7 +118,7 @@ wasmFile path = wasmURL :: String -> String -> Wasm wasmURL method url = - let r = HTTPRequest { url = url, header = Nothing, method = Just method } in + let r = HTTPRequest { url = url, headers = Nothing, method = Just method } in URL WasmURL { req = r, urlName = Nothing, urlHash = Nothing } wasmCode :: B.ByteString -> Wasm diff --git a/manifest/schema.json b/manifest/schema.json index 9dc5608..4cc802c 100644 --- a/manifest/schema.json +++ b/manifest/schema.json @@ -22,11 +22,11 @@ }, "memory": { "default": { - "max": null + "max_pages": null }, "allOf": [ { - "$ref": "#/definitions/ManifestMemory" + "$ref": "#/definitions/MemoryOptions" } ] }, @@ -34,15 +34,15 @@ "default": [], "type": "array", "items": { - "$ref": "#/definitions/ManifestWasm" + "$ref": "#/definitions/Wasm" } } }, "definitions": { - "ManifestMemory": { + "MemoryOptions": { "type": "object", "properties": { - "max": { + "max_pages": { "type": [ "integer", "null" @@ -52,7 +52,7 @@ } } }, - "ManifestWasm": { + "Wasm": { "anyOf": [ { "type": "object", @@ -113,7 +113,7 @@ "null" ] }, - "header": { + "headers": { "default": {}, "type": "object", "additionalProperties": { diff --git a/manifest/src/lib.rs b/manifest/src/lib.rs index 08d0ad8..6b9ef72 100644 --- a/manifest/src/lib.rs +++ b/manifest/src/lib.rs @@ -1,9 +1,13 @@ use std::collections::BTreeMap; +#[deprecated] +pub type ManifestMemory = MemoryOptions; + #[derive(Default, serde::Serialize, serde::Deserialize)] #[cfg_attr(feature = "json_schema", derive(schemars::JsonSchema))] -pub struct ManifestMemory { - pub max: Option, +pub struct MemoryOptions { + #[serde(alias = "max")] + pub max_pages: Option, } #[derive(serde::Serialize, serde::Deserialize)] @@ -11,7 +15,8 @@ pub struct ManifestMemory { pub struct HttpRequest { pub url: String, #[serde(default)] - pub header: std::collections::BTreeMap, + #[serde(alias = "header")] + pub headers: std::collections::BTreeMap, pub method: Option, } @@ -19,7 +24,7 @@ impl HttpRequest { pub fn new(url: impl Into) -> HttpRequest { HttpRequest { url: url.into(), - header: Default::default(), + headers: Default::default(), method: None, } } @@ -30,35 +35,112 @@ impl HttpRequest { } pub fn with_header(mut self, key: impl Into, value: impl Into) -> HttpRequest { - self.header.insert(key.into(), value.into()); + self.headers.insert(key.into(), value.into()); self } } +#[derive(Default, serde::Serialize, serde::Deserialize)] +#[cfg_attr(feature = "json_schema", derive(schemars::JsonSchema))] +pub struct WasmMetadata { + pub name: Option, + pub hash: Option, +} + +impl From for Wasm { + fn from(req: HttpRequest) -> Self { + Wasm::Url { + req, + meta: WasmMetadata::default(), + } + } +} + +impl From for Wasm { + fn from(path: std::path::PathBuf) -> Self { + Wasm::File { + path, + meta: WasmMetadata::default(), + } + } +} + +impl From> for Wasm { + fn from(data: Vec) -> Self { + Wasm::Data { + data, + meta: WasmMetadata::default(), + } + } +} + +#[deprecated] +pub type ManifestWasm = Wasm; + #[derive(serde::Serialize, serde::Deserialize)] #[cfg_attr(feature = "json_schema", derive(schemars::JsonSchema))] #[serde(untagged)] -pub enum ManifestWasm { +pub enum Wasm { File { path: std::path::PathBuf, - name: Option, - hash: Option, + + #[serde(flatten)] + meta: WasmMetadata, }, Data { #[serde(with = "base64")] #[cfg_attr(feature = "json_schema", schemars(schema_with = "base64_schema"))] data: Vec, - name: Option, - hash: Option, + #[serde(flatten)] + meta: WasmMetadata, }, Url { #[serde(flatten)] req: HttpRequest, - name: Option, - hash: Option, + #[serde(flatten)] + meta: WasmMetadata, }, } +impl Wasm { + pub fn file(path: impl AsRef) -> Self { + Wasm::File { + path: path.as_ref().to_path_buf(), + meta: Default::default(), + } + } + + pub fn data(data: impl Into>) -> Self { + Wasm::Data { + data: data.into(), + meta: Default::default(), + } + } + + pub fn url(req: HttpRequest) -> Self { + Wasm::Url { + req, + meta: Default::default(), + } + } + + pub fn meta(&self) -> &WasmMetadata { + match self { + Wasm::File { path: _, meta } => &meta, + Wasm::Data { data: _, meta } => &meta, + Wasm::Url { req: _, meta } => &meta, + } + } + + pub fn meta_mut(&mut self) -> &mut WasmMetadata { + match self { + Wasm::File { path: _, meta } => meta, + Wasm::Data { data: _, meta } => meta, + Wasm::Url { req: _, meta } => meta, + } + } +} + #[cfg(feature = "json_schema")] fn base64_schema(gen: &mut schemars::gen::SchemaGenerator) -> schemars::schema::Schema { use schemars::{schema::SchemaObject, JsonSchema}; @@ -71,15 +153,25 @@ fn base64_schema(gen: &mut schemars::gen::SchemaGenerator) -> schemars::schema:: #[cfg_attr(feature = "json_schema", derive(schemars::JsonSchema))] pub struct Manifest { #[serde(default)] - pub wasm: Vec, + pub wasm: Vec, #[serde(default)] - pub memory: ManifestMemory, + pub memory: MemoryOptions, #[serde(default)] pub config: BTreeMap, #[serde(default)] pub allowed_hosts: Option>, } +impl Manifest { + /// Create a new manifest + pub fn new(wasm: impl Into>) -> Manifest { + Manifest { + wasm: wasm.into(), + ..Default::default() + } + } +} + mod base64 { use serde::{Deserialize, Serialize}; use serde::{Deserializer, Serializer}; diff --git a/node/src/index.ts b/node/src/index.ts index c3ea6f5..affe02a 100644 --- a/node/src/index.ts +++ b/node/src/index.ts @@ -145,7 +145,7 @@ export type ManifestWasmData = { * Memory options for the {@link Plugin} */ export type ManifestMemory = { - max?: number; + max_pages?: number; }; /** diff --git a/ocaml/lib/extism.ml b/ocaml/lib/extism.ml index de9384c..63dbc1b 100644 --- a/ocaml/lib/extism.ml +++ b/ocaml/lib/extism.ml @@ -93,7 +93,7 @@ end type error = [ `Msg of string ] module Manifest = struct - type memory = { max : int option [@yojson.option] } [@@deriving yojson] + type memory = { max_pages : int option [@yojson.option] } [@@deriving yojson] type wasm_file = { path : string; @@ -116,7 +116,7 @@ module Manifest = struct type wasm_url = { url : string; - header : (string * string) list option; [@yojson.option] + headers : (string * string) list option; [@yojson.option] name : string option; [@yojson.option] meth : string option; [@yojson.option] [@key "method"] hash : string option; [@yojson.option] @@ -161,7 +161,7 @@ module Manifest = struct let file ?name ?hash path = File { path; name; hash } let data ?name ?hash data = Data { data; name; hash } - let url ?header ?name ?meth ?hash url = Url { header; name; meth; hash; url } + let url ?headers ?name ?meth ?hash url = Url { headers; name; meth; hash; url } let v ?config ?memory ?allowed_hosts wasm = { config; wasm; memory; allowed_hosts } diff --git a/ocaml/lib/extism.mli b/ocaml/lib/extism.mli index 277552d..04d4eca 100644 --- a/ocaml/lib/extism.mli +++ b/ocaml/lib/extism.mli @@ -4,7 +4,7 @@ type error = [ `Msg of string ] val extism_version : unit -> string module Manifest : sig - type memory = { max : int option } [@@deriving yojson] + type memory = { max_pages : int option } [@@deriving yojson] type wasm_file = { path : string; @@ -20,7 +20,7 @@ module Manifest : sig type wasm_url = { url : string; - header : (string * string) list option; [@yojson.option] + headers : (string * string) list option; [@yojson.option] name : string option; [@yojson.option] meth : string option; [@yojson.option] [@key "method"] hash : string option; [@yojson.option] @@ -40,7 +40,7 @@ module Manifest : sig val data : ?name:string -> ?hash:string -> string -> wasm val url : - ?header:(string * string) list -> + ?headers:(string * string) list -> ?name:string -> ?meth:string -> ?hash:string -> diff --git a/python/tests/test_extism.py b/python/tests/test_extism.py index 5cc18da..e8b77d9 100644 --- a/python/tests/test_extism.py +++ b/python/tests/test_extism.py @@ -66,7 +66,7 @@ class TestExtism(unittest.TestCase): def _manifest(self): wasm = self._count_vowels_wasm() hash = hashlib.sha256(wasm).hexdigest() - return {"wasm": [{"data": wasm, "hash": hash}], "memory": {"max": 5}} + return {"wasm": [{"data": wasm, "hash": hash}], "memory": {"max_pages": 5}} def _count_vowels_wasm(self): path = join(dirname(__file__), "code.wasm") diff --git a/runtime/src/manifest.rs b/runtime/src/manifest.rs index 2c0cbbf..2ea7e82 100644 --- a/runtime/src/manifest.rs +++ b/runtime/src/manifest.rs @@ -61,18 +61,15 @@ fn check_hash(hash: &Option, data: &[u8]) -> Result<(), Error> { } /// Convert from manifest to a wasmtime Module -fn to_module( - engine: &Engine, - wasm: &extism_manifest::ManifestWasm, -) -> Result<(String, Module), Error> { +fn to_module(engine: &Engine, wasm: &extism_manifest::Wasm) -> Result<(String, Module), Error> { match wasm { - extism_manifest::ManifestWasm::File { path, name, hash } => { + extism_manifest::Wasm::File { path, meta } => { if cfg!(not(feature = "register-filesystem")) { return Err(anyhow::format_err!("File-based registration is disabled")); } // Figure out a good name for the file - let name = match name { + let name = match &meta.name { None => { let name = path.with_extension(""); name.file_name().unwrap().to_string_lossy().to_string() @@ -85,31 +82,30 @@ fn to_module( let mut file = std::fs::File::open(path)?; file.read_to_end(&mut buf)?; - check_hash(hash, &buf)?; + check_hash(&meta.hash, &buf)?; Ok((name, Module::new(engine, buf)?)) } - extism_manifest::ManifestWasm::Data { name, data, hash } => { - check_hash(hash, data)?; + extism_manifest::Wasm::Data { meta, data } => { + check_hash(&meta.hash, data)?; Ok(( - name.as_deref().unwrap_or("main").to_string(), + meta.name.as_deref().unwrap_or("main").to_string(), Module::new(engine, data)?, )) } #[allow(unused)] - extism_manifest::ManifestWasm::Url { - name, + extism_manifest::Wasm::Url { req: extism_manifest::HttpRequest { url, - header, + headers, method, }, - hash, + meta, } => { // Get the file name let file_name = url.split('/').last().unwrap_or_default(); - let name = match name { + let name = match &meta.name { Some(name) => name.as_str(), None => { let mut name = "main"; @@ -124,9 +120,9 @@ fn to_module( } }; - if let Some(h) = hash { + if let Some(h) = &meta.hash { if let Ok(Some(data)) = cache_get_file(h) { - check_hash(hash, &data)?; + check_hash(&meta.hash, &data)?; let module = Module::new(engine, data)?; return Ok((name.to_string(), module)); } @@ -142,7 +138,7 @@ fn to_module( // Setup request let mut req = ureq::request(method.as_deref().unwrap_or("GET"), url); - for (k, v) in header.iter() { + for (k, v) in headers.iter() { req = req.set(k, v); } @@ -152,11 +148,11 @@ fn to_module( r.read_to_end(&mut data)?; // Try to cache file - if let Some(hash) = hash { + if let Some(hash) = &meta.hash { cache_add_file(hash, &data); } - check_hash(hash, &data)?; + check_hash(&meta.hash, &data)?; // Convert fetched data to module let module = Module::new(engine, data)?; diff --git a/runtime/src/pdk.rs b/runtime/src/pdk.rs index f9dfde3..ad5cf7b 100644 --- a/runtime/src/pdk.rs +++ b/runtime/src/pdk.rs @@ -334,7 +334,7 @@ pub(crate) fn http_request( let mut r = ureq::request(req.method.as_deref().unwrap_or("GET"), &req.url); - for (k, v) in req.header.iter() { + for (k, v) in req.headers.iter() { r = r.set(k, v); } diff --git a/runtime/src/plugin.rs b/runtime/src/plugin.rs index 84392fe..3566d63 100644 --- a/runtime/src/plugin.rs +++ b/runtime/src/plugin.rs @@ -90,7 +90,10 @@ impl Plugin { let engine = Engine::default(); let (manifest, modules) = Manifest::new(&engine, wasm.as_ref())?; let mut store = Store::new(&engine, Internal::new(&manifest, with_wasi)?); - let memory = Memory::new(&mut store, MemoryType::new(4, manifest.as_ref().memory.max))?; + let memory = Memory::new( + &mut store, + MemoryType::new(4, manifest.as_ref().memory.max_pages), + )?; let mut memory = PluginMemory::new(store, memory); let mut linker = Linker::new(&engine);