refactor!: make UserData more generic

- Adds `memory.max_var_bytes` to the manifest to limit the number of
bytes allowed to be stored in Extism vars - if `max_var_bytes` is set to
0 then vars are disabled.
- Adds some builder functions to `MemoryOptions` struct
- Sets the default var store size to 1mb
- Includes a test to make sure `var_set` returns an error when the limit
is reached

.github/workflows/kernel.yml

@@ -39,6 +39,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         if: ${{ env.GIT_EXIT_CODE }} != 0
         with:
+          author: "zshipko <zshipko@users.noreply.github.com>"
           title: "update(kernel): extism-runtime.wasm in ${{ github.event.pull_request.head.ref }}"
           body: "Automated PR to update `runtime/src/extism-runtime.wasm` in PR #${{ github.event.number }}"
           base: "${{ github.event.pull_request.head.ref }}"

.github/workflows/release-rust.yaml

@@ -36,6 +36,18 @@ jobs:
           override: true
           target: ${{ matrix.target }}
 
+      - name: Release Rust convert-macros Crate
+        env:
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_API_TOKEN }}
+        run: |
+          version=$(cargo metadata --format-version=1 | jq -r '.packages[] | select(.name == "extism") | .version')
+
+          if ! &>/dev/null curl -sLIf https://crates.io/api/v1/crates/extism-convert-macros/${version}/download; then
+            cargo publish --manifest-path convert-macros/Cargo.toml --allow-dirty
+          else
+            echo "already published ${version}"
+          fi
+
       - name: Release Rust Convert Crate
         env:
           CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_API_TOKEN }}

.github/workflows/release.yml

@@ -28,48 +28,56 @@ jobs:
             target: 'x86_64-apple-darwin'
             artifact: 'libextism.dylib'
             static-artifact: 'libextism.a'
+            static-dll-artifact: ''
             pc-in: 'extism.pc.in'
             static-pc-in: 'extism-static.pc.in'
           - os: 'macos'
             target: 'aarch64-apple-darwin'
             artifact: 'libextism.dylib'
             static-artifact: 'libextism.a'
+            static-dll-artifact: ''
             pc-in: 'extism.pc.in'
             static-pc-in: 'extism-static.pc.in'
           - os: 'ubuntu'
             target: 'aarch64-unknown-linux-gnu'
             artifact: 'libextism.so'
             static-artifact: 'libextism.a'
+            static-dll-artifact: ''
             pc-in: 'extism.pc.in'
             static-pc-in: 'extism-static.pc.in'
           - os: 'ubuntu'
             target: 'aarch64-unknown-linux-musl'
             artifact: 'libextism.so'
             static-artifact: 'libextism.a'
+            static-dll-artifact: ''
             pc-in: 'extism.pc.in'
             static-pc-in: 'extism-static.pc.in'
           - os: 'ubuntu'
             target: 'x86_64-unknown-linux-gnu'
             artifact: 'libextism.so'
             static-artifact: 'libextism.a'
+            static-dll-artifact: ''
             pc-in: 'extism.pc.in'
             static-pc-in: 'extism-static.pc.in'
           - os: 'ubuntu'
             target: 'x86_64-unknown-linux-musl'
             artifact: ''
             static-artifact: 'libextism.a'
+            static-dll-artifact: ''
             pc-in: ''
             static-pc-in: 'extism-static.pc.in'
           - os: 'windows'
             target: 'x86_64-pc-windows-gnu'
             artifact: 'extism.dll'
             static-artifact: 'libextism.a'
+            static-dll-artifact: 'libextism.dll.a'
             pc-in: 'extism.pc.in'
             static-pc-in: 'extism-static.pc.in'
           - os: 'windows'
             target: 'x86_64-pc-windows-msvc'
             artifact: 'extism.dll'
             static-artifact: 'extism.lib'
+            static-dll-artifact: 'extism.dll.lib'
             pc-in: ''
             static-pc-in: ''
 
@@ -158,7 +166,8 @@ jobs:
           cp LICENSE ${SRC_DIR}
           tar -C ${SRC_DIR} -czvf ${ARCHIVE} extism.h \
             ${{ matrix.artifact }} ${{ matrix.static-artifact }}  \
-            ${{ matrix.pc-in }} ${{ matrix.static-pc-in }}
+            ${{ matrix.pc-in }} ${{ matrix.static-pc-in }} \
+            ${{ matrix.static-dll-artifact }}
           ls -ll ${ARCHIVE}
 
           if &>/dev/null which shasum; then

.gitignore

@@ -1,6 +1,7 @@
 target
 runtime/ast.json
 runtime/target
+Cargo.lock
 .DS_Store
 .vscode
 **/libextism.dylib

Cargo.toml

@@ -1,6 +1,6 @@
 [workspace]
 resolver = "2"
-members = ["extism-maturin", "manifest", "runtime", "libextism", "convert"]
+members = ["extism-maturin", "manifest", "runtime", "libextism", "convert", "convert-macros"]
 exclude = ["kernel"]
 
 [workspace.package]
@@ -14,4 +14,5 @@ version = "0.0.0+replaced-by-ci"
 [workspace.dependencies]
 extism = { path = "./runtime", version = "0.0.0+replaced-by-ci" }
 extism-convert = { path = "./convert", version = "0.0.0+replaced-by-ci" }
+extism-convert-macros = { path = "./convert-macros", version = "0.0.0+replaced-by-ci" }
 extism-manifest = { path = "./manifest", version = "0.0.0+replaced-by-ci" }

README.md

@@ -1,65 +1,111 @@
-# [Extism](https://extism.org)
+<div align="center">
+    <a href="https://extism.org">
+    <picture>
+        <source media="(prefers-color-scheme: dark)" srcset=".github/assets/logo-horizontal-darkmode.png">
+        <img alt="Extism - the WebAssembly framework" width="75%" style="max-width: 600px" src=".github/assets/logo-horizontal.png">
+    </picture>
+    </a>
 
-[![Discord](https://img.shields.io/discord/1011124058408112148?color=%23404eed&label=Community%20Chat&logo=Discord&logoColor=%23404eed)](https://discord.gg/cx3usBCWnc)
+[![Discord](https://img.shields.io/discord/1011124058408112148?color=%23404eed&label=Community%20Chat&logo=Discord&logoColor=%23404eed)](https://extism.org/discord)
 ![GitHub Org's stars](https://img.shields.io/github/stars/extism)
-![GitHub all releases](https://img.shields.io/github/downloads/extism/extism/total)
+![Downloads](https://img.shields.io/crates/d/extism)
 ![GitHub License](https://img.shields.io/github/license/extism/extism)
 ![GitHub release (with filter)](https://img.shields.io/github/v/release/extism/extism)
 
-The universal plug-in system. Run WebAssembly extensions inside your app. Use idiomatic Host SDKs for [Go](https://github.com/extism/go-sdk#readme),
-[Ruby](https://github.com/extism/ruby-sdk#readme),
-[Python](https://github.com/extism/python-sdk#readme),
-[JavaScript](https://github.com/extism/js-sdk#readme),
-[Rust](/runtime/#readme),
-[C](libextism/#readme),
-[C++](https://github.com/extism/cpp-sdk/#readme),
-[OCaml](https://github.com/extism/ocaml-sdk#readme),
-[Haskell](https://github.com/extism/haskell-sdk#readme),
-[PHP](https://github.com/extism/php-sdk#readme),
-[Elixir](https://github.com/extism/elixir-sdk#readme),
-[.NET](https://github.com/extism/dotnet-sdk#readme),
-[Java](https://github.com/extism/java-sdk#readme),
-[Zig](https://github.com/extism/zig-sdk#readme),
-[D](https://github.com/extism/d-sdk#readme),
-&amp; more (others coming soon).
+</div>
 
-Plug-in development kits (PDK) for plug-in authors supported in [Rust](https://github.com/extism/rust-pdk#readme), [AssemblyScript](https://github.com/extism/assemblyscript-pdk#readme), [Go](https://github.com/extism/go-pdk#readme), [C/C++](https://github.com/extism/c-pdk#readme), [Haskell](https://github.com/extism/haskell-pdk#readme), [JavaScript](https://github.com/extism/js-pdk#readme), [C#](https://github.com/extism/dotnet-pdk#readme), [F#](https://github.com/extism/dotnet-pdk#readme) and [Zig](https://github.com/extism/zig-pdk#readme).
+# Overview
 
-<p align="center">
-  <img style="width: 70%;" src="https://user-images.githubusercontent.com/7517515/210286900-39b144fd-1b26-4dd0-b7a9-2b5755bc174d.png" alt="Extism embedded SDK language support"/>
-</p>
+Extism is a lightweight framework for building with WebAssembly (Wasm). It
+supports running Wasm code on servers, the edge, CLIs, IoT, browsers and
+everything in between. Extism is designed to be "universal" in that it supports
+a common interface, no matter where it runs.
 
-Add a flexible, secure, and _bLaZiNg FaSt_ plug-in system to your project. Server, desktop, mobile, web, database -- you name it. Enable users to write and execute safe extensions to your software in **3 easy steps:**
+> **Note:** One of the primary use cases for Extism is **building extensible
+> software & plugins**. You want to be able to execute arbitrary, untrusted code
+> from your users? Extism makes this safe and practical to do.
 
-### 1. Import
+Additionally, Extism adds some extra utilities on top of standard Wasm runtimes.
+For example, we support persistent memory/module-scope variables, secure &
+host-controlled HTTP without WASI, runtime limiters & timers, simpler host
+function linking, and more. Extism users build:
 
-Import an Extism Host SDK into your code as a library dependency.
+- plug-in systems
+- FaaS platforms
+- code generators
+- web applications
+- & much more...
 
-### 2. Integrate 
+# Run WebAssembly In Your App
 
-Identify the place(s) in your code where some arbitrary logic should run (the plug-in!), returning your code some results.
+Pick a SDK to import into your program, and refer to the documentation to get
+started:
 
-### 3. Execute
+| Type        | Language                                                                                       | Source Code                                                             | Package                                                                 |
+| ----------- | ---------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------- | ----------------------------------------------------------------------- |
+| Rust SDK    | <img alt="Rust SDK" src="https://extism.org/img/sdk-languages/rust.svg" width="50px"/>         | https://github.com/extism/extism/tree/main/runtime                      | [Crates.io](https://crates.io/crates/extism)                            |
+| JS SDK      | <img alt="JS SDK" src="https://extism.org/img/sdk-languages/js.svg" width="50px"/>             | https://github.com/extism/js-sdk <br/>(supports Web, Node, Deno & Bun!) | [NPM](https://www.npmjs.com/package/@extism/extism)                     |
+| Elixir SDK  | <img alt="Elixir SDK" src="https://extism.org/img/sdk-languages/elixir.svg" width="50px"/>     | https://github.com/extism/elixir-sdk                                    | [Hex](https://hex.pm/packages/extism)                                   |
+| Go SDK      | <img alt="Go SDK" src="https://extism.org/img/sdk-languages/go.svg" width="50px"/>             | https://github.com/extism/go-sdk                                        | [Go mod](https://pkg.go.dev/github.com/extism/go-sdk)                   |
+| Haskell SDK | <img alt="Haskell SDK" src="https://extism.org/img/sdk-languages/haskell.svg" width="50px"/>   | https://github.com/extism/haskell-sdk                                   | [Hackage](https://hackage.haskell.org/package/extism)                   |
+| Java SDK    | <img alt="Java SDK" src="https://extism.org/img/sdk-languages/java-android.svg" width="50px"/> | https://github.com/extism/java-sdk                                      | [Sonatype](https://central.sonatype.com/artifact/org.extism.sdk/extism) |
+| .NET SDK    | <img alt=".NET SDK" src="https://extism.org/img/sdk-languages/dotnet.svg" width="50px"/>       | https://github.com/extism/dotnet-sdk <br/>(supports C# & F#!)           | [Nuget](https://www.nuget.org/packages/Extism.Sdk)                      |
+| OCaml SDK   | <img alt="OCaml SDK" src="https://extism.org/img/sdk-languages/ocaml.svg" width="50px"/>       | https://github.com/extism/ocaml-sdk                                     | [opam](https://opam.ocaml.org/packages/extism/)                         |
+| PHP SDK     | <img alt="PHP SDK" src="https://extism.org/img/sdk-languages/php.svg" width="50px"/>           | https://github.com/extism/php-sdk                                       | [Packagist](https://packagist.org/packages/extism/extism)               |
+| Python SDK  | <img alt="Python SDK" src="https://extism.org/img/sdk-languages/python.svg" width="50px"/>     | https://github.com/extism/python-sdk                                    | [PyPi](https://pypi.org/project/extism/)                                |
+| Ruby SDK    | <img alt="Ruby SDK" src="https://extism.org/img/sdk-languages/ruby.svg" width="50px"/>         | https://github.com/extism/ruby-sdk                                      | [RubyGems](https://rubygems.org/gems/extism)                            |
+| Zig SDK     | <img alt="Zig SDK" src="https://extism.org/img/sdk-languages/zig.svg" width="50px"/>           | https://github.com/extism/zig-sdk                                       | N/A                                                                     |
+| C SDK       | <img alt="C SDK" src="https://extism.org/img/sdk-languages/c.svg" width="50px"/>               | https://github.com/extism/extism/tree/main/libextism                    | N/A                                                                     |
+| C++ SDK     | <img alt="C++ SDK" src="https://extism.org/img/sdk-languages/cpp.svg" width="50px"/>           | https://github.com/extism/cpp-sdk                                       | N/A                                                                     |
 
-Load WebAssembly modules at any time in your app's lifetime and Extism will execute them in a secure sandbox, fully isolated from your program's memory.
+# Compile WebAssembly to run in Extism Hosts
 
-# API Status
+Extism Hosts (running the SDK) must execute WebAssembly code that has a PDK
+library compiled in to the `.wasm` binary. PDKs make it easy for plug-in /
+extension code authors to read input from the host and return data back, read
+provided configuration, set/get variables, make outbound HTTP calls if allowed,
+and more.
 
-**Please note:** This project still under active development and APIs are still changing. We are aiming for a stable 1.0 release in January, 2024.
-The main branch may have breaking changes until that point, but if you starting today, a 1.0.0-rcx release is the best place to start.
+Pick a PDK to import into your Wasm program, and refer to the documentation to
+get started:
 
-If you experience any problems or have any questions, please join our [Discord](https://discord.gg/cx3usBCWnc) and let us know.
-Our community is very responsive and happy to help get you started.
+| Type               | Language                                                                                                   | Source Code                                                   | Package                                                   |
+| ------------------ | ---------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------- | --------------------------------------------------------- |
+| Rust PDK           | <img alt="Rust PDK" src="https://extism.org/img/sdk-languages/rust.svg" width="50px"/>                     | https://github.com/extism/rust-pdk                            | [Crates.io](https://crates.io/crates/extism-pdk)          |
+| JS PDK             | <img alt="JS PDK" src="https://extism.org/img/sdk-languages/js.svg" width="50px"/>                         | https://github.com/extism/js-pdk                              | N/A                                                       |
+| Go PDK             | <img alt="Go PDK" src="https://extism.org/img/sdk-languages/go.svg" width="50px"/>                         | https://github.com/extism/go-pdk                              | [Go mod](https://pkg.go.dev/github.com/extism/go-pdk)     |
+| Haskell PDK        | <img alt="Haskell PDK" src="https://extism.org/img/sdk-languages/haskell.svg" width="50px"/>               | https://github.com/extism/haskell-pdk                         | [Hackage](https://hackage.haskell.org/package/extism-pdk) |
+| AssemblyScript PDK | <img alt="AssemblyScript PDK" src="https://extism.org/img/sdk-languages/assemblyscript.svg" width="50px"/> | https://github.com/extism/assemblyscript-pdk                  | [NPM](https://www.npmjs.com/package/@extism/as-pdk)       |
+| .NET PDK           | <img alt=".NET PDK" src="https://extism.org/img/sdk-languages/dotnet.svg" width="50px"/>                   | https://github.com/extism/dotnet-pdk <br/>(supports C# & F#!) | https://www.nuget.org/packages/Extism.Pdk                 |
+| C PDK              | <img alt="C PDK" src="https://extism.org/img/sdk-languages/c.svg" width="50px"/>                           | https://github.com/extism/c-pdk                               | N/A                                                       |
+| Zig PDK            | <img alt="Zig PDK" src="https://extism.org/img/sdk-languages/zig.svg" width="50px"/>                       | https://github.com/extism/zig-pdk                             | N/A                                                       |
+
+# Support
+
+## Discord
+
+If you experience any problems or have any questions, please join our
+[Discord](https://extism.org/discord) and let us know. Our community is very
+responsive and happy to help get you started.
 
 ## Usage
 
-Head to the [project website](https://extism.org) for more information and docs. Also, consider reading an [overview](https://extism.org/docs/overview) of Extism and its goals & approach.
+Head to the [project website](https://extism.org) for more information and docs.
+Also, consider reading an [overview](https://extism.org/docs/overview) of Extism
+and its goals & approach.
 
 ## Contribution
 
-Thank you for considering a contribution to Extism, we are happy to help you make a PR or find something to work on! 
+Thank you for considering a contribution to Extism, we are happy to help you
+make a PR or find something to work on!
 
-The easiest way to start would be to join the [Discord](https://discord.gg/cx3usBCWnc) or open an issue on the [`extism/proposals`](https://github.com/extism/proposals) issue tracker, which can eventually become an Extism Improvement Proposal (EIP).
+The easiest way to start would be to join the
+[Discord](https://extism.org/discord) or open an issue on the
+[`extism/proposals`](https://github.com/extism/proposals) issue tracker, which
+can eventually become an Extism Improvement Proposal (EIP).
+
+For more information, please read the
+[Contributing](https://extism.org/docs/concepts/contributing) guide.
 
 ---
 
@@ -68,8 +114,8 @@ The easiest way to start would be to join the [Discord](https://discord.gg/cx3us
 Extism is an open-source product from the team at:
 
 <p align="left">
-  <a href="https://dylib.so" _target="blanks"><img width="200px" src="https://user-images.githubusercontent.com/7517515/198204119-5afdebb9-a5d8-4322-bd2a-46179c8d7b24.svg"/></a>
+  <a href="https://dylibso.com" _target="blanks"><img width="200px" src="https://user-images.githubusercontent.com/7517515/198204119-5afdebb9-a5d8-4322-bd2a-46179c8d7b24.svg"/></a>
 </p>
 
-
-_Reach out and tell us what you're building! We'd love to help._
+_Reach out and tell us what you're building! We'd love to help:_
+<a href="mailto:hello@dylibso.com">hello@dylibso.com</a>

convert-macros/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "extism-convert-macros"
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+homepage.workspace = true
+repository.workspace = true
+version.workspace = true
+description = "Macros to remove boilerplate with Extism"
+
+[lib]
+proc-macro = true
+
+[features]
+extism-path = []
+extism-pdk-path = []
+
+[dependencies]
+manyhow.version = "0.11.0"
+proc-macro-crate = "3.1.0"
+proc-macro2 = "1.0.78"
+quote = "1.0.35"
+syn = { version = "2.0.48", features = ["derive"] }
+
+[dev-dependencies]
+trybuild = "1.0.89"

convert-macros/src/lib.rs

@@ -0,0 +1,108 @@
+use std::iter;
+
+use manyhow::{ensure, error_message, manyhow, Result};
+use proc_macro_crate::{crate_name, FoundCrate};
+use quote::{format_ident, quote, ToTokens};
+use syn::{parse_quote, Attribute, DeriveInput, Path};
+
+/// Tries to resolve the path to `extism_convert` dynamically, falling back to feature flags when unsuccessful.
+fn convert_path() -> Path {
+    match (
+        crate_name("extism"),
+        crate_name("extism-convert"),
+        crate_name("extism-pdk"),
+    ) {
+        (Ok(FoundCrate::Name(name)), ..) => {
+            let ident = format_ident!("{name}");
+            parse_quote!(::#ident::convert)
+        }
+        (_, Ok(FoundCrate::Name(name)), ..) | (.., Ok(FoundCrate::Name(name))) => {
+            let ident = format_ident!("{name}");
+            parse_quote!(::#ident)
+        }
+        (Ok(FoundCrate::Itself), ..) => parse_quote!(::extism::convert),
+        (_, Ok(FoundCrate::Itself), ..) => parse_quote!(::extism_convert),
+        (.., Ok(FoundCrate::Itself)) => parse_quote!(::extism_pdk),
+        _ if cfg!(feature = "extism-path") => parse_quote!(::extism::convert),
+        _ if cfg!(feature = "extism-pdk-path") => parse_quote!(::extism_pdk),
+        _ => parse_quote!(::extism_convert),
+    }
+}
+
+fn extract_encoding(attrs: &[Attribute]) -> Result<Path> {
+    let encodings: Vec<_> = attrs
+        .iter()
+        .filter(|attr| attr.path().is_ident("encoding"))
+        .collect();
+    ensure!(!encodings.is_empty(), "encoding needs to be specified"; try = "`#[encoding(ToJson)]`");
+    ensure!(encodings.len() < 2, encodings[1], "only one encoding can be specified"; try = "remove `{}`", encodings[1].to_token_stream());
+
+    Ok(encodings[0].parse_args().map_err(
+        |e| error_message!(e.span(), "{e}"; note= "expects a path"; try = "`#[encoding(ToJson)]`"),
+    )?)
+}
+
+#[manyhow]
+#[proc_macro_derive(ToBytes, attributes(encoding))]
+pub fn to_bytes(
+    DeriveInput {
+        attrs,
+        ident,
+        generics,
+        ..
+    }: DeriveInput,
+) -> Result {
+    let encoding = extract_encoding(&attrs)?;
+    let convert = convert_path();
+
+    let (_, type_generics, _) = generics.split_for_impl();
+
+    let mut generics = generics.clone();
+    generics.make_where_clause().predicates.push(
+        parse_quote!(for<'__to_bytes_b> #encoding<&'__to_bytes_b Self>: #convert::ToBytes<'__to_bytes_b>)
+    );
+    generics.params = iter::once(parse_quote!('__to_bytes_a))
+        .chain(generics.params)
+        .collect();
+    let (impl_generics, _, where_clause) = generics.split_for_impl();
+
+    Ok(quote! {
+        impl #impl_generics #convert::ToBytes<'__to_bytes_a> for #ident #type_generics #where_clause
+        {
+            type Bytes = ::std::vec::Vec<u8>;
+
+            fn to_bytes(&self) -> Result<Self::Bytes, #convert::Error> {
+                #convert::ToBytes::to_bytes(&#encoding(self)).map(|__bytes| __bytes.as_ref().to_vec())
+            }
+        }
+
+    })
+}
+
+#[manyhow]
+#[proc_macro_derive(FromBytes, attributes(encoding))]
+pub fn from_bytes(
+    DeriveInput {
+        attrs,
+        ident,
+        mut generics,
+        ..
+    }: DeriveInput,
+) -> Result {
+    let encoding = extract_encoding(&attrs)?;
+    let convert = convert_path();
+    generics
+        .make_where_clause()
+        .predicates
+        .push(parse_quote!(#encoding<Self>: #convert::FromBytesOwned));
+    let (impl_generics, type_generics, where_clause) = generics.split_for_impl();
+    Ok(quote! {
+        impl #impl_generics #convert::FromBytesOwned for #ident #type_generics #where_clause
+        {
+            fn from_bytes_owned(__data: &[u8]) -> Result<Self, #convert::Error> {
+                <#encoding<Self> as #convert::FromBytesOwned>::from_bytes_owned(__data).map(|__encoding| __encoding.0)
+            }
+        }
+
+    })
+}

convert-macros/tests/ui.rs

@@ -0,0 +1,5 @@
+#[test]
+fn ui() {
+    let t = trybuild::TestCases::new();
+    t.compile_fail("tests/ui/*.rs");
+}

convert-macros/tests/ui/invalid-encoding.rs

@@ -0,0 +1,23 @@
+use extism_convert_macros::ToBytes;
+
+#[derive(ToBytes)]
+struct MissingEncoding;
+
+#[derive(ToBytes)]
+#[encoding]
+struct EmptyAttr;
+
+#[derive(ToBytes)]
+#[encoding = "string"]
+struct EqNoParen;
+
+#[derive(ToBytes)]
+#[encoding(something, else)]
+struct NotAPath;
+
+#[derive(ToBytes)]
+#[encoding(Multiple)]
+#[encoding(Encodings)]
+struct MultipleEncodings;
+
+fn main() {}

convert-macros/tests/ui/invalid-encoding.stderr

@@ -0,0 +1,44 @@
+error: encoding needs to be specified
+
+         = try: `#[encoding(ToJson)]`
+ --> tests/ui/invalid-encoding.rs:3:10
+  |
+3 | #[derive(ToBytes)]
+  |          ^^^^^^^
+  |
+  = note: this error originates in the derive macro `ToBytes` (in Nightly builds, run with -Z macro-backtrace for more info)
+
+error: expected attribute arguments in parentheses: #[encoding(...)]
+
+         = note: expects a path
+         = try: `#[encoding(ToJson)]`
+ --> tests/ui/invalid-encoding.rs:7:3
+  |
+7 | #[encoding]
+  |   ^^^^^^^^
+
+error: expected parentheses: #[encoding(...)]
+
+         = note: expects a path
+         = try: `#[encoding(ToJson)]`
+  --> tests/ui/invalid-encoding.rs:11:12
+   |
+11 | #[encoding = "string"]
+   |            ^
+
+error: unexpected token
+
+         = note: expects a path
+         = try: `#[encoding(ToJson)]`
+  --> tests/ui/invalid-encoding.rs:15:21
+   |
+15 | #[encoding(something, else)]
+   |                     ^
+
+error: only one encoding can be specified
+
+         = try: remove `#[encoding(Encodings)]`
+  --> tests/ui/invalid-encoding.rs:20:1
+   |
+20 | #[encoding(Encodings)]
+   | ^^^^^^^^^^^^^^^^^^^^^^

convert/Cargo.toml

@@ -11,16 +11,21 @@ description = "Traits to make Rust types usable with Extism"
 
 [dependencies]
 anyhow = "1.0.75"
-base64 = "~0.21"
+base64 = "~0.22"
+bytemuck = {version = "1.14.0", optional = true }
 prost = { version = "0.12.0", optional = true }
+protobuf = { version = "3.2.0", optional = true }
 rmp-serde = { version = "1.1.2", optional = true }
 serde = "1.0.186"
 serde_json = "1.0.105"
+extism-convert-macros.workspace = true
 
 [dev-dependencies]
 serde = { version = "1.0.186", features = ["derive"] }
 
 [features]
-default = ["msgpack", "protobuf"]
+default = ["msgpack", "prost", "raw"]
 msgpack = ["rmp-serde"]
-protobuf = ["prost"]
+raw = ["bytemuck"]
+extism-path = ["extism-convert-macros/extism-path"]
+extism-pdk-path = ["extism-convert-macros/extism-pdk-path"]

convert/src/encoding.rs

@@ -11,8 +11,8 @@ use base64::Engine;
 /// extism_convert::encoding!(MyJson, serde_json::to_vec, serde_json::from_slice);
 /// ```
 ///
-/// This will create a struct `struct MyJson<T>(pub T)` and implement `ToBytes` using `serde_json::to_vec`
-/// and `FromBytesOwned` using `serde_json::from_vec`
+/// This will create a struct `struct MyJson<T>(pub T)` and implement [`ToBytes`] using [`serde_json::to_vec`]
+/// and [`FromBytesOwned`] using [`serde_json::from_slice`]
 #[macro_export]
 macro_rules! encoding {
     ($pub:vis $name:ident, $to_vec:expr, $from_slice:expr) => {
@@ -112,19 +112,19 @@ impl FromBytesOwned for Base64<String> {
 /// Protobuf encoding
 ///
 /// Allows for `prost` Protobuf messages to be used as arguments to Extism plugin calls
-#[cfg(feature = "protobuf")]
+#[cfg(feature = "prost")]
 #[derive(Debug)]
-pub struct Protobuf<T: prost::Message>(pub T);
+pub struct Prost<T: prost::Message>(pub T);
 
-#[cfg(feature = "protobuf")]
-impl<T: prost::Message> From<T> for Protobuf<T> {
+#[cfg(feature = "prost")]
+impl<T: prost::Message> From<T> for Prost<T> {
     fn from(data: T) -> Self {
         Self(data)
     }
 }
 
-#[cfg(feature = "protobuf")]
-impl<'a, T: prost::Message> ToBytes<'a> for Protobuf<T> {
+#[cfg(feature = "prost")]
+impl<'a, T: prost::Message> ToBytes<'a> for Prost<T> {
     type Bytes = Vec<u8>;
 
     fn to_bytes(&self) -> Result<Self::Bytes, Error> {
@@ -132,9 +132,56 @@ impl<'a, T: prost::Message> ToBytes<'a> for Protobuf<T> {
     }
 }
 
-#[cfg(feature = "protobuf")]
-impl<T: Default + prost::Message> FromBytesOwned for Protobuf<T> {
+#[cfg(feature = "prost")]
+impl<T: Default + prost::Message> FromBytesOwned for Prost<T> {
     fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
-        Ok(Protobuf(T::decode(data)?))
+        Ok(Prost(T::decode(data)?))
     }
 }
+
+/// Protobuf encoding
+///
+/// Allows for `rust-protobuf` Protobuf messages to be used as arguments to Extism plugin calls
+#[cfg(feature = "protobuf")]
+pub struct Protobuf<T: protobuf::Message>(pub T);
+
+#[cfg(feature = "protobuf")]
+impl<'a, T: protobuf::Message> ToBytes<'a> for Protobuf<T> {
+    type Bytes = Vec<u8>;
+
+    fn to_bytes(&self) -> Result<Self::Bytes, Error> {
+        Ok(self.0.write_to_bytes()?)
+    }
+}
+
+#[cfg(feature = "protobuf")]
+impl<T: Default + protobuf::Message> FromBytesOwned for Protobuf<T> {
+    fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
+        Ok(Protobuf(T::parse_from_bytes(data)?))
+    }
+}
+
+/// Raw does no conversion, it just copies the memory directly.
+/// Note: This will only work for types that implement [bytemuck::Pod](https://docs.rs/bytemuck/latest/bytemuck/trait.Pod.html)
+#[cfg(all(feature = "raw", target_endian = "little"))]
+pub struct Raw<'a, T: bytemuck::Pod>(pub &'a T);
+
+#[cfg(all(feature = "raw", target_endian = "little"))]
+impl<'a, T: bytemuck::Pod> ToBytes<'a> for Raw<'a, T> {
+    type Bytes = &'a [u8];
+
+    fn to_bytes(&self) -> Result<Self::Bytes, Error> {
+        Ok(bytemuck::bytes_of(self.0))
+    }
+}
+
+#[cfg(all(feature = "raw", target_endian = "little"))]
+impl<'a, T: bytemuck::Pod> FromBytes<'a> for Raw<'a, T> {
+    fn from_bytes(data: &'a [u8]) -> Result<Self, Error> {
+        let x = bytemuck::try_from_bytes(data).map_err(|x| Error::msg(x.to_string()))?;
+        Ok(Raw(x))
+    }
+}
+
+#[cfg(all(feature = "raw", target_endian = "big"))]
+compile_error!("The raw feature is only supported on little endian targets");

convert/src/from_bytes.rs

@@ -1,14 +1,68 @@
 use crate::*;
 
+pub use extism_convert_macros::FromBytes;
+
 /// `FromBytes` is used to define how a type should be decoded when working with
 /// Extism memory. It is used for plugin output and host function input.
+///
+/// `FromBytes` can be derived by delegating encoding to generic type implementing
+/// `FromBytes`, e.g., [`Json`], [`Msgpack`].
+///
+/// ```
+/// use extism_convert::{Json, FromBytes};
+/// use serde::Deserialize;
+///
+/// #[derive(FromBytes, Deserialize, PartialEq, Debug)]
+/// #[encoding(Json)]
+/// struct Struct {
+///     hello: String,
+/// }
+///
+/// assert_eq!(Struct::from_bytes(br#"{"hello":"hi"}"#)?, Struct { hello: "hi".into() });
+/// # Ok::<(), extism_convert::Error>(())
+/// ```
+///
+/// Custom encodings can also be used, through new-types with a single generic
+/// argument, i.e., `Type<T>(T)`, that implement `FromBytesOwned` for the struct.
+///
+/// ```
+/// use std::str::{self, FromStr};
+/// use std::convert::Infallible;
+/// use extism_convert::{Error, FromBytes, FromBytesOwned};
+///
+/// // Custom serialization using `FromStr`
+/// struct StringEnc<T>(T);
+/// impl<T: FromStr> FromBytesOwned for StringEnc<T> where Error: From<<T as FromStr>::Err> {
+///     fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
+///         Ok(Self(str::from_utf8(data)?.parse()?))
+///     }
+/// }
+///
+/// #[derive(FromBytes, PartialEq, Debug)]
+/// #[encoding(StringEnc)]
+/// struct Struct {
+///     hello: String,
+/// }
+///
+/// impl FromStr for Struct {
+///     type Err = Infallible;
+///     fn from_str(s: &str) -> Result<Self, Infallible> {
+///         Ok(Self { hello: s.to_owned() })
+///     }
+/// }
+///
+/// assert_eq!(Struct::from_bytes(b"hi")?, Struct { hello: "hi".into() });
+/// # Ok::<(), extism_convert::Error>(())
+/// ```
 pub trait FromBytes<'a>: Sized {
     /// Decode a value from a slice of bytes
     fn from_bytes(data: &'a [u8]) -> Result<Self, Error>;
 }
 
-/// `FromBytesOwned` is similar to `FromBytes` but it doesn't borrow from the input slice.
-/// `FromBytes` is automatically implemented for all types that implement `FromBytesOwned`
+/// `FromBytesOwned` is similar to [`FromBytes`] but it doesn't borrow from the input slice.
+/// [`FromBytes`] is automatically implemented for all types that implement `FromBytesOwned`.
+///
+/// `FromBytesOwned` can be derived through [`#[derive(FromBytes)]`](FromBytes).
 pub trait FromBytesOwned: Sized {
     /// Decode a value from a slice of bytes, the resulting value should not borrow the input
     /// data.
@@ -98,3 +152,13 @@ impl<'a, T: FromBytes<'a>> FromBytes<'a> for std::io::Cursor<T> {
         Ok(std::io::Cursor::new(T::from_bytes(data)?))
     }
 }
+
+impl<'a, T: FromBytes<'a>> FromBytes<'a> for Option<T> {
+    fn from_bytes(data: &'a [u8]) -> Result<Self, Error> {
+        if data.is_empty() {
+            return Ok(None);
+        }
+
+        T::from_bytes(data).map(Some)
+    }
+}

convert/src/lib.rs

@@ -5,6 +5,9 @@
 //! similar to [axum extractors](https://docs.rs/axum/latest/axum/extract/index.html#intro) - they are
 //! implemented as a tuple struct with a single field that is meant to be extracted using pattern matching.
 
+// Makes proc-macros able to resolve `::extism_convert` correctly
+extern crate self as extism_convert;
+
 pub use anyhow::Error;
 
 mod encoding;
@@ -18,9 +21,15 @@ pub use encoding::{Base64, Json};
 #[cfg(feature = "msgpack")]
 pub use encoding::Msgpack;
 
+#[cfg(feature = "prost")]
+pub use encoding::Prost;
+
 #[cfg(feature = "protobuf")]
 pub use encoding::Protobuf;
 
+#[cfg(all(feature = "raw", target_endian = "little"))]
+pub use encoding::Raw;
+
 pub use from_bytes::{FromBytes, FromBytesOwned};
 pub use memory_handle::MemoryHandle;
 pub use to_bytes::ToBytes;

convert/src/tests.rs

@@ -20,6 +20,7 @@ fn roundtrip_json() {
 }
 
 #[test]
+#[cfg(feature = "msgpack")]
 fn roundtrip_msgpack() {
     let x = Testing {
         a: "foobar".to_string(),
@@ -37,3 +38,53 @@ fn roundtrip_base64() {
     let Base64(s): Base64<String> = FromBytes::from_bytes(bytes.as_bytes()).unwrap();
     assert_eq!(s, "this is a test");
 }
+
+#[test]
+fn rountrip_option() {
+    // `None` case
+    let e0: Option<Json<Testing>> = FromBytes::from_bytes(&[]).unwrap();
+    let b = e0.to_bytes().unwrap();
+    let e1: Option<Json<Testing>> = FromBytes::from_bytes(&b).unwrap();
+    assert!(e0.is_none());
+    assert_eq!(e0.is_none(), e1.is_none());
+
+    // `Some` case
+    let x = Testing {
+        a: "foobar".to_string(),
+        b: 123,
+        c: 456.7,
+    };
+    let bytes = Json(&x).to_bytes().unwrap();
+    let y: Option<Json<Testing>> = FromBytes::from_bytes(&bytes).unwrap();
+    let b = ToBytes::to_bytes(&y).unwrap();
+    let z: Option<Json<Testing>> = FromBytes::from_bytes(&b).unwrap();
+    assert_eq!(y.unwrap().0, z.unwrap().0);
+}
+
+#[cfg(all(feature = "raw", target_endian = "little"))]
+mod tests {
+    use crate::*;
+
+    #[test]
+    fn test_raw() {
+        #[derive(Debug, Clone, Copy, PartialEq)]
+        struct TestRaw {
+            a: i32,
+            b: f64,
+            c: bool,
+        }
+        unsafe impl bytemuck::Pod for TestRaw {}
+        unsafe impl bytemuck::Zeroable for TestRaw {}
+        let x = TestRaw {
+            a: 123,
+            b: 45678.91011,
+            c: true,
+        };
+        let raw = Raw(&x).to_bytes().unwrap();
+        let y = Raw::from_bytes(&raw).unwrap();
+        assert_eq!(&x, y.0);
+
+        let y: Result<Raw<[u8; std::mem::size_of::<TestRaw>()]>, Error> = Raw::from_bytes(&raw);
+        assert!(y.is_ok());
+    }
+}

convert/src/to_bytes.rs

@@ -1,7 +1,58 @@
 use crate::*;
 
+pub use extism_convert_macros::ToBytes;
+
 /// `ToBytes` is used to define how a type should be encoded when working with
 /// Extism memory. It is used for plugin input and host function output.
+///
+/// `ToBytes` can be derived by delegating encoding to generic type implementing
+/// `ToBytes`, e.g., [`Json`], [`Msgpack`].
+///
+/// ```
+/// use extism_convert::{Json, ToBytes};
+/// use serde::Serialize;
+///
+/// #[derive(ToBytes, Serialize)]
+/// #[encoding(Json)]
+/// struct Struct {
+///     hello: String,
+/// }
+///
+/// assert_eq!(Struct { hello: "hi".into() }.to_bytes()?, br#"{"hello":"hi"}"#);
+/// # Ok::<(), extism_convert::Error>(())
+/// ```
+///
+/// But custom types can also be used, as long as they are new-types with a single
+/// generic argument, i.e., `Type<T>(T)`, that implement `ToBytes` for the struct.
+///
+/// ```
+/// use extism_convert::{Error, ToBytes};
+///
+/// // Custom serialization using `ToString`
+/// struct StringEnc<T>(T);
+/// impl<T: ToString> ToBytes<'_> for StringEnc<&T> {
+///     type Bytes = String;
+///     
+///     fn to_bytes(&self) -> Result<String, Error> {
+///         Ok(self.0.to_string())
+///     }
+/// }
+///
+/// #[derive(ToBytes)]
+/// #[encoding(StringEnc)]
+/// struct Struct {
+///     hello: String,
+/// }
+///
+/// impl ToString for Struct {
+///    fn to_string(&self) -> String {
+///        self.hello.clone()     
+///    }
+/// }
+///
+/// assert_eq!(Struct { hello: "hi".into() }.to_bytes()?, b"hi");
+/// # Ok::<(), Error>(())
+/// ```
 pub trait ToBytes<'a> {
     /// A configurable byte slice representation, allows any type that implements `AsRef<[u8]>`
     type Bytes: AsRef<[u8]>;
@@ -100,3 +151,26 @@ impl<'a, T: ToBytes<'a>> ToBytes<'a> for &'a T {
         <T as ToBytes>::to_bytes(self)
     }
 }
+
+impl<'a, T: ToBytes<'a>> ToBytes<'a> for Option<T> {
+    type Bytes = Vec<u8>;
+
+    fn to_bytes(&self) -> Result<Self::Bytes, Error> {
+        match self {
+            Some(x) => x.to_bytes().map(|x| x.as_ref().to_vec()),
+            None => Ok(vec![]),
+        }
+    }
+}
+
+#[test]
+fn test() {
+    use extism_convert::{Json, ToBytes};
+    use serde::Serialize;
+
+    #[derive(ToBytes, Serialize)]
+    #[encoding(Json)]
+    struct Struct {
+        hello: String,
+    }
+}

flake.lock

@@ -1,94 +0,0 @@
-{
-  "nodes": {
-    "flake-utils": {
-      "inputs": {
-        "systems": "systems"
-      },
-      "locked": {
-        "lastModified": 1701680307,
-        "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "4022d587cbbfd70fe950c1e2083a02621806a725",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
-    "naersk": {
-      "inputs": {
-        "nixpkgs": "nixpkgs"
-      },
-      "locked": {
-        "lastModified": 1698420672,
-        "narHash": "sha256-/TdeHMPRjjdJub7p7+w55vyABrsJlt5QkznPYy55vKA=",
-        "owner": "nix-community",
-        "repo": "naersk",
-        "rev": "aeb58d5e8faead8980a807c840232697982d47b9",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "naersk",
-        "type": "github"
-      }
-    },
-    "nixpkgs": {
-      "locked": {
-        "lastModified": 1701693815,
-        "narHash": "sha256-7BkrXykVWfkn6+c1EhFA3ko4MLi3gVG0p9G96PNnKTM=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "09ec6a0881e1a36c29d67497693a67a16f4da573",
-        "type": "github"
-      },
-      "original": {
-        "id": "nixpkgs",
-        "type": "indirect"
-      }
-    },
-    "nixpkgs_2": {
-      "locked": {
-        "lastModified": 1701693815,
-        "narHash": "sha256-7BkrXykVWfkn6+c1EhFA3ko4MLi3gVG0p9G96PNnKTM=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "09ec6a0881e1a36c29d67497693a67a16f4da573",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixpkgs-unstable",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "root": {
-      "inputs": {
-        "flake-utils": "flake-utils",
-        "naersk": "naersk",
-        "nixpkgs": "nixpkgs_2"
-      }
-    },
-    "systems": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
-      }
-    }
-  },
-  "root": "root",
-  "version": 7
-}

flake.nix

@@ -1,32 +0,0 @@
-{  
-  inputs = {
-    flake-utils.url = "github:numtide/flake-utils";
-    naersk.url = "github:nix-community/naersk";
-    nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
-  };
-
-  outputs = { self, flake-utils, naersk, nixpkgs }:
-    flake-utils.lib.eachDefaultSystem (system:
-      let
-        pkgs = (import nixpkgs) {
-          inherit system;
-        };
-
-        naersk' = pkgs.callPackage naersk {};
-
-      in {
-        # For `nix build` & `nix run`:
-        defaultPackage = naersk'.buildPackage {
-          name = "libextism";
-          src = ./.;
-          copyLibs = true;
-          postInstall = "mkdir -p $out/include; cp runtime/extism.h $out/include";
-        };
-
-        # For `nix develop`:
-        devShell = pkgs.mkShell {
-          nativeBuildInputs = with pkgs; [ rustc cargo clippy gnumake ];
-        };
-      }
-    );
-}

kernel/Cargo.lock

@@ -1,7 +0,0 @@
-# This file is automatically @generated by Cargo.
-# It is not intended for manual editing.
-version = 3
-
-[[package]]
-name = "extism-runtime-kernel"
-version = "0.1.0"

kernel/Cargo.toml

@@ -5,6 +5,9 @@ edition = "2021"
 
 [dependencies]
 
+[dev-dependencies]
+wasm-bindgen-test = "0.3.39"
+
 [features]
 default = ["bounds-checking"]
 bounds-checking = []

kernel/src/bin/extism-runtime.rs

@@ -3,7 +3,7 @@
 
 pub use extism_runtime_kernel::*;
 
-#[cfg(target_arch = "wasm32")]
+#[cfg(all(target_arch = "wasm32", not(test)))]
 #[panic_handler]
 fn panic(_info: &core::panic::PanicInfo) -> ! {
     core::arch::wasm32::unreachable()

kernel/src/lib.rs

@@ -39,7 +39,7 @@
 use core::sync::atomic::*;
 
 pub type Pointer = u64;
-pub type Length = u64;
+pub type Handle = u64;
 
 /// WebAssembly page size
 const PAGE_SIZE: usize = 65536;
@@ -81,13 +81,13 @@ pub struct MemoryRoot {
     /// Offset of error block
     pub error: AtomicU64,
     /// Input position in memory
-    pub input_offset: Pointer,
+    pub input_offset: Handle,
     /// Input length
-    pub input_length: Length,
+    pub input_length: u64,
     /// Output position in memory
     pub output_offset: Pointer,
     /// Output length
-    pub output_length: Length,
+    pub output_length: u64,
     /// A pointer to the start of the first block
     pub blocks: [MemoryBlock; 0],
 }
@@ -139,7 +139,9 @@ impl MemoryRoot {
         }
 
         // Ensure that at least one page is allocated to store the `MemoryRoot` data
-        if core::arch::wasm32::memory_size(0) == 0 && core::arch::wasm32::memory_grow(0, 1) == usize::MAX {
+        if core::arch::wasm32::memory_size(0) == 0
+            && core::arch::wasm32::memory_grow(0, 1) == usize::MAX
+        {
             core::arch::wasm32::unreachable()
         }
 
@@ -168,12 +170,15 @@ impl MemoryRoot {
 
     /// Resets the position of the allocator and zeroes out all allocations
     pub unsafe fn reset(&mut self) {
+        // Clear allocated data
+        let self_position = self.position.fetch_and(0, Ordering::SeqCst);
         core::ptr::write_bytes(
             self.blocks.as_mut_ptr() as *mut u8,
-            0,
-            self.length.load(Ordering::Acquire) as usize,
+            MemoryStatus::Unused as u8,
+            self_position as usize,
         );
-        self.position.store(0, Ordering::Release);
+
+        // Clear extism runtime metadata
         self.error.store(0, Ordering::Release);
         self.input_offset = 0;
         self.input_length = 0;
@@ -193,15 +198,15 @@ impl MemoryRoot {
     fn pointer_in_bounds_fast(p: Pointer) -> bool {
         // Similar to `pointer_in_bounds` but less accurate on the upper bound. This uses the total memory size,
         // instead of checking `MemoryRoot::length`
-        let end = core::arch::wasm32::memory_size(0) << 16;
-        p >= core::mem::size_of::<Self>() as Pointer && p <= end as Pointer
+        let end = (core::arch::wasm32::memory_size(0) as u64) << 16;
+        p >= core::mem::size_of::<Self>() as Pointer && p <= end as u64
     }
 
     // Find a block that is free to use, this can be a new block or an existing freed block. The `self_position` argument
     // is used to avoid loading the allocators position more than once when performing an allocation.
     unsafe fn find_free_block(
         &mut self,
-        length: Length,
+        length: u64,
         self_position: u64,
     ) -> Option<&'static mut MemoryBlock> {
         // Get the first block
@@ -223,7 +228,7 @@ impl MemoryRoot {
             if status == MemoryStatus::Free as u8 && b.size >= length as usize {
                 // Split block if there is too much excess
                 if b.size - length as usize >= 128 {
-                    b.size -= length as usize;
+                    b.size -= length as usize + core::mem::size_of::<MemoryBlock>();
                     b.used = 0;
 
                     let block1 = b.data.as_mut_ptr().add(b.size) as *mut MemoryBlock;
@@ -247,7 +252,7 @@ impl MemoryRoot {
 
     /// Create a new `MemoryBlock`, when `Some(block)` is returned, `block` will contain at least enough room for `length` bytes
     /// but may be as large as `length` + `BLOCK_SPLIT_SIZE` bytes. When `None` is returned the allocation has failed.
-    pub unsafe fn alloc(&mut self, length: Length) -> Option<&'static mut MemoryBlock> {
+    pub unsafe fn alloc(&mut self, length: u64) -> Option<&'static mut MemoryBlock> {
         let self_position = self.position.load(Ordering::Acquire);
         let self_length = self.length.load(Ordering::Acquire);
         let b = self.find_free_block(length, self_position);
@@ -265,12 +270,13 @@ impl MemoryRoot {
 
         // Get the number of bytes available
         let mem_left = self_length - self_position - core::mem::size_of::<MemoryRoot>() as u64;
+        let length_with_block = length + core::mem::size_of::<MemoryBlock>() as u64;
 
         // When the allocation is larger than the number of bytes available
         // we will need to try to grow the memory
-        if length >= mem_left {
+        if length_with_block >= mem_left {
             // Calculate the number of pages needed to cover the remaining bytes
-            let npages = num_pages(length - mem_left);
+            let npages = num_pages(length_with_block - mem_left);
             let x = core::arch::wasm32::memory_grow(0, npages);
             if x == usize::MAX {
                 return None;
@@ -301,9 +307,26 @@ impl MemoryRoot {
         if !Self::pointer_in_bounds_fast(offs) {
             return None;
         }
-        let ptr = offs - core::mem::size_of::<MemoryBlock>() as u64;
-        let ptr = ptr as *mut MemoryBlock;
-        Some(&mut *ptr)
+
+        // Get the first block
+        let mut block = self.blocks.as_mut_ptr();
+
+        // Only loop while the block pointer is less then the current position
+        while (block as u64) < self.blocks.as_ptr() as u64 + offs {
+            let b = &mut *block;
+
+            // Get the block status, this lets us know if we are able to re-use it
+            let status = b.status.load(Ordering::Acquire);
+
+            if status == MemoryStatus::Active as u8 && b.data.as_ptr() as Pointer == offs {
+                return Some(b);
+            }
+
+            // Get the next block
+            block = b.next_ptr();
+        }
+
+        None
     }
 }
 
@@ -328,21 +351,21 @@ impl MemoryBlock {
 
 /// Allocate a block of memory and return the offset
 #[no_mangle]
-pub unsafe fn alloc(n: Length) -> Pointer {
+pub unsafe fn alloc(n: u64) -> Handle {
     if n == 0 {
         return 0;
     }
     let region = MemoryRoot::new();
     let block = region.alloc(n);
     match block {
-        Some(block) => block.data.as_mut_ptr() as Pointer,
+        Some(block) => block.data.as_mut_ptr() as Handle,
         None => 0,
     }
 }
 
 /// Free allocated memory
 #[no_mangle]
-pub unsafe fn free(p: Pointer) {
+pub unsafe fn free(p: Handle) {
     if p == 0 {
         return;
     }
@@ -360,13 +383,42 @@ pub unsafe fn free(p: Pointer) {
 }
 
 /// Get the length of an allocated memory block
+///
+/// Note: this should only be called on memory handles returned
+/// by a call to `alloc` - it will return garbage on invalid offsets
 #[no_mangle]
-pub unsafe fn length(p: Pointer) -> Length {
+pub unsafe fn length_unsafe(p: Handle) -> u64 {
     if p == 0 {
         return 0;
     }
+
+    if !MemoryRoot::pointer_in_bounds_fast(p) {
+        return 0;
+    }
+
+    let ptr = p - core::mem::size_of::<MemoryBlock>() as u64;
+    let block = &mut *(ptr as *mut MemoryBlock);
+
+    // Simplest sanity check to verify the pointer is a block
+    if block.status.load(Ordering::Acquire) != MemoryStatus::Active as u8 {
+        return 0;
+    }
+
+    block.used as u64
+}
+
+/// Get the length but returns 0 if the offset is not a valid handle.
+///
+/// Note: this function walks each node in the allocations list, which ensures correctness, but is also
+/// slow
+#[no_mangle]
+pub unsafe fn length(p: Pointer) -> u64 {
+    if p == 0 {
+        return 0;
+    }
+
     if let Some(block) = MemoryRoot::new().find_block(p) {
-        block.used as Length
+        block.used as u64
     } else {
         0
     }
@@ -394,24 +446,24 @@ pub unsafe fn load_u64(p: Pointer) -> u64 {
 
 /// Load a byte from the input data
 #[no_mangle]
-pub unsafe fn input_load_u8(p: Pointer) -> u8 {
+pub unsafe fn input_load_u8(offset: u64) -> u8 {
     let root = MemoryRoot::new();
     #[cfg(feature = "bounds-checking")]
-    if p >= root.input_length {
+    if offset >= root.input_length {
         return 0;
     }
-    *((root.input_offset + p) as *mut u8)
+    *((root.input_offset + offset) as *mut u8)
 }
 
 /// Load a u64 from the input data
 #[no_mangle]
-pub unsafe fn input_load_u64(p: Pointer) -> u64 {
+pub unsafe fn input_load_u64(offset: u64) -> u64 {
     let root = MemoryRoot::new();
     #[cfg(feature = "bounds-checking")]
-    if p + core::mem::size_of::<u64>() as Pointer > root.input_length {
+    if offset + core::mem::size_of::<u64>() as u64 > root.input_length {
         return 0;
     }
-    *((root.input_offset + p) as *mut u64)
+    *((root.input_offset + offset) as *mut u64)
 }
 
 /// Write a byte in Extism-managed memory
@@ -435,22 +487,24 @@ pub unsafe fn store_u64(p: Pointer, x: u64) {
 }
 
 /// Set the range of the input data in memory
+/// h must always be a handle so that length works on it
+/// len must match length(handle)
 #[no_mangle]
-pub unsafe fn input_set(p: Pointer, len: Length) {
+pub unsafe fn input_set(h: Handle, len: u64) {
     let root = MemoryRoot::new();
     #[cfg(feature = "bounds-checking")]
     {
-        if !root.pointer_in_bounds(p) || !root.pointer_in_bounds(p + len - 1) {
+        if !root.pointer_in_bounds(h) || !root.pointer_in_bounds(h + len - 1) {
             return;
         }
     }
-    root.input_offset = p;
+    root.input_offset = h;
     root.input_length = len;
 }
 
 /// Set the range of the output data in memory
 #[no_mangle]
-pub unsafe fn output_set(p: Pointer, len: Length) {
+pub unsafe fn output_set(p: Pointer, len: u64) {
     let root = MemoryRoot::new();
     #[cfg(feature = "bounds-checking")]
     {
@@ -464,25 +518,25 @@ pub unsafe fn output_set(p: Pointer, len: Length) {
 
 /// Get the input length
 #[no_mangle]
-pub fn input_length() -> Length {
+pub fn input_length() -> u64 {
     unsafe { MemoryRoot::new().input_length }
 }
 
 /// Get the input offset in Exitsm-managed memory
 #[no_mangle]
-pub fn input_offset() -> Length {
+pub fn input_offset() -> Handle {
     unsafe { MemoryRoot::new().input_offset }
 }
 
 /// Get the output length
 #[no_mangle]
-pub fn output_length() -> Length {
+pub fn output_length() -> u64 {
     unsafe { MemoryRoot::new().output_length }
 }
 
 /// Get the output offset in Extism-managed memory
 #[no_mangle]
-pub unsafe fn output_offset() -> Length {
+pub unsafe fn output_offset() -> Pointer {
     MemoryRoot::new().output_offset
 }
 
@@ -494,30 +548,94 @@ pub unsafe fn reset() {
 
 /// Set the error message offset
 #[no_mangle]
-pub unsafe fn error_set(ptr: Pointer) {
+pub unsafe fn error_set(h: Handle) {
     let root = MemoryRoot::new();
 
     // Allow ERROR to be set to 0
-    if ptr == 0 {
-        root.error.store(ptr, Ordering::SeqCst);
+    if h == 0 {
+        root.error.store(h, Ordering::SeqCst);
         return;
     }
 
     #[cfg(feature = "bounds-checking")]
-    if !root.pointer_in_bounds(ptr) {
+    if !root.pointer_in_bounds(h) {
         return;
     }
-    root.error.store(ptr, Ordering::SeqCst);
+    root.error.store(h, Ordering::SeqCst);
 }
 
 /// Get the error message offset, if it's `0` then no error has been set
 #[no_mangle]
-pub unsafe fn error_get() -> Pointer {
+pub unsafe fn error_get() -> Handle {
     MemoryRoot::new().error.load(Ordering::SeqCst)
 }
 
 /// Get the position of the allocator, this can be used as an indication of how many bytes are currently in-use
 #[no_mangle]
-pub unsafe fn memory_bytes() -> Length {
+pub unsafe fn memory_bytes() -> u64 {
     MemoryRoot::new().length.load(Ordering::Acquire)
 }
+
+#[cfg(test)]
+mod test {
+    use crate::*;
+    use wasm_bindgen_test::*;
+
+    // See https://github.com/extism/extism/pull/659
+    #[wasm_bindgen_test]
+    fn test_659() {
+        unsafe {
+            // Warning: These offsets will need to change if we adjust the kernel memory layout at all
+            reset();
+            assert_eq!(alloc(1065), 77);
+            assert_eq!(alloc(288), 1154);
+            assert_eq!(alloc(128), 1454);
+            assert_eq!(length(1154), 288);
+            assert_eq!(length(1454), 128);
+            free(1454);
+            assert_eq!(alloc(213), 1594);
+            length_unsafe(1594);
+            assert_eq!(alloc(511), 1819);
+            assert_eq!(alloc(4), 1454);
+            assert_eq!(length(1454), 4);
+            assert_eq!(length(1819), 511);
+            assert_eq!(alloc(13), 2342);
+            assert_eq!(length(2342), 13);
+            assert_eq!(alloc(336), 2367);
+            assert_eq!(alloc(1077), 2715);
+            assert_eq!(length(2367), 336);
+            assert_eq!(length(2715), 1077);
+            free(2715);
+            assert_eq!(alloc(1094), 3804);
+            length_unsafe(3804);
+
+            // Allocate 4 bytes, expect to receive address 3788
+            assert_eq!(alloc(4), 3788);
+
+            assert_eq!(alloc(4), 3772);
+            assert_eq!(length(3772), 4);
+
+            // Address 3788 has not been freed yet, so expect it to have 4 bytes allocated
+            assert_eq!(length(3788), 4);
+        }
+    }
+
+    #[wasm_bindgen_test]
+    fn test_oom() {
+        let size = 1024 * 1024 * 5;
+
+        let mut last = 0;
+        for _ in 0..1024 {
+            unsafe {
+                let ptr = alloc(size);
+                last = ptr;
+                if ptr == 0 {
+                    break;
+                }
+                assert_eq!(length(ptr), size);
+            }
+        }
+
+        assert_eq!(last, 0);
+    }
+}

kernel/test.sh

@@ -0,0 +1,5 @@
+# install wasm-bindgen-cli to get wasm-bindgen-runner if it is not installed yet
+which wasm-bindgen-test-runner 1>/dev/null || cargo install -f wasm-bindgen-cli
+
+# run tests with the wasm-bindgen-runner
+CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER=wasm-bindgen-test-runner cargo test --release --target=wasm32-unknown-unknown

libextism/API.md

@@ -90,6 +90,14 @@ Get the plugin's output data.
 const uint8_t *extism_plugin_output_data(ExtismPlugin *plugin);
 ```
 
+### `extism_plugin_reset`
+
+Reset the Extism runtime, this will invalidate all allocated memory.
+
+```c
+bool extism_plugin_reset(ExtismPlugin *plugin);
+```
+
 ### `extism_log_file`
 
 Set log file and level.

manifest/Cargo.toml

@@ -10,7 +10,7 @@ version.workspace = true
 
 [dependencies]
 serde = { version = "1", features = ["derive"] }
-base64 = "~0.21"
+base64 = "~0.22"
 schemars = { version = "0.8", optional = true }
 serde_json = "1"
 

manifest/schema.json

@@ -1,9 +1,11 @@
 {
   "$schema": "http://json-schema.org/draft-07/schema#",
   "title": "Manifest",
+  "description": "The `Manifest` type is used to configure the runtime and specify how to load modules.",
   "type": "object",
   "properties": {
     "allowed_hosts": {
+      "description": "Specifies which hosts may be accessed via HTTP, if this is empty then no hosts may be accessed. Wildcards may be used.",
       "default": null,
       "type": [
         "array",
@@ -14,6 +16,7 @@
       }
     },
     "allowed_paths": {
+      "description": "Specifies which paths should be made available on disk when using WASI. This is a mapping from the path on disk to the path it should be available inside the plugin. For example, `\".\": \"/tmp\"` would mount the current directory as `/tmp` inside the module",
       "default": null,
       "type": [
         "object",
@@ -24,6 +27,7 @@
       }
     },
     "config": {
+      "description": "Config values are made accessible using the PDK `extism_config_get` function",
       "default": {},
       "type": "object",
       "additionalProperties": {
@@ -31,8 +35,11 @@
       }
     },
     "memory": {
+      "description": "Memory options",
       "default": {
-        "max_pages": null
+        "max_http_response_bytes": null,
+        "max_pages": null,
+        "max_var_bytes": null
       },
       "allOf": [
         {
@@ -41,6 +48,8 @@
       ]
     },
     "timeout_ms": {
+      "description": "The plugin timeout in milliseconds",
+      "default": null,
       "type": [
         "integer",
         "null"
@@ -49,6 +58,7 @@
       "minimum": 0.0
     },
     "wasm": {
+      "description": "WebAssembly modules, the `main` module should be named `main` or listed last",
       "default": [],
       "type": "array",
       "items": {
@@ -56,35 +66,63 @@
       }
     }
   },
+  "additionalProperties": false,
   "definitions": {
     "MemoryOptions": {
+      "description": "Configure memory settings",
       "type": "object",
       "properties": {
+        "max_http_response_bytes": {
+          "description": "The maximum number of bytes allowed in an HTTP response",
+          "default": null,
+          "type": [
+            "integer",
+            "null"
+          ],
+          "format": "uint64",
+          "minimum": 0.0
+        },
         "max_pages": {
+          "description": "The max number of WebAssembly pages that should be allocated",
           "type": [
             "integer",
             "null"
           ],
           "format": "uint32",
           "minimum": 0.0
+        },
+        "max_var_bytes": {
+          "description": "The maximum number of bytes allowed to be used by plugin vars. Setting this to 0 will disable Extism vars. The default value is 1mb.",
+          "default": 1048576,
+          "type": [
+            "integer",
+            "null"
+          ],
+          "format": "uint64",
+          "minimum": 0.0
         }
-      }
+      },
+      "additionalProperties": false
     },
     "Wasm": {
+      "description": "The `Wasm` type specifies how to access a WebAssembly module",
       "anyOf": [
         {
+          "description": "From disk",
           "type": "object",
           "required": [
             "path"
           ],
           "properties": {
             "hash": {
+              "description": "Module hash, if the data loaded from disk or via HTTP doesn't match an error will be raised",
               "type": [
                 "string",
                 "null"
               ]
             },
             "name": {
+              "description": "Module name, this is used by Extism to determine which is the `main` module",
               "type": [
                 "string",
                 "null"
@@ -93,45 +131,72 @@
             "path": {
               "type": "string"
             }
-          }
+          },
+          "additionalProperties": false
         },
         {
+          "description": "From memory",
           "type": "object",
           "required": [
             "data"
           ],
           "properties": {
             "data": {
-              "type": "string",
-              "format": "string"
+              "type": [
+                "string",
+                "object"
+              ],
+              "required": [
+                "len",
+                "ptr"
+              ],
+              "properties": {
+                "len": {
+                  "type": "integer",
+                  "format": "uint64",
+                  "minimum": 0.0
+                },
+                "ptr": {
+                  "type": "integer",
+                  "format": "uint64",
+                  "minimum": 0.0
+                }
+              },
+              "additionalProperties": false
             },
             "hash": {
+              "description": "Module hash, if the data loaded from disk or via HTTP doesn't match an error will be raised",
               "type": [
                 "string",
                 "null"
               ]
             },
             "name": {
+              "description": "Module name, this is used by Extism to determine which is the `main` module",
               "type": [
                 "string",
                 "null"
               ]
             }
-          }
+          },
+          "additionalProperties": false
         },
         {
+          "description": "Via HTTP",
           "type": "object",
           "required": [
             "url"
           ],
           "properties": {
             "hash": {
+              "description": "Module hash, if the data loaded from disk or via HTTP doesn't match an error will be raised",
               "type": [
                 "string",
                 "null"
               ]
             },
             "headers": {
+              "description": "Request headers",
               "default": {},
               "type": "object",
               "additionalProperties": {
@@ -139,21 +204,25 @@
               }
             },
             "method": {
+              "description": "Request method",
               "type": [
                 "string",
                 "null"
               ]
             },
             "name": {
+              "description": "Module name, this is used by Extism to determine which is the `main` module",
               "type": [
                 "string",
                 "null"
               ]
             },
             "url": {
+              "description": "The request URL",
               "type": "string"
             }
-          }
+          },
+          "additionalProperties": false
         }
       ]
     }

manifest/src/lib.rs

@@ -12,6 +12,44 @@ pub struct MemoryOptions {
     /// The max number of WebAssembly pages that should be allocated
     #[serde(alias = "max")]
     pub max_pages: Option<u32>,
+
+    /// The maximum number of bytes allowed in an HTTP response
+    #[serde(default)]
+    pub max_http_response_bytes: Option<u64>,
+
+    /// The maximum number of bytes allowed to be used by plugin vars. Setting this to 0
+    /// will disable Extism vars. The default value is 1mb.
+    #[serde(default = "default_var_bytes")]
+    pub max_var_bytes: Option<u64>,
+}
+
+impl MemoryOptions {
+    /// Create an empty `MemoryOptions` value
+    pub fn new() -> Self {
+        Default::default()
+    }
+
+    /// Set max pages
+    pub fn with_max_pages(mut self, pages: u32) -> Self {
+        self.max_pages = Some(pages);
+        self
+    }
+
+    /// Set max HTTP response size
+    pub fn with_max_http_response_bytes(mut self, bytes: u64) -> Self {
+        self.max_http_response_bytes = Some(bytes);
+        self
+    }
+
+    /// Set max size of Extism vars
+    pub fn with_max_var_bytes(mut self, bytes: u64) -> Self {
+        self.max_var_bytes = Some(bytes);
+        self
+    }
+}
+
+fn default_var_bytes() -> Option<u64> {
+    Some(1024 * 1024)
 }
 
 /// Generic HTTP request structure
@@ -111,8 +149,8 @@ pub enum Wasm {
 
     /// From memory
     Data {
-        #[serde(with = "base64")]
-        #[cfg_attr(feature = "json_schema", schemars(schema_with = "base64_schema"))]
+        #[serde(with = "wasmdata")]
+        #[cfg_attr(feature = "json_schema", schemars(schema_with = "wasmdata_schema"))]
         data: Vec<u8>,
         #[serde(flatten)]
         meta: WasmMetadata,
@@ -195,11 +233,25 @@ impl Wasm {
     }
 }
 
+#[derive(Default, Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
+#[cfg_attr(feature = "json_schema", derive(schemars::JsonSchema))]
+#[serde(deny_unknown_fields)]
+struct DataPtrLength {
+    ptr: u64,
+    len: u64,
+}
+
 #[cfg(feature = "json_schema")]
-fn base64_schema(gen: &mut schemars::gen::SchemaGenerator) -> schemars::schema::Schema {
+fn wasmdata_schema(gen: &mut schemars::gen::SchemaGenerator) -> schemars::schema::Schema {
     use schemars::{schema::SchemaObject, JsonSchema};
     let mut schema: SchemaObject = <String>::json_schema(gen).into();
-    schema.format = Some("string".to_owned());
+    let objschema: SchemaObject = <DataPtrLength>::json_schema(gen).into();
+    let types = schemars::schema::SingleOrVec::<schemars::schema::InstanceType>::Vec(vec![
+        schemars::schema::InstanceType::String,
+        schemars::schema::InstanceType::Object,
+    ]);
+    schema.instance_type = Some(types);
+    schema.object = objschema.object;
     schema.into()
 }
 
@@ -211,6 +263,7 @@ pub struct Manifest {
     /// WebAssembly modules, the `main` module should be named `main` or listed last
     #[serde(default)]
     pub wasm: Vec<Wasm>,
+
     /// Memory options
     #[serde(default)]
     pub memory: MemoryOptions,
@@ -230,21 +283,16 @@ pub struct Manifest {
     #[serde(default)]
     pub allowed_paths: Option<BTreeMap<PathBuf, PathBuf>>,
 
-    /// The plugin timeout, by default this is set to 30s
-    #[serde(default = "default_timeout")]
+    /// The plugin timeout in milliseconds
+    #[serde(default)]
     pub timeout_ms: Option<u64>,
 }
 
-fn default_timeout() -> Option<u64> {
-    Some(30000)
-}
-
 impl Manifest {
     /// Create a new manifest
     pub fn new(wasm: impl IntoIterator<Item = impl Into<Wasm>>) -> Manifest {
         Manifest {
             wasm: wasm.into_iter().map(|x| x.into()).collect(),
-            timeout_ms: default_timeout(),
             ..Default::default()
         }
     }
@@ -340,10 +388,12 @@ impl Manifest {
     }
 }
 
-mod base64 {
+mod wasmdata {
+    use crate::DataPtrLength;
     use base64::{engine::general_purpose, Engine as _};
     use serde::{Deserialize, Serialize};
     use serde::{Deserializer, Serializer};
+    use std::slice;
 
     pub fn serialize<S: Serializer>(v: &Vec<u8>, s: S) -> Result<S::Ok, S::Error> {
         let base64 = general_purpose::STANDARD.encode(v.as_slice());
@@ -351,10 +401,22 @@ mod base64 {
     }
 
     pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Vec<u8>, D::Error> {
-        let base64 = String::deserialize(d)?;
-        general_purpose::STANDARD
-            .decode(base64.as_bytes())
-            .map_err(serde::de::Error::custom)
+        #[derive(Deserialize)]
+        #[serde(untagged)]
+        enum WasmDataTypes {
+            String(String),
+            DataPtrLength(DataPtrLength),
+        }
+        Ok(match WasmDataTypes::deserialize(d)? {
+            WasmDataTypes::String(string) => general_purpose::STANDARD
+                .decode(string.as_bytes())
+                .map_err(serde::de::Error::custom)?,
+            WasmDataTypes::DataPtrLength(ptrlen) => {
+                let slice =
+                    unsafe { slice::from_raw_parts(ptrlen.ptr as *const u8, ptrlen.len as usize) };
+                slice.to_vec()
+            }
+        })
     }
 }
 

runtime/Cargo.toml

@@ -9,8 +9,8 @@ repository.workspace = true
 version.workspace = true
 
 [dependencies]
-wasmtime = ">= 14.0.0, < 16.0.0"
-wasmtime-wasi = ">= 14.0.0, < 16.0.0"
+wasmtime = ">= 14.0.0, < 18.0.0"
+wasmtime-wasi = ">= 14.0.0, < 18.0.0"
 anyhow = "1"
 serde = {version = "1", features = ["derive"]}
 serde_json = "1"
@@ -22,7 +22,7 @@ url = "2"
 glob = "0.3"
 ureq = {version = "2.5", optional=true}
 extism-manifest = { workspace = true }
-extism-convert = { workspace = true }
+extism-convert = { workspace = true, features = ["extism-path"] }
 uuid = { version = "1", features = ["v4"] }
 libc = "0.2"
 
@@ -33,10 +33,12 @@ register-filesystem = [] # enables wasm to be loaded from disk
 http = ["ureq"]          # enables extism_http_request
 
 [build-dependencies]
-cbindgen = "0.26"
+cbindgen = { version = "0.26", default-features = false }
 
 [dev-dependencies]
 criterion = "0.5.1"
+quickcheck = "1"
+rand = "0.8.5"
 
 [[bench]]
 name = "bench"

runtime/README.md

@@ -12,7 +12,7 @@ To use the `extism` crate, you can add it to your Cargo file:
 
 ```toml
 [dependencies]
-extism = "^1.0.0-rc3"
+extism = "1.0.0"
 ```
 
 ## Environment variables

runtime/benches/bench.rs

@@ -118,9 +118,7 @@ pub fn echo(c: &mut Criterion) {
 
 pub fn reflect(c: &mut Criterion) {
     let mut g = c.benchmark_group("reflect");
-    g.sample_size(500);
-    g.noise_threshold(1.0);
-    g.significance_level(0.2);
+
     let mut plugin = PluginBuilder::new(REFLECT)
         .with_wasi(true)
         .with_function(
@@ -132,22 +130,23 @@ pub fn reflect(c: &mut Criterion) {
         )
         .build()
         .unwrap();
-
     for (i, elements) in [
         b"a".repeat(65536),
         b"a".repeat(65536 * 10),
         b"a".repeat(65536 * 100),
+        b"a".repeat(65536),
     ]
     .iter()
     .enumerate()
     {
         g.throughput(criterion::Throughput::Bytes(elements.len() as u64));
         g.bench_with_input(
-            format!("reflect {} bytes", 10u32.pow(i as u32) * 65536),
+            format!("{i}: reflect {} bytes", elements.len()),
             elements,
             |b, elems| {
                 b.iter(|| {
                     assert_eq!(elems, plugin.call::<_, &[u8]>("reflect", &elems).unwrap());
+                    // plugin.reset().unwrap();
                 });
             },
         );

runtime/examples/readme.rs

@@ -1,7 +1,7 @@
 use extism::*;
 
 // pretend this is redis or something :)
-type KVStore = std::collections::BTreeMap<String, Vec<u8>>;
+type KVStore = std::sync::Arc<std::sync::Mutex<std::collections::BTreeMap<String, Vec<u8>>>>;
 
 // When a first argument separated with a semicolon is provided to `host_fn` it is used as the
 // variable name and type for the `UserData` parameter

runtime/extism.h

@@ -97,6 +97,11 @@ typedef void (*ExtismFunctionType)(ExtismCurrentPlugin *plugin,
                                    ExtismSize n_outputs,
                                    void *data);
 
+/**
+ * Log drain callback
+ */
+typedef void (*ExtismLogDrainFunctionType)(const char *data, ExtismSize size);
+
 
 
 #ifdef __cplusplus
@@ -264,7 +269,12 @@ bool extism_log_custom(const char *log_level);
  * Calls the provided callback function for each buffered log line.
  * This is only needed when `extism_log_custom` is used.
  */
-void extism_log_drain(void (*handler)(const char*, uintptr_t));
+void extism_log_drain(ExtismLogDrainFunctionType handler);
+
+/**
+ * Reset the Extism runtime, this will invalidate all allocated memory
+ */
+bool extism_plugin_reset(ExtismPlugin *plugin);
 
 /**
  * Get the Extism version string

runtime/src/current_plugin.rs

@@ -87,18 +87,27 @@ impl CurrentPlugin {
     }
 
     /// Access memory bytes as `str`
-    pub fn memory_str(&mut self, handle: MemoryHandle) -> Result<&mut str, Error> {
-        let bytes = self.memory_bytes(handle)?;
+    pub fn memory_str_mut(&mut self, handle: MemoryHandle) -> Result<&mut str, Error> {
+        let bytes = self.memory_bytes_mut(handle)?;
         let s = std::str::from_utf8_mut(bytes)?;
         Ok(s)
     }
 
+    pub fn memory_str(&mut self, handle: MemoryHandle) -> Result<&str, Error> {
+        let bytes = self.memory_bytes(handle)?;
+        let s = std::str::from_utf8(bytes)?;
+        Ok(s)
+    }
+
     /// Allocate a handle large enough for the encoded Rust type and copy it into Extism memory
     pub fn memory_new<'a, T: ToBytes<'a>>(&mut self, t: T) -> Result<MemoryHandle, Error> {
         let data = t.to_bytes()?;
         let data = data.as_ref();
+        if data.is_empty() {
+            return Ok(MemoryHandle::null());
+        }
         let handle = self.memory_alloc(data.len() as u64)?;
-        let bytes = self.memory_bytes(handle)?;
+        let bytes = self.memory_bytes_mut(handle)?;
         bytes.copy_from_slice(data.as_ref());
         Ok(handle)
     }
@@ -144,7 +153,7 @@ impl CurrentPlugin {
         Ok(())
     }
 
-    pub fn memory_bytes(&mut self, handle: MemoryHandle) -> Result<&mut [u8], Error> {
+    pub fn memory_bytes_mut(&mut self, handle: MemoryHandle) -> Result<&mut [u8], Error> {
         let (linker, mut store) = self.linker_and_store();
         if let Some(mem) = linker.get(&mut store, EXTISM_ENV_MODULE, "memory") {
             let mem = mem.into_memory().unwrap();
@@ -158,6 +167,20 @@ impl CurrentPlugin {
         anyhow::bail!("{} unable to locate extism memory", self.id)
     }
 
+    pub fn memory_bytes(&mut self, handle: MemoryHandle) -> Result<&[u8], Error> {
+        let (linker, mut store) = self.linker_and_store();
+        if let Some(mem) = linker.get(&mut store, EXTISM_ENV_MODULE, "memory") {
+            let mem = mem.into_memory().unwrap();
+            let ptr = unsafe { mem.data_ptr(&store).add(handle.offset() as usize) };
+            if ptr.is_null() {
+                return Ok(&[]);
+            }
+            return Ok(unsafe { std::slice::from_raw_parts(ptr, handle.len()) });
+        }
+
+        anyhow::bail!("{} unable to locate extism memory", self.id)
+    }
+
     pub fn memory_alloc(&mut self, n: u64) -> Result<MemoryHandle, Error> {
         if n == 0 {
             return Ok(MemoryHandle {
@@ -223,6 +246,26 @@ impl CurrentPlugin {
         Ok(len)
     }
 
+    pub fn memory_length_unsafe(&mut self, offs: u64) -> Result<u64, Error> {
+        let (linker, mut store) = self.linker_and_store();
+        let output = &mut [Val::I64(0)];
+        if let Some(f) = linker.get(&mut store, EXTISM_ENV_MODULE, "length_unsafe") {
+            f.into_func()
+                .unwrap()
+                .call(&mut store, &[Val::I64(offs as i64)], output)?;
+        } else {
+            anyhow::bail!("unable to locate an extism kernel function: length_unsafe",)
+        }
+        let len = output[0].unwrap_i64() as u64;
+        trace!(
+            plugin = self.id.to_string(),
+            "memory_length_unsafe({}) = {}",
+            offs,
+            len
+        );
+        Ok(len)
+    }
+
     /// Access a plugin's variables
     pub fn vars(&self) -> &std::collections::BTreeMap<String, Vec<u8>> {
         &self.vars

runtime/src/function.rs

@@ -81,18 +81,18 @@ pub(crate) enum UserDataHandle {
 /// using `UserData::get`. The `C` data is stored as a pointer and cleanup function and isn't usable from Rust. The cleanup function
 /// will be called when the inner `CPtr` is dropped.
 #[derive(Debug)]
-pub enum UserData<T: Sized> {
+pub enum UserData<T: Sync + Clone + Sized> {
     C(Arc<CPtr>),
-    Rust(Arc<std::sync::Mutex<T>>),
+    Rust(T),
 }
 
-impl<T: Default> Default for UserData<T> {
+impl<T: Default + Sync + Clone> Default for UserData<T> {
     fn default() -> Self {
         UserData::new(T::default())
     }
 }
 
-impl<T> Clone for UserData<T> {
+impl<T: Sync + Clone> Clone for UserData<T> {
     fn clone(&self) -> Self {
         match self {
             UserData::C(ptr) => UserData::C(ptr.clone()),
@@ -101,7 +101,7 @@ impl<T> Clone for UserData<T> {
     }
 }
 
-impl<T> UserData<T> {
+impl<T: Sync + Clone> UserData<T> {
     /// Create a new `UserData` from an existing pointer and free function, this is used
     /// by the C API to wrap C pointers into user data
     pub(crate) fn new_pointer(
@@ -126,12 +126,11 @@ impl<T> UserData<T> {
     ///
     /// This will wrap the provided value in a reference-counted mutex
     pub fn new(x: T) -> Self {
-        let data = Arc::new(std::sync::Mutex::new(x));
-        UserData::Rust(data)
+        UserData::Rust(x)
     }
 
     /// Get a copy of the inner value
-    pub fn get(&self) -> Result<Arc<std::sync::Mutex<T>>, Error> {
+    pub fn get(&self) -> Result<T, Error> {
         match self {
             UserData::C { .. } => anyhow::bail!("C UserData should not be used from Rust"),
             UserData::Rust(data) => Ok(data.clone()),
@@ -150,8 +149,8 @@ impl Drop for CPtr {
     }
 }
 
-unsafe impl<T> Send for UserData<T> {}
-unsafe impl<T> Sync for UserData<T> {}
+unsafe impl<T: Sync + Clone> Send for UserData<T> {}
+unsafe impl<T: Sync + Clone> Sync for UserData<T> {}
 unsafe impl Send for CPtr {}
 unsafe impl Sync for CPtr {}
 
@@ -180,7 +179,7 @@ pub struct Function {
 
 impl Function {
     /// Create a new host function
-    pub fn new<T: 'static, F>(
+    pub fn new<T: 'static + Sync + Clone, F>(
         name: impl Into<String>,
         args: impl IntoIterator<Item = ValType>,
         returns: impl IntoIterator<Item = ValType>,
@@ -211,7 +210,9 @@ impl Function {
             namespace: None,
             _user_data: match &user_data {
                 UserData::C(ptr) => UserDataHandle::C(ptr.clone()),
-                UserData::Rust(x) => UserDataHandle::Rust(x.clone()),
+                UserData::Rust(x) => {
+                    UserDataHandle::Rust(std::sync::Arc::new(std::sync::Mutex::new(x.clone())))
+                }
             },
         }
     }

runtime/src/lib.rs

@@ -1,8 +1,12 @@
+// Makes proc-macros able to resolve `::extism` correctly
+extern crate self as extism;
+
 pub(crate) use extism_convert::*;
 pub(crate) use std::collections::BTreeMap;
 use std::str::FromStr;
 pub(crate) use wasmtime::*;
 
+#[doc(inline)]
 pub use extism_convert as convert;
 
 pub use anyhow::Error;

runtime/src/manifest.rs

@@ -117,10 +117,17 @@ pub(crate) fn load(
     match input {
         WasmInput::Data(data) => {
             let has_magic = data.len() >= 4 && data[0..4] == WASM_MAGIC;
-            let is_wat = data.starts_with(b"(module") || data.starts_with(b";;");
+            let s = std::str::from_utf8(&data);
+            let is_wat = s.is_ok_and(|s| {
+                let s = s.trim_start();
+                let starts_with_module = s.len() > 2
+                    && data[0] == b'('   // First character is `(`
+                    && s[1..].trim_start().starts_with("module"); // Then `module` (after any whitespace)
+                starts_with_module || s.starts_with(";;") || s.starts_with("(;")
+            });
             if !has_magic && !is_wat {
                 trace!("Loading manifest");
-                if let Ok(s) = std::str::from_utf8(&data) {
+                if let Ok(s) = s {
                     let t = if let Ok(t) = toml::from_str::<extism_manifest::Manifest>(s) {
                         trace!("Manifest is TOML");
                         modules(engine, &t, &mut mods)?;

runtime/src/pdk.rs

@@ -97,19 +97,13 @@ pub(crate) fn var_set(
 ) -> Result<(), Error> {
     let data: &mut CurrentPlugin = caller.data_mut();
 
-    let mut size = 0;
-    for v in data.vars.values() {
-        size += v.len();
+    if data.manifest.memory.max_var_bytes.is_some_and(|x| x == 0) {
+        anyhow::bail!("Vars are disabled by this host")
     }
 
     let voffset = args!(input, 1, i64) as u64;
-
-    // If the store is larger than 100MB then stop adding things
-    if size > 1024 * 1024 * 100 && voffset != 0 {
-        return Err(Error::msg("Variable store is full"));
-    }
-
     let key_offs = args!(input, 0, i64) as u64;
+
     let key = {
         let handle = match data.memory_handle(key_offs) {
             Some(h) => h,
@@ -132,6 +126,22 @@ pub(crate) fn var_set(
         None => anyhow::bail!("invalid handle offset for var value: {voffset}"),
     };
 
+    let mut size = std::mem::size_of::<String>()
+        + std::mem::size_of::<Vec<u8>>()
+        + key.len()
+        + handle.length as usize;
+
+    for (k, v) in data.vars.iter() {
+        size += k.len();
+        size += v.len();
+        size += std::mem::size_of::<String>() + std::mem::size_of::<Vec<u8>>();
+    }
+
+    // If the store is larger than the configured size, or 1mb by default, then stop adding things
+    if size > data.manifest.memory.max_var_bytes.unwrap_or(1024 * 1024) as usize && voffset != 0 {
+        return Err(Error::msg("Variable store is full"));
+    }
+
     let value = data.memory_bytes(handle)?.to_vec();
 
     // Insert the value from memory into the `vars` map
@@ -226,20 +236,29 @@ pub(crate) fn http_request(
                 Some(res.into_reader())
             }
             Err(e) => {
+                let msg = e.to_string();
                 if let Some(res) = e.into_response() {
                     data.http_status = res.status();
                     Some(res.into_reader())
                 } else {
-                    None
+                    return Err(Error::msg(msg));
                 }
             }
         };
 
         if let Some(reader) = reader {
             let mut buf = Vec::new();
-            reader
-                .take(1024 * 1024 * 50) // TODO: make this limit configurable
-                .read_to_end(&mut buf)?;
+            let max = if let Some(max) = &data.manifest.memory.max_http_response_bytes {
+                reader.take(*max + 1).read_to_end(&mut buf)?;
+                *max
+            } else {
+                reader.take(1024 * 1024 * 50 + 1).read_to_end(&mut buf)?;
+                1024 * 1024 * 50
+            };
+
+            if buf.len() > max as usize {
+                anyhow::bail!("HTTP response exceeds the configured maximum number of bytes: {max}")
+            }
 
             let mem = data.memory_new(&buf)?;
             output[0] = Val::I64(mem.offset() as i64);

runtime/src/plugin.rs

@@ -1,4 +1,7 @@
-use std::{collections::BTreeMap, path::PathBuf};
+use std::{
+    collections::{BTreeMap, BTreeSet},
+    path::PathBuf,
+};
 
 use crate::*;
 
@@ -74,7 +77,7 @@ pub struct Plugin {
 
     /// Set to `true` when de-initializarion may have occured (i.e.a call to `_start`),
     /// in this case we need to re-initialize the entire module.
-    pub(crate) needs_reset: bool,
+    pub(crate) store_needs_reset: bool,
 
     pub(crate) debug_options: DebugOptions,
 }
@@ -177,6 +180,39 @@ impl<'a> From<&'a Vec<u8>> for WasmInput<'a> {
     }
 }
 
+fn add_module<T: 'static>(
+    store: &mut Store<T>,
+    linker: &mut Linker<T>,
+    linked: &mut BTreeSet<String>,
+    modules: &BTreeMap<String, Module>,
+    name: String,
+    module: &Module,
+) -> Result<(), Error> {
+    if linked.contains(&name) {
+        return Ok(());
+    }
+
+    for import in module.imports() {
+        if !linked.contains(import.module()) {
+            if let Some(m) = modules.get(import.module()) {
+                add_module(
+                    store,
+                    linker,
+                    linked,
+                    modules,
+                    import.module().to_string(),
+                    m,
+                )?;
+            }
+        }
+    }
+
+    linker.module(store, name.as_str(), module)?;
+    linked.insert(name);
+
+    Ok(())
+}
+
 impl Plugin {
     /// Create a new plugin from a Manifest or WebAssembly module, and host functions. The `with_wasi`
     /// parameter determines whether or not the module should be executed with WASI enabled.
@@ -240,22 +276,6 @@ impl Plugin {
         store.set_epoch_deadline(1);
 
         let mut linker = Linker::new(&engine);
-        linker.allow_shadowing(true);
-
-        // If wasi is enabled then add it to the linker
-        if with_wasi {
-            wasmtime_wasi::add_to_linker(&mut linker, |x: &mut CurrentPlugin| {
-                &mut x.wasi.as_mut().unwrap().ctx
-            })?;
-        }
-
-        let main = &modules[MAIN_KEY];
-        for (name, module) in modules.iter() {
-            if name != MAIN_KEY {
-                linker.module(&mut store, name, module)?;
-            }
-        }
-
         let mut imports: Vec<_> = imports.into_iter().collect();
         // Define PDK functions
         macro_rules! add_funcs {
@@ -281,14 +301,37 @@ impl Plugin {
             log_error(I64);
         );
 
+        let mut linked = BTreeSet::new();
+        linker.module(&mut store, EXTISM_ENV_MODULE, &modules[EXTISM_ENV_MODULE])?;
+        linked.insert(EXTISM_ENV_MODULE.to_string());
+
+        // If wasi is enabled then add it to the linker
+        if with_wasi {
+            wasmtime_wasi::add_to_linker(&mut linker, |x: &mut CurrentPlugin| {
+                &mut x.wasi.as_mut().unwrap().ctx
+            })?;
+        }
+
         for f in &mut imports {
-            let name = f.name().to_string();
+            let name = f.name();
             let ns = f.namespace().unwrap_or(EXTISM_USER_MODULE);
             unsafe {
-                linker.func_new(ns, &name, f.ty().clone(), &*(f.f.as_ref() as *const _))?;
+                linker.func_new(ns, name, f.ty().clone(), &*(f.f.as_ref() as *const _))?;
             }
         }
 
+        for (name, module) in modules.iter() {
+            add_module(
+                &mut store,
+                &mut linker,
+                &mut linked,
+                &modules,
+                name.clone(),
+                module,
+            )?;
+        }
+
+        let main = &modules[MAIN_KEY];
         let instance_pre = linker.instantiate_pre(main)?;
         let timer_tx = Timer::tx();
         let mut plugin = Plugin {
@@ -303,7 +346,7 @@ impl Plugin {
             cancel_handle: CancelHandle { id, timer_tx },
             instantiations: 0,
             output: Output::default(),
-            needs_reset: false,
+            store_needs_reset: false,
             debug_options,
             _functions: imports,
         };
@@ -324,7 +367,7 @@ impl Plugin {
         &mut self,
         instance_lock: &mut std::sync::MutexGuard<Option<Instance>>,
     ) -> Result<(), Error> {
-        if self.instantiations > 100 {
+        if self.store_needs_reset {
             let engine = self.store.engine().clone();
             let internal = self.current_plugin_mut();
             self.store = Store::new(
@@ -355,9 +398,9 @@ impl Plugin {
             }
             self.instantiations = 0;
             self.instance_pre = self.linker.instantiate_pre(main)?;
+            **instance_lock = None;
+            self.store_needs_reset = false;
         }
-
-        **instance_lock = None;
         Ok(())
     }
 
@@ -403,7 +446,18 @@ impl Plugin {
     pub fn function_exists(&mut self, function: impl AsRef<str>) -> bool {
         self.modules[MAIN_KEY]
             .get_export(function.as_ref())
-            .map(|x| x.func().is_some())
+            .map(|x| {
+                if let Some(f) = x.func() {
+                    let (params, mut results) = (f.params(), f.results());
+                    match (params.len(), results.len()) {
+                        (0, 1) => results.next() == Some(wasmtime::ValType::I32),
+                        (0, 0) => true,
+                        _ => false,
+                    }
+                } else {
+                    false
+                }
+            })
             .unwrap_or(false)
     }
 
@@ -428,12 +482,7 @@ impl Plugin {
         let bytes = unsafe { std::slice::from_raw_parts(input, len) };
         debug!(plugin = &id, "input size: {}", bytes.len());
 
-        if let Some(f) = self.linker.get(&mut self.store, EXTISM_ENV_MODULE, "reset") {
-            f.into_func().unwrap().call(&mut self.store, &[], &mut [])?;
-        } else {
-            error!(plugin = &id, "call to extism:host/env::reset failed");
-        }
-
+        self.reset()?;
         let handle = self.current_plugin_mut().memory_new(bytes)?;
 
         if let Some(f) = self
@@ -450,6 +499,19 @@ impl Plugin {
         Ok(())
     }
 
+    /// Reset Extism runtime, this will invalidate all allocated memory
+    pub fn reset(&mut self) -> Result<(), Error> {
+        let id = self.id.to_string();
+
+        if let Some(f) = self.linker.get(&mut self.store, EXTISM_ENV_MODULE, "reset") {
+            f.into_func().unwrap().call(&mut self.store, &[], &mut [])?;
+        } else {
+            error!(plugin = &id, "call to extism:host/env::reset failed");
+        }
+
+        Ok(())
+    }
+
     /// Determine if wasi is enabled
     pub fn has_wasi(&self) -> bool {
         self.current_plugin().wasi.is_some()
@@ -615,14 +677,11 @@ impl Plugin {
         let name = name.as_ref();
         let input = input.as_ref();
 
-        if self.needs_reset {
-            if let Err(e) = self.reset_store(lock) {
-                error!(
-                    plugin = self.id.to_string(),
-                    "call to Plugin::reset_store failed: {e:?}"
-                );
-            }
-            self.needs_reset = false;
+        if let Err(e) = self.reset_store(lock) {
+            error!(
+                plugin = self.id.to_string(),
+                "call to Plugin::reset_store failed: {e:?}"
+            );
         }
 
         self.instantiate(lock).map_err(|e| (e, -1))?;
@@ -667,7 +726,7 @@ impl Plugin {
         self.store
             .epoch_deadline_callback(|_| Ok(UpdateDeadline::Continue(1)));
         let _ = self.timer_tx.send(TimerAction::Stop { id: self.id });
-        self.needs_reset = name == "_start";
+        self.store_needs_reset = name == "_start";
 
         // Get extism error
         self.get_output_after_call().map_err(|x| (x, -1))?;
@@ -819,6 +878,24 @@ impl Plugin {
             .and_then(move |_| self.output())
     }
 
+    /// Similar to `Plugin::call`, but returns the Extism error code along with the
+    /// `Error`. It is assumed if `Ok(_)` is returned that the error code was `0`.
+    ///
+    /// All Extism plugin calls return an error code, `Plugin::call` consumes the error code,
+    /// while `Plugin::call_get_error_code` preserves it - this function should only be used
+    /// when you need to inspect the actual return value of a plugin function when it fails.
+    pub fn call_get_error_code<'a, 'b, T: ToBytes<'a>, U: FromBytes<'b>>(
+        &'b mut self,
+        name: impl AsRef<str>,
+        input: T,
+    ) -> Result<U, (Error, i32)> {
+        let lock = self.instance.clone();
+        let mut lock = lock.lock().unwrap();
+        let data = input.to_bytes().map_err(|e| (e, -1))?;
+        self.raw_call(&mut lock, name, data)
+            .and_then(move |_| self.output().map_err(|e| (e, -1)))
+    }
+
     /// Get a `CancelHandle`, which can be used from another thread to cancel a running plugin
     pub fn cancel_handle(&self) -> CancelHandle {
         self.cancel_handle.clone()

runtime/src/plugin_builder.rs

@@ -60,7 +60,7 @@ impl<'a> PluginBuilder<'a> {
     }
 
     /// Add a single host function
-    pub fn with_function<T: 'static, F>(
+    pub fn with_function<T: Sync + Clone + 'static, F>(
         mut self,
         name: impl Into<String>,
         args: impl IntoIterator<Item = ValType>,
@@ -80,7 +80,7 @@ impl<'a> PluginBuilder<'a> {
     }
 
     /// Add a single host function in a specific namespace
-    pub fn with_function_in_namespace<T: 'static, F>(
+    pub fn with_function_in_namespace<T: Sync + Clone + 'static, F>(
         mut self,
         namespace: impl Into<String>,
         name: impl Into<String>,

runtime/src/sdk.rs

@@ -38,6 +38,9 @@ pub type ExtismFunctionType = extern "C" fn(
     data: *mut std::ffi::c_void,
 );
 
+/// Log drain callback
+pub type ExtismLogDrainFunctionType = extern "C" fn(data: *const std::ffi::c_char, size: Size);
+
 impl From<&wasmtime::Val> for ExtismVal {
     fn from(value: &wasmtime::Val) -> Self {
         match value.ty() {
@@ -667,11 +670,11 @@ unsafe fn set_log_buffer(filter: &str) -> Result<(), Error> {
 #[no_mangle]
 /// Calls the provided callback function for each buffered log line.
 /// This is only needed when `extism_log_custom` is used.
-pub unsafe extern "C" fn extism_log_drain(handler: extern "C" fn(*const std::ffi::c_char, usize)) {
+pub unsafe extern "C" fn extism_log_drain(handler: ExtismLogDrainFunctionType) {
     if let Some(buf) = &mut LOG_BUFFER {
         if let Ok(mut buf) = buf.buffer.lock() {
             for (line, len) in buf.drain(..) {
-                handler(line.as_ptr(), len);
+                handler(line.as_ptr(), len as u64);
             }
         }
     }
@@ -702,6 +705,30 @@ impl std::io::Write for LogBuffer {
     }
 }
 
+/// Reset the Extism runtime, this will invalidate all allocated memory
+#[no_mangle]
+pub unsafe extern "C" fn extism_plugin_reset(plugin: *mut Plugin) -> bool {
+    let plugin = &mut *plugin;
+
+    if let Err(e) = plugin.reset() {
+        error!(
+            plugin = plugin.id.to_string(),
+            "unable to reset plugin: {}",
+            e.to_string()
+        );
+        if let Err(e) = plugin.current_plugin_mut().set_error(e.to_string()) {
+            error!(
+                plugin = plugin.id.to_string(),
+                "unable to set error after failed plugin reset: {}",
+                e.to_string()
+            );
+        }
+        false
+    } else {
+        true
+    }
+}
+
 /// Get the Extism version string
 #[no_mangle]
 pub unsafe extern "C" fn extism_version() -> *const c_char {

runtime/src/tests/kernel.rs

@@ -1,4 +1,5 @@
 use crate::*;
+use quickcheck::*;
 
 const KERNEL: &[u8] = include_bytes!("../extism-runtime.wasm");
 
@@ -22,6 +23,20 @@ fn extism_length<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance,
     out[0].unwrap_i64() as u64
 }
 
+fn extism_length_unsafe<T>(
+    mut store: &mut wasmtime::Store<T>,
+    instance: &mut Instance,
+    p: u64,
+) -> u64 {
+    let out = &mut [Val::I64(0)];
+    instance
+        .get_func(&mut store, "length_unsafe")
+        .unwrap()
+        .call(&mut store, &[Val::I64(p as i64)], out)
+        .unwrap();
+    out[0].unwrap_i64() as u64
+}
+
 fn extism_load_u8<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) -> u8 {
     let out = &mut [Val::I32(0)];
     instance
@@ -173,6 +188,7 @@ fn test_kernel_allocations() {
     let first_alloc = p;
     assert!(p > 0);
     assert_eq!(extism_length(&mut store, instance, p), 1);
+    assert_eq!(extism_length_unsafe(&mut store, instance, p), 1);
     extism_free(&mut store, instance, p);
 
     // 2 bytes
@@ -180,24 +196,31 @@ fn test_kernel_allocations() {
     assert!(x > 0);
     assert!(x != p);
     assert_eq!(extism_length(&mut store, instance, x), 2);
+    assert_eq!(extism_length_unsafe(&mut store, instance, x), 2);
     extism_free(&mut store, instance, x);
 
     for i in 0..64 {
         let p = extism_alloc(&mut store, instance, 64 - i);
         assert!(p > 0);
         assert_eq!(extism_length(&mut store, instance, p), 64 - i);
+        assert_eq!(extism_length_unsafe(&mut store, instance, p), 64 - i);
         extism_free(&mut store, instance, p);
 
         // should re-use the last allocation
         let q = extism_alloc(&mut store, instance, 64 - i);
         assert_eq!(p, q);
         assert_eq!(extism_length(&mut store, instance, q), 64 - i);
+        assert_eq!(extism_length_unsafe(&mut store, instance, q), 64 - i);
         extism_free(&mut store, instance, q);
     }
 
     // 512 bytes, test block re-use + splitting
     let p = extism_alloc(&mut store, instance, 512);
     assert_eq!(extism_length(&mut store, instance, p), 512);
+    assert_eq!(extism_length(&mut store, instance, p + 1), 0);
+    assert_eq!(extism_length(&mut store, instance, p + 2), 0);
+    assert_eq!(extism_length(&mut store, instance, p + 3), 0);
+    assert_eq!(extism_length(&mut store, instance, p + 4), 0);
     extism_free(&mut store, instance, p);
 
     // 128 bytes, should be split off the 512 byte block
@@ -210,7 +233,7 @@ fn test_kernel_allocations() {
     let r = extism_alloc(&mut store, instance, 128);
     assert!(p <= r && r < p + 512);
     assert!(r > p);
-    assert_eq!(extism_length(&mut store, instance, q), 128);
+    assert_eq!(extism_length(&mut store, instance, r), 128);
     extism_free(&mut store, instance, q);
 
     // 100 pages
@@ -297,3 +320,158 @@ fn test_load_input() {
     // Out of bounds should return 0
     assert_eq!(extism_input_load_u64(&mut store, instance, 123457), 0);
 }
+
+#[test]
+fn test_failed_quickcheck1() {
+    let (mut store, mut instance) = init_kernel_test();
+    let allocs = [
+        20622, 23162, 58594, 32421, 25928, 44611, 26318, 24455, 5798, 60202, 42126, 64928, 57832,
+        50888, 63256, 37562, 46334, 47985, 60836, 28132, 65535, 37800, 33150, 48768, 38457, 57249,
+        5734, 58587, 26294, 26653, 24519, 1,
+    ];
+
+    extism_reset(&mut store, &mut instance);
+    for a in allocs {
+        println!("Alloc: {a}");
+        let n = extism_alloc(&mut store, &mut instance, a);
+        if n == 0 {
+            continue;
+        }
+        assert_eq!(a, extism_length(&mut store, &mut instance, n));
+    }
+}
+
+#[test]
+fn test_failed_quickcheck2() {
+    let (mut store, mut instance) = init_kernel_test();
+    let allocs = [352054710, 1248853976, 2678441931, 14567928];
+
+    extism_reset(&mut store, &mut instance);
+    for a in allocs {
+        println!("Alloc: {a}");
+        let n = extism_alloc(&mut store, &mut instance, a);
+        if n == 0 {
+            continue;
+        }
+        assert_eq!(a, extism_length(&mut store, &mut instance, n));
+    }
+}
+
+quickcheck! {
+    fn check_alloc(amounts: Vec<u16>) -> bool {
+        let (mut store, mut instance) = init_kernel_test();
+        let instance = &mut instance;
+        for a in amounts {
+            let ptr = extism_alloc(&mut store, instance, a as u64);
+            if ptr == 0 || ptr == u64::MAX {
+                continue
+            }
+            if extism_length(&mut store, instance, ptr) != a as u64 {
+                return false
+            }
+        }
+
+        true
+    }
+}
+
+quickcheck! {
+    fn check_large_alloc(amounts: Vec<u32>) -> bool {
+        let (mut store, mut instance) = init_kernel_test();
+        let instance = &mut instance;
+        for a in amounts {
+            let ptr = extism_alloc(&mut store, instance, a as u64);
+            if ptr == 0 {
+                continue
+            }
+            let len = extism_length_unsafe(&mut store, instance, ptr);
+            if len != a as u64 {
+                return false
+            }
+        }
+
+        true
+    }
+}
+
+quickcheck! {
+    fn check_alloc_with_frees(amounts: Vec<u16>) -> bool {
+        let (mut store, mut instance) = init_kernel_test();
+        let instance = &mut instance;
+        let mut prev = 0;
+        for a in amounts {
+            let ptr = extism_alloc(&mut store, instance, a as u64);
+            if ptr == 0 {
+                continue
+            }
+            if extism_length(&mut store, instance, ptr) != a as u64  {
+                return false
+            }
+
+            if a % 2 == 0 {
+                extism_free(&mut store, instance, ptr);
+            } else if a % 3 == 0 {
+                extism_free(&mut store, instance, prev);
+            }
+
+            prev = ptr;
+        }
+
+        true
+    }
+}
+
+quickcheck! {
+    fn check_large_alloc_with_frees(amounts: Vec<u32>) -> bool {
+        let (mut store, mut instance) = init_kernel_test();
+        let instance = &mut instance;
+        let mut prev = 0;
+        for a in amounts {
+            let ptr = extism_alloc(&mut store, instance, a as u64);
+            if ptr == 0 || ptr == u64::MAX {
+                continue
+            }
+            if extism_length(&mut store, instance, ptr) != a as u64 {
+                return false
+            }
+            if a % 2 == 0 {
+                extism_free(&mut store, instance, ptr);
+            } else if a % 3 == 0 {
+                extism_free(&mut store, instance, prev);
+            }
+
+            prev = ptr;
+
+        }
+
+        true
+    }
+}
+
+quickcheck! {
+    fn check_alloc_with_load_and_store(amounts: Vec<u16>) -> bool {
+        use rand::Rng;
+        let mut rng = rand::thread_rng();
+        let (mut store, mut instance) = init_kernel_test();
+        let instance = &mut instance;
+        for a in amounts {
+            let ptr = extism_alloc(&mut store, instance, a as u64);
+            if ptr == 0 || ptr == u64::MAX {
+                continue
+            }
+            if extism_length(&mut store, instance, ptr) != a as u64 {
+                return false
+            }
+
+            for _ in 0..16 {
+                let i = rng.gen_range(ptr..ptr+a as u64);
+                extism_store_u8(&mut store, instance, i, i as u8);
+                if extism_load_u8(&mut store, instance, i as u64) != i as u8 {
+                    return false
+                }
+            }
+        }
+
+        true
+    }
+}

runtime/src/tests/runtime.rs

@@ -1,3 +1,5 @@
+use extism_manifest::MemoryOptions;
+
 use crate::*;
 use std::{io::Write, time::Instant};
 
@@ -39,11 +41,12 @@ pub struct Count {
 
 #[test]
 fn it_works() {
-    tracing_subscriber::fmt()
+    let log = tracing_subscriber::fmt()
         .with_ansi(false)
         .with_env_filter("extism=debug")
         .with_writer(std::fs::File::create("test.log").unwrap())
-        .init();
+        .try_init()
+        .is_ok();
 
     let wasm_start = Instant::now();
 
@@ -143,8 +146,10 @@ fn it_works() {
     println!("wasm function call (avg, N = {}): {:?}", num_tests, avg);
 
     // Check that log file was written to
-    let meta = std::fs::metadata("test.log").unwrap();
-    assert!(meta.len() > 0);
+    if log {
+        let meta = std::fs::metadata("test.log").unwrap();
+        assert!(meta.len() > 0);
+    }
 }
 
 #[test]
@@ -283,7 +288,7 @@ fn test_multiple_instantiations() {
 #[test]
 fn test_globals() {
     let mut plugin = Plugin::new(WASM_GLOBALS, [], true).unwrap();
-    for i in 0..1000 {
+    for i in 0..100000 {
         let Json(count) = plugin.call::<_, Json<Count>>("globals", "").unwrap();
         assert_eq!(count.count, i);
     }
@@ -448,7 +453,7 @@ fn hello_world_user_data(
     _plugin: &mut CurrentPlugin,
     inputs: &[Val],
     outputs: &mut [Val],
-    user_data: UserData<std::fs::File>,
+    user_data: UserData<std::sync::Arc<std::sync::Mutex<std::fs::File>>>,
 ) -> Result<(), Error> {
     let data = user_data.get()?;
     let mut data = data.lock().unwrap();
@@ -465,7 +470,8 @@ fn test_userdata() {
         if path.exists() {
             std::fs::remove_file(&path).unwrap();
         }
-        let file = std::fs::File::create(&path).unwrap();
+        let file =
+            std::sync::Arc::new(std::sync::Mutex::new(std::fs::File::create(&path).unwrap()));
         let f = Function::new(
             "hello_world",
             [PTR],
@@ -572,3 +578,112 @@ fn test_disable_cache() {
 
     assert!(t < t1);
 }
+
+#[test]
+fn test_manifest_ptr_len() {
+    let manifest = serde_json::json!({
+        "wasm" : [
+            {
+                "data" : {
+                    "ptr" : WASM_NO_FUNCTIONS.as_ptr() as u64,
+                    "len" : WASM_NO_FUNCTIONS.len()
+                }
+            }
+        ]
+    });
+    let mut plugin = Plugin::new(manifest.to_string().as_bytes(), [], true).unwrap();
+    let output = plugin.call("count_vowels", "abc123").unwrap();
+    let count: serde_json::Value = serde_json::from_slice(output).unwrap();
+    assert_eq!(count.get("count").unwrap().as_i64().unwrap(), 1);
+}
+
+#[test]
+fn test_no_vars() {
+    let data = br#"
+(module
+    (import "extism:host/env" "var_set" (func $var_set (param i64 i64)))
+    (import "extism:host/env" "input_offset" (func $input_offset (result i64)))
+    (func (export "test") (result i32)
+        (call $input_offset)
+        (call $input_offset)
+        (call $var_set)
+        (i32.const 0)
+    )
+)
+    "#;
+    let manifest = Manifest::new([Wasm::data(data)])
+        .with_memory_options(MemoryOptions::new().with_max_var_bytes(1));
+    let mut plugin = Plugin::new(manifest, [], true).unwrap();
+    let output: Result<(), Error> = plugin.call("test", b"A".repeat(1024));
+    assert!(output.is_err());
+    let output: Result<(), Error> = plugin.call("test", vec![]);
+    assert!(output.is_ok());
+}
+
+#[test]
+fn test_linking() {
+    let manifest = Manifest::new([
+        Wasm::Data {
+            data: br#"
+                (module
+                    (import "wasi_snapshot_preview1" "random_get" (func $random (param i32 i32) (result i32)))
+                    (import "extism:host/env" "alloc" (func $alloc (param i64) (result i64)))
+                    (import "extism:host/user" "hello" (func $hello))
+                    (global $counter (mut i32) (i32.const 0))
+                    (func $start (export "_start")
+                        (global.set $counter (i32.add (global.get $counter) (i32.const 1)))
+                    )
+                    (func (export "read_counter") (result i32)
+                        (global.get $counter)
+                    )
+                    (start $start)
+                )
+            "#.to_vec(),
+            meta: WasmMetadata {
+                name: Some("commander".to_string()),
+                hash: None,
+            },
+        },
+        Wasm::Data {
+            data: br#"
+                (module
+                    (import "commander" "_start" (func $commander_start))
+                    (import "commander" "read_counter" (func $commander_read_counter (result i32)))
+                    (import "extism:host/env" "store_u64" (func $store_u64 (param i64 i64)))
+                    (import "extism:host/env" "alloc" (func $alloc (param i64) (result i64)))
+                    (import "extism:host/user" "hello" (func $hello))
+                    (import "extism:host/env" "output_set" (func $output_set (param i64 i64)))
+                    (func (export "run") (result i32)
+                        (local $output i64)
+                        (local.set $output (call $alloc (i64.const 8)))
+
+                        (call $commander_start)
+                        (call $commander_start)
+                        (call $commander_start)
+                        (call $commander_start)
+                        (call $hello)
+                        (call $store_u64 (local.get $output) (i64.extend_i32_u (call $commander_read_counter)))
+                        (call $output_set (local.get $output) (i64.const 8))
+                        i32.const 0
+                    )
+                )
+            "#.to_vec(),
+            meta: WasmMetadata {
+                name: Some("main".to_string()),
+                hash: None,
+            },
+        },
+    ]);
+    let mut plugin = PluginBuilder::new(manifest)
+        .with_wasi(true)
+        .with_function("hello", [], [], UserData::new(()), |_, _, _, _| {
+            eprintln!("hello!");
+            Ok(())
+        })
+        .build()
+        .unwrap();
+
+    for _ in 0..5 {
+        assert_eq!(plugin.call::<&str, i64>("run", "Hello, world!").unwrap(), 1);
+    }
+}