fix(manifest): derive JsonSchema for LocalPath

As mentioned in
https://github.com/extism/extism/pull/696/files#r1993086470, the key
parameter doesnt seem very useful. You can achieve the same effect by
doing `HashMap<Key, PluginPool>` in the client code.

---------

Co-authored-by: zach <zach@dylibso.com>

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @zshipko

.github/dependabot.yml

@@ -9,33 +9,8 @@ updates:
     directory: "/" # Location of package manifests
     schedule:
       interval: "weekly"
+    ignore:
+      - dependency-name: "wasmtime"
+      - dependency-name: "wasi-common"
+      - dependency-name: "wiggle"
 
-  - package-ecosystem: "pip"
-    directory: "python"
-    schedule:
-      interval: "weekly"
-
-  - package-ecosystem: "mix"
-    directory: "elixir"
-    schedule:
-      interval: "weekly"
-
-  - package-ecosystem: "npm"
-    directory: "node"
-    schedule:
-      interval: "weekly"
-
-  - package-ecosystem: "composer"
-    directory: "/"
-    schedule:
-      interval: "weekly"
-
-  - package-ecosystem: "bundler"
-    directory: "ruby"
-    schedule:
-      interval: "weekly"
-
-  - package-ecosystem: "gomod"
-    directory: "/"
-    schedule:
-      interval: "weekly"

.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-on: 
+on:
   pull_request:
     paths:
       - .github/actions/extism/**
@@ -52,7 +52,7 @@ jobs:
         shell: bash
         run: cargo build --release -p ${{ env.LIBEXTISM_CRATE }}
       - name: Upload artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: libextism-${{ matrix.os }}
           path: |
@@ -116,4 +116,9 @@ jobs:
           path: target/**
           key: ${{ runner.os }}-target-${{ github.sha }}
       - run: cargo install cargo-criterion
-      - run: cargo criterion
+      - run:
+          cargo criterion
+      - run: |
+          git fetch
+          git checkout main
+          cargo criterion

.github/workflows/kernel.yml

@@ -21,6 +21,9 @@ jobs:
           target: wasm32-unknown-unknown
       - uses: Swatinem/rust-cache@v2
 
+      - name: install wasm-tools
+        uses: bytecodealliance/actions/wasm-tools/setup@v1
+
       - name: Install deps
         run: |
           sudo apt install wabt --yes
@@ -39,6 +42,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         if: ${{ env.GIT_EXIT_CODE }} != 0
         with:
+          author: "zshipko <zshipko@users.noreply.github.com>"
           title: "update(kernel): extism-runtime.wasm in ${{ github.event.pull_request.head.ref }}"
           body: "Automated PR to update `runtime/src/extism-runtime.wasm` in PR #${{ github.event.number }}"
           base: "${{ github.event.pull_request.head.ref }}"

.github/workflows/release-dotnet-native.yaml

@@ -1,4 +1,6 @@
 on:
+  release:
+    types: [published, edited]
   workflow_dispatch:
 
 name: Release .NET Native NuGet Packages
@@ -18,10 +20,13 @@ jobs:
         uses: actions/setup-dotnet@v3.0.3
         with:
            dotnet-version: 7.x
-      - uses: dawidd6/action-download-artifact@v2
-        with:
-          workflow: release.yml
-          name: release-artifacts
+      - name: download release
+        run: |
+          tag='${{ github.ref }}'
+          tag="${tag/refs\/tags\//}"
+          gh release download "$tag" -p 'libextism-*.tar.gz'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Extract Archive
         run: |      
           extract_archive() {

.github/workflows/release-rust.yaml

@@ -36,6 +36,18 @@ jobs:
           override: true
           target: ${{ matrix.target }}
 
+      - name: Release Rust convert-macros Crate
+        env:
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_API_TOKEN }}
+        run: |
+          version=$(cargo metadata --format-version=1 | jq -r '.packages[] | select(.name == "extism") | .version')
+
+          if ! &>/dev/null curl -sLIf https://crates.io/api/v1/crates/extism-convert-macros/${version}/download; then
+            cargo publish --manifest-path convert-macros/Cargo.toml --allow-dirty
+          else
+            echo "already published ${version}"
+          fi
+
       - name: Release Rust Convert Crate
         env:
           CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_API_TOKEN }}

.github/workflows/release.yml

@@ -7,11 +7,6 @@ on:
 
 name: Release
 
-env:
-  RUNTIME_MANIFEST: runtime/Cargo.toml
-  RUNTIME_CRATE: libextism
-  RUSTFLAGS: -C target-feature=-crt-static
-  ARTIFACT_DIR: release-artifacts
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -21,6 +16,11 @@ jobs:
   release:
     name: ${{ matrix.os }} ${{ matrix.target }}
     runs-on: ${{ matrix.os }}-latest
+    env:
+      RUNTIME_MANIFEST: runtime/Cargo.toml
+      RUNTIME_CRATE: libextism
+      RUSTFLAGS: -C target-feature=-crt-static
+      ARTIFACT_DIR: release-artifacts-${{ matrix.os }}-${{ matrix.target }}
     strategy:
       matrix:
         include:
@@ -28,48 +28,56 @@ jobs:
             target: 'x86_64-apple-darwin'
             artifact: 'libextism.dylib'
             static-artifact: 'libextism.a'
+            static-dll-artifact: ''
             pc-in: 'extism.pc.in'
             static-pc-in: 'extism-static.pc.in'
           - os: 'macos'
             target: 'aarch64-apple-darwin'
             artifact: 'libextism.dylib'
             static-artifact: 'libextism.a'
+            static-dll-artifact: ''
             pc-in: 'extism.pc.in'
             static-pc-in: 'extism-static.pc.in'
           - os: 'ubuntu'
             target: 'aarch64-unknown-linux-gnu'
             artifact: 'libextism.so'
             static-artifact: 'libextism.a'
+            static-dll-artifact: ''
             pc-in: 'extism.pc.in'
             static-pc-in: 'extism-static.pc.in'
           - os: 'ubuntu'
             target: 'aarch64-unknown-linux-musl'
             artifact: 'libextism.so'
             static-artifact: 'libextism.a'
+            static-dll-artifact: ''
             pc-in: 'extism.pc.in'
             static-pc-in: 'extism-static.pc.in'
           - os: 'ubuntu'
             target: 'x86_64-unknown-linux-gnu'
             artifact: 'libextism.so'
             static-artifact: 'libextism.a'
+            static-dll-artifact: ''
             pc-in: 'extism.pc.in'
             static-pc-in: 'extism-static.pc.in'
           - os: 'ubuntu'
             target: 'x86_64-unknown-linux-musl'
-            artifact: ''
+            artifact: 'libextism.so'
             static-artifact: 'libextism.a'
-            pc-in: ''
+            static-dll-artifact: ''
+            pc-in: 'extism.pc.in'
             static-pc-in: 'extism-static.pc.in'
           - os: 'windows'
             target: 'x86_64-pc-windows-gnu'
             artifact: 'extism.dll'
             static-artifact: 'libextism.a'
+            static-dll-artifact: 'libextism.dll.a'
             pc-in: 'extism.pc.in'
             static-pc-in: 'extism-static.pc.in'
           - os: 'windows'
             target: 'x86_64-pc-windows-msvc'
             artifact: 'extism.dll'
             static-artifact: 'extism.lib'
+            static-dll-artifact: 'extism.dll.lib'
             pc-in: ''
             static-pc-in: ''
 
@@ -90,13 +98,11 @@ jobs:
           pyproject="$(cat extism-maturin/pyproject.toml)"
           <<<"$pyproject" >extism-maturin/pyproject.toml sed -e 's/^version = "0.0.0.replaced-by-ci"/version = "'"$version"'"/g'
 
-      - name: Install Rust
-        uses: actions-rs/toolchain@v1
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
-          toolchain: stable
-          profile: minimal
-          override: true
           target: ${{ matrix.target }}
+          override: true
+          toolchain: stable
 
       - uses: Swatinem/rust-cache@v2
         with:
@@ -105,11 +111,15 @@ jobs:
           cache-on-failure: "true"
 
       - name: Build Target (${{ matrix.os }} ${{ matrix.target }})
-        uses: actions-rs/cargo@v1
-        with:
-          use-cross: ${{ matrix.os != 'windows' }}
-          command: build
-          args: --release --target ${{ matrix.target }} -p ${{ env.RUNTIME_CRATE }}
+        if: ${{ matrix.os != 'windows' }}
+        run: |
+          cargo install cross
+          cross build --release --target ${{ matrix.target }} -p ${{ env.RUNTIME_CRATE }}
+
+      - name: Build Target (${{ matrix.os }} ${{ matrix.target }})
+        if: ${{ matrix.os == 'windows' }}
+        run: |
+          cargo build --release --target ${{ matrix.target }} -p ${{ env.RUNTIME_CRATE }}
 
       - uses: actions/setup-python@v4
         with:
@@ -158,7 +168,8 @@ jobs:
           cp LICENSE ${SRC_DIR}
           tar -C ${SRC_DIR} -czvf ${ARCHIVE} extism.h \
             ${{ matrix.artifact }} ${{ matrix.static-artifact }}  \
-            ${{ matrix.pc-in }} ${{ matrix.static-pc-in }}
+            ${{ matrix.pc-in }} ${{ matrix.static-pc-in }} \
+            ${{ matrix.static-dll-artifact }}
           ls -ll ${ARCHIVE}
 
           if &>/dev/null which shasum; then
@@ -181,7 +192,7 @@ jobs:
           ls -ll ${DEST_DIR}
 
       - name: Upload Artifact to Summary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ env.ARTIFACT_DIR }}
           path: ${{ env.ARTIFACT_DIR }}
@@ -200,9 +211,10 @@ jobs:
     needs: [release]
     if: github.ref == 'refs/heads/main'
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
-          name: ${{ env.ARTIFACT_DIR }}
+          pattern: release-artifacts-*
+          merge-multiple: true
 
       - uses: "marvinpinto/action-automatic-releases@latest"
         with:

Cargo.toml

@@ -1,6 +1,6 @@
 [workspace]
 resolver = "2"
-members = ["extism-maturin", "manifest", "runtime", "libextism", "convert"]
+members = ["extism-maturin", "manifest", "runtime", "libextism", "convert", "convert-macros"]
 exclude = ["kernel"]
 
 [workspace.package]
@@ -14,4 +14,5 @@ version = "0.0.0+replaced-by-ci"
 [workspace.dependencies]
 extism = { path = "./runtime", version = "0.0.0+replaced-by-ci" }
 extism-convert = { path = "./convert", version = "0.0.0+replaced-by-ci" }
+extism-convert-macros = { path = "./convert-macros", version = "0.0.0+replaced-by-ci" }
 extism-manifest = { path = "./manifest", version = "0.0.0+replaced-by-ci" }

Makefile

@@ -4,10 +4,12 @@ AEXT=a
 FEATURES?=default
 DEFAULT_FEATURES?=yes
 RUST_TARGET?=
+EXTRA_LIBS=
 
 UNAME := $(shell uname -s)
 ifeq ($(UNAME),Darwin)
 	SOEXT=dylib
+	EXTRA_LIBS=-framework Security
 endif
 
 ifeq ($(DEFAULT_FEATURES),no)
@@ -29,7 +31,8 @@ endif
 build:
 	cargo build --release $(FEATURE_FLAGS) --manifest-path libextism/Cargo.toml $(TARGET_FLAGS)
 	sed -e "s%@CMAKE_INSTALL_PREFIX@%$(DEST)%" libextism/extism.pc.in > libextism/extism.pc
-	sed -e "s%@CMAKE_INSTALL_PREFIX@%$(DEST)%" libextism/extism-static.pc.in > libextism/extism-static.pc
+	sed -e "s%@CMAKE_INSTALL_PREFIX@%$(DEST)%" \
+	    -e "s%Libs: %Libs: $(EXTRA_LIBS) %" libextism/extism-static.pc.in > libextism/extism-static.pc
 
 bench:
 	@(cargo criterion $(TARGET_FLAGS) || echo 'For nicer output use cargo-criterion: `cargo install cargo-criterion` - using `cargo bench`') && cargo bench $(TARGET_FLAGS)

README.md

@@ -1,65 +1,188 @@
-# [Extism](https://extism.org)
+<div align="center">
+    <a href="https://extism.org">
+    <picture>
+        <source media="(prefers-color-scheme: dark)" srcset=".github/assets/logo-horizontal-darkmode.png">
+        <img alt="Extism - the WebAssembly framework" width="75%" style="max-width: 600px" src=".github/assets/logo-horizontal.png">
+    </picture>
+    </a>
 
-[![Discord](https://img.shields.io/discord/1011124058408112148?color=%23404eed&label=Community%20Chat&logo=Discord&logoColor=%23404eed)](https://discord.gg/cx3usBCWnc)
+[![Discord](https://img.shields.io/discord/1011124058408112148?color=%23404eed&label=Community%20Chat&logo=Discord&logoColor=%23404eed)](https://extism.org/discord)
 ![GitHub Org's stars](https://img.shields.io/github/stars/extism)
-![GitHub all releases](https://img.shields.io/github/downloads/extism/extism/total)
+![Downloads](https://img.shields.io/crates/d/extism-manifest)
 ![GitHub License](https://img.shields.io/github/license/extism/extism)
 ![GitHub release (with filter)](https://img.shields.io/github/v/release/extism/extism)
 
-The universal plug-in system. Run WebAssembly extensions inside your app. Use idiomatic Host SDKs for [Go](https://github.com/extism/go-sdk#readme),
-[Ruby](https://github.com/extism/ruby-sdk#readme),
-[Python](https://github.com/extism/python-sdk#readme),
-[JavaScript](https://github.com/extism/js-sdk#readme),
-[Rust](/runtime/#readme),
-[C](libextism/#readme),
-[C++](https://github.com/extism/cpp-sdk/#readme),
-[OCaml](https://github.com/extism/ocaml-sdk#readme),
-[Haskell](https://github.com/extism/haskell-sdk#readme),
-[PHP](https://github.com/extism/php-sdk#readme),
-[Elixir](https://github.com/extism/elixir-sdk#readme),
-[.NET](https://github.com/extism/dotnet-sdk#readme),
-[Java](https://github.com/extism/java-sdk#readme),
-[Zig](https://github.com/extism/zig-sdk#readme),
-[D](https://github.com/extism/d-sdk#readme),
-&amp; more (others coming soon).
+</div>
 
-Plug-in development kits (PDK) for plug-in authors supported in [Rust](https://github.com/extism/rust-pdk#readme), [AssemblyScript](https://github.com/extism/assemblyscript-pdk#readme), [Go](https://github.com/extism/go-pdk#readme), [C/C++](https://github.com/extism/c-pdk#readme), [Haskell](https://github.com/extism/haskell-pdk#readme), [JavaScript](https://github.com/extism/js-pdk#readme), [C#](https://github.com/extism/dotnet-pdk#readme), [F#](https://github.com/extism/dotnet-pdk#readme) and [Zig](https://github.com/extism/zig-pdk#readme).
+# Overview
 
-<p align="center">
-  <img style="width: 70%;" src="https://user-images.githubusercontent.com/7517515/210286900-39b144fd-1b26-4dd0-b7a9-2b5755bc174d.png" alt="Extism embedded SDK language support"/>
-</p>
+Extism is a lightweight framework for building with WebAssembly (Wasm). It
+supports running Wasm code on servers, the edge, CLIs, IoT, browsers and
+everything in between. Extism is designed to be "universal" in that it supports
+a common interface, no matter where it runs.
 
-Add a flexible, secure, and _bLaZiNg FaSt_ plug-in system to your project. Server, desktop, mobile, web, database -- you name it. Enable users to write and execute safe extensions to your software in **3 easy steps:**
+> **Note:** One of the primary use cases for Extism is **building extensible
+> software & plugins**. You want to be able to execute arbitrary, untrusted code
+> from your users? Extism makes this safe and practical to do.
 
-### 1. Import
+Additionally, Extism adds some extra utilities on top of standard Wasm runtimes.
+For example, we support persistent memory/module-scope variables, secure &
+host-controlled HTTP without WASI, runtime limiters & timers, simpler host
+function linking, and more. Extism users build:
 
-Import an Extism Host SDK into your code as a library dependency.
+- plug-in systems
+- FaaS platforms
+- code generators
+- web applications
+- & much more...
 
-### 2. Integrate 
+# Supported Targets
 
-Identify the place(s) in your code where some arbitrary logic should run (the plug-in!), returning your code some results.
+We currently provide releases for the following targets:
 
-### 3. Execute
+- aarch64-apple-darwin
+- aarch64-unknown-linux-gnu
+- aarch64-unknown-linux-musl
+- x86_64-apple-darwin
+- x86_64-pc-windows-gnu
+- x86_64-pc-windows-msvc
+- x86_64-unknown-linux-gnu
+- x86_64-unknown-linux-musl
 
-Load WebAssembly modules at any time in your app's lifetime and Extism will execute them in a secure sandbox, fully isolated from your program's memory.
+For Android we suggest taking a look at the [Chicory SDK](https://github.com/extism/chicory-sdk) for a pure Java
+Extism runtime.
 
-# API Status
+# Run WebAssembly In Your App
 
-**Please note:** This project still under active development and APIs are still changing. We are aiming for a stable 1.0 release in January, 2024.
-The main branch may have breaking changes until that point, but if you starting today, a 1.0.0-rcx release is the best place to start.
+Pick a SDK to import into your program, and refer to the documentation to get
+started:
 
-If you experience any problems or have any questions, please join our [Discord](https://discord.gg/cx3usBCWnc) and let us know.
-Our community is very responsive and happy to help get you started.
+| Type        | Language                                                                                       | Source Code                                                             | Package                                                                 |
+| ----------- | ---------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------- | ----------------------------------------------------------------------- |
+| Rust SDK    | <img alt="Rust SDK" src="https://extism.org/img/sdk-languages/rust.svg" width="50px"/>         | https://github.com/extism/extism/tree/main/runtime                      | [Crates.io](https://crates.io/crates/extism)                            |
+| JS SDK      | <img alt="JS SDK" src="https://extism.org/img/sdk-languages/js.svg" width="50px"/>             | https://github.com/extism/js-sdk <br/>(supports Web, Node, Deno & Bun!) | [NPM](https://www.npmjs.com/package/@extism/extism)                     |
+| Elixir SDK  | <img alt="Elixir SDK" src="https://extism.org/img/sdk-languages/elixir.svg" width="50px"/>     | https://github.com/extism/elixir-sdk                                    | [Hex](https://hex.pm/packages/extism)                                   |
+| Go SDK      | <img alt="Go SDK" src="https://extism.org/img/sdk-languages/go.svg" width="50px"/>             | https://github.com/extism/go-sdk                                        | [Go mod](https://pkg.go.dev/github.com/extism/go-sdk)                   |
+| Haskell SDK | <img alt="Haskell SDK" src="https://extism.org/img/sdk-languages/haskell.svg" width="50px"/>   | https://github.com/extism/haskell-sdk                                   | [Hackage](https://hackage.haskell.org/package/extism)                   |
+| Java SDK    | <img alt="Java SDK" src="https://extism.org/img/sdk-languages/java-android.svg" width="50px"/> | https://github.com/extism/java-sdk                                      | [Sonatype](https://central.sonatype.com/artifact/org.extism.sdk/extism) |
+| .NET SDK    | <img alt=".NET SDK" src="https://extism.org/img/sdk-languages/dotnet.svg" width="50px"/>       | https://github.com/extism/dotnet-sdk <br/>(supports C# & F#!)           | [Nuget](https://www.nuget.org/packages/Extism.Sdk)                      |
+| OCaml SDK   | <img alt="OCaml SDK" src="https://extism.org/img/sdk-languages/ocaml.svg" width="50px"/>       | https://github.com/extism/ocaml-sdk                                     | [opam](https://opam.ocaml.org/packages/extism/)                         |
+| Perl SDK    | <img alt="Perl SDK" src="https://extism.org/img/sdk-languages/perl.svg" width="50px"/>         | https://github.com/extism/perl-sdk                                      | [CPAN](https://metacpan.org/pod/Extism)                                 |
+| PHP SDK     | <img alt="PHP SDK" src="https://extism.org/img/sdk-languages/php.svg" width="50px"/>           | https://github.com/extism/php-sdk                                       | [Packagist](https://packagist.org/packages/extism/extism)               |
+| Python SDK  | <img alt="Python SDK" src="https://extism.org/img/sdk-languages/python.svg" width="50px"/>     | https://github.com/extism/python-sdk                                    | [PyPi](https://pypi.org/project/extism/)                                |
+| Ruby SDK    | <img alt="Ruby SDK" src="https://extism.org/img/sdk-languages/ruby.svg" width="50px"/>         | https://github.com/extism/ruby-sdk                                      | [RubyGems](https://rubygems.org/gems/extism)                            |
+| Zig SDK     | <img alt="Zig SDK" src="https://extism.org/img/sdk-languages/zig.svg" width="50px"/>           | https://github.com/extism/zig-sdk                                       | N/A                                                                     |
+| C SDK       | <img alt="C SDK" src="https://extism.org/img/sdk-languages/c.svg" width="50px"/>               | https://github.com/extism/extism/tree/main/libextism                    | N/A                                                                     |
+| C++ SDK     | <img alt="C++ SDK" src="https://extism.org/img/sdk-languages/cpp.svg" width="50px"/>           | https://github.com/extism/cpp-sdk                                       | N/A                                                                     |
+
+# Compile WebAssembly to run in Extism Hosts
+
+Extism Hosts (running the SDK) must execute WebAssembly code that has a
+[PDK, or Plug-in Development Kit](https://extism.org/docs/concepts/pdk), library
+compiled in to the `.wasm` binary. PDKs make it easy for plug-in / extension
+code authors to read input from the host and return data back, read provided
+configuration, set/get variables, make outbound HTTP calls if allowed, and more.
+
+Pick a PDK to import into your Wasm program, and refer to the documentation to
+get started:
+
+| Type               | Language                                                                                                   | Source Code                                                   | Package                                                   |
+| ------------------ | ---------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------- | --------------------------------------------------------- |
+| Rust PDK           | <img alt="Rust PDK" src="https://extism.org/img/sdk-languages/rust.svg" width="50px"/>                     | https://github.com/extism/rust-pdk                            | [Crates.io](https://crates.io/crates/extism-pdk)          |
+| JS PDK             | <img alt="JS PDK" src="https://extism.org/img/sdk-languages/js.svg" width="50px"/>                         | https://github.com/extism/js-pdk                              | N/A                                                       |
+| Python PDK         | <img alt="Python PDK" src="https://extism.org/img/sdk-languages/python.svg" width="50px"/>                 | https://github.com/extism/python-pdk                          | N/A                                                       |
+| Go PDK             | <img alt="Go PDK" src="https://extism.org/img/sdk-languages/go.svg" width="50px"/>                         | https://github.com/extism/go-pdk                              | [Go mod](https://pkg.go.dev/github.com/extism/go-pdk)     |
+| Haskell PDK        | <img alt="Haskell PDK" src="https://extism.org/img/sdk-languages/haskell.svg" width="50px"/>               | https://github.com/extism/haskell-pdk                         | [Hackage](https://hackage.haskell.org/package/extism-pdk) |
+| AssemblyScript PDK | <img alt="AssemblyScript PDK" src="https://extism.org/img/sdk-languages/assemblyscript.svg" width="50px"/> | https://github.com/extism/assemblyscript-pdk                  | [NPM](https://www.npmjs.com/package/@extism/as-pdk)       |
+| .NET PDK           | <img alt=".NET PDK" src="https://extism.org/img/sdk-languages/dotnet.svg" width="50px"/>                   | https://github.com/extism/dotnet-pdk <br/>(supports C# & F#!) | [Nuget](https://www.nuget.org/packages/Extism.Pdk)        |
+| C PDK              | <img alt="C PDK" src="https://extism.org/img/sdk-languages/c.svg" width="50px"/>                           | https://github.com/extism/c-pdk                               | N/A                                                       |
+| C++ PDK            | <img alt="C++ PDK" src="https://extism.org/img/sdk-languages/cpp.svg" width="50px"/>                       | https://github.com/extism/cpp-pdk                             | N/A                                                       |
+| Zig PDK            | <img alt="Zig PDK" src="https://extism.org/img/sdk-languages/zig.svg" width="50px"/>                       | https://github.com/extism/zig-pdk                             | N/A                                                       |
+
+# Generating Bindings
+
+It's often very useful to define a schema to describe the function signatures
+and types you want to use between Extism SDK and PDK languages.
+
+[XTP Bindgen](https://github.com/dylibso/xtp-bindgen) is an open source
+framework to generate PDK bindings for Extism plug-ins. It's used by the
+[XTP Platform](https://www.getxtp.com/), but can be used outside of the platform
+to define any Extism compatible plug-in system.
+
+## 1. Install the `xtp` CLI.
+
+See installation instructions
+[here](https://docs.xtp.dylibso.com/docs/cli#installation).
+
+## 2. Create a schema using our OpenAPI-inspired IDL:
+
+```yaml
+version: v1-draft
+exports: 
+  CountVowels:
+      input: 
+          type: string
+          contentType: text/plain; charset=utf-8
+      output:
+          $ref: "#/components/schemas/VowelReport"
+          contentType: application/json
+# components.schemas defined in example-schema.yaml...
+```
+
+> See an example in [example-schema.yaml](./example-schema.yaml), or a full
+> "kitchen sink" example on
+> [the docs page](https://docs.xtp.dylibso.com/docs/concepts/xtp-schema/).
+
+## 3. Generate bindings to use from your plugins:
+
+```
+xtp plugin init --schema-file ./example-schema.yaml
+  > 1. TypeScript                      
+    2. Go                              
+    3. Rust                            
+    4. Python                          
+    5. C#                              
+    6. Zig                             
+    7. C++                             
+    8. GitHub Template                 
+    9. Local Template
+```
+
+This will create an entire boilerplate plugin project for you to get started
+with. Implement the empty function(s), and run `xtp plugin build` to compile
+your plugin.
+
+> For more information about XTP Bindgen, see the
+> [dylibso/xtp-bindgen](https://github.com/dylibso/xtp-bindgen) repository and
+> the official
+> [XTP Schema documentation](https://docs.xtp.dylibso.com/docs/concepts/xtp-schema).
+
+# Support
+
+## Discord
+
+If you experience any problems or have any questions, please join our
+[Discord](https://extism.org/discord) and let us know. Our community is very
+responsive and happy to help get you started.
 
 ## Usage
 
-Head to the [project website](https://extism.org) for more information and docs. Also, consider reading an [overview](https://extism.org/docs/overview) of Extism and its goals & approach.
+Head to the [project website](https://extism.org) for more information and docs.
+Also, consider reading an [overview](https://extism.org/docs/overview) of Extism
+and its goals & approach.
 
 ## Contribution
 
-Thank you for considering a contribution to Extism, we are happy to help you make a PR or find something to work on! 
+Thank you for considering a contribution to Extism, we are happy to help you
+make a PR or find something to work on!
 
-The easiest way to start would be to join the [Discord](https://discord.gg/cx3usBCWnc) or open an issue on the [`extism/proposals`](https://github.com/extism/proposals) issue tracker, which can eventually become an Extism Improvement Proposal (EIP).
+The easiest way to start would be to join the
+[Discord](https://extism.org/discord) or open an issue on the
+[`extism/proposals`](https://github.com/extism/proposals) issue tracker, which
+can eventually become an Extism Improvement Proposal (EIP).
+
+For more information, please read the
+[Contributing](https://extism.org/docs/concepts/contributing) guide.
 
 ---
 
@@ -68,8 +191,8 @@ The easiest way to start would be to join the [Discord](https://discord.gg/cx3us
 Extism is an open-source product from the team at:
 
 <p align="left">
-  <a href="https://dylib.so" _target="blanks"><img width="200px" src="https://user-images.githubusercontent.com/7517515/198204119-5afdebb9-a5d8-4322-bd2a-46179c8d7b24.svg"/></a>
+  <a href="https://dylibso.com" _target="blanks"><img width="200px" src="https://user-images.githubusercontent.com/7517515/198204119-5afdebb9-a5d8-4322-bd2a-46179c8d7b24.svg"/></a>
 </p>
 
-
-_Reach out and tell us what you're building! We'd love to help._
+_Reach out and tell us what you're building! We'd love to help:_
+<a href="mailto:hello@dylibso.com">hello@dylibso.com</a>

convert-macros/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "extism-convert-macros"
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+homepage.workspace = true
+repository.workspace = true
+version.workspace = true
+description = "Macros to remove boilerplate with Extism"
+
+[lib]
+proc-macro = true
+
+[features]
+extism-path = []
+extism-pdk-path = []
+
+[dependencies]
+manyhow.version = "0.11.0"
+proc-macro-crate = "3.1.0"
+proc-macro2 = "1.0.78"
+quote = "1.0.35"
+syn = { version = "2.0.48", features = ["derive"] }
+
+[dev-dependencies]
+trybuild = "1.0.89"

convert-macros/src/lib.rs

@@ -0,0 +1,108 @@
+use std::iter;
+
+use manyhow::{ensure, error_message, manyhow, Result};
+use proc_macro_crate::{crate_name, FoundCrate};
+use quote::{format_ident, quote, ToTokens};
+use syn::{parse_quote, Attribute, DeriveInput, Path};
+
+/// Tries to resolve the path to `extism_convert` dynamically, falling back to feature flags when unsuccessful.
+fn convert_path() -> Path {
+    match (
+        crate_name("extism"),
+        crate_name("extism-convert"),
+        crate_name("extism-pdk"),
+    ) {
+        (Ok(FoundCrate::Name(name)), ..) => {
+            let ident = format_ident!("{name}");
+            parse_quote!(::#ident::convert)
+        }
+        (_, Ok(FoundCrate::Name(name)), ..) | (.., Ok(FoundCrate::Name(name))) => {
+            let ident = format_ident!("{name}");
+            parse_quote!(::#ident)
+        }
+        (Ok(FoundCrate::Itself), ..) => parse_quote!(::extism::convert),
+        (_, Ok(FoundCrate::Itself), ..) => parse_quote!(::extism_convert),
+        (.., Ok(FoundCrate::Itself)) => parse_quote!(::extism_pdk),
+        _ if cfg!(feature = "extism-path") => parse_quote!(::extism::convert),
+        _ if cfg!(feature = "extism-pdk-path") => parse_quote!(::extism_pdk),
+        _ => parse_quote!(::extism_convert),
+    }
+}
+
+fn extract_encoding(attrs: &[Attribute]) -> Result<Path> {
+    let encodings: Vec<_> = attrs
+        .iter()
+        .filter(|attr| attr.path().is_ident("encoding"))
+        .collect();
+    ensure!(!encodings.is_empty(), "encoding needs to be specified"; try = "`#[encoding(Json)]`");
+    ensure!(encodings.len() < 2, encodings[1], "only one encoding can be specified"; try = "remove `{}`", encodings[1].to_token_stream());
+
+    Ok(encodings[0].parse_args().map_err(
+        |e| error_message!(e.span(), "{e}"; note= "expects a path"; try = "`#[encoding(Json)]`"),
+    )?)
+}
+
+#[manyhow]
+#[proc_macro_derive(ToBytes, attributes(encoding))]
+pub fn to_bytes(
+    DeriveInput {
+        attrs,
+        ident,
+        generics,
+        ..
+    }: DeriveInput,
+) -> Result {
+    let encoding = extract_encoding(&attrs)?;
+    let convert = convert_path();
+
+    let (_, type_generics, _) = generics.split_for_impl();
+
+    let mut generics = generics.clone();
+    generics.make_where_clause().predicates.push(
+        parse_quote!(for<'__to_bytes_b> #encoding<&'__to_bytes_b Self>: #convert::ToBytes<'__to_bytes_b>)
+    );
+    generics.params = iter::once(parse_quote!('__to_bytes_a))
+        .chain(generics.params)
+        .collect();
+    let (impl_generics, _, where_clause) = generics.split_for_impl();
+
+    Ok(quote! {
+        impl #impl_generics #convert::ToBytes<'__to_bytes_a> for #ident #type_generics #where_clause
+        {
+            type Bytes = ::std::vec::Vec<u8>;
+
+            fn to_bytes(&self) -> Result<Self::Bytes, #convert::Error> {
+                #convert::ToBytes::to_bytes(&#encoding(self)).map(|__bytes| __bytes.as_ref().to_vec())
+            }
+        }
+
+    })
+}
+
+#[manyhow]
+#[proc_macro_derive(FromBytes, attributes(encoding))]
+pub fn from_bytes(
+    DeriveInput {
+        attrs,
+        ident,
+        mut generics,
+        ..
+    }: DeriveInput,
+) -> Result {
+    let encoding = extract_encoding(&attrs)?;
+    let convert = convert_path();
+    generics
+        .make_where_clause()
+        .predicates
+        .push(parse_quote!(#encoding<Self>: #convert::FromBytesOwned));
+    let (impl_generics, type_generics, where_clause) = generics.split_for_impl();
+    Ok(quote! {
+        impl #impl_generics #convert::FromBytesOwned for #ident #type_generics #where_clause
+        {
+            fn from_bytes_owned(__data: &[u8]) -> Result<Self, #convert::Error> {
+                <#encoding<Self> as #convert::FromBytesOwned>::from_bytes_owned(__data).map(|__encoding| __encoding.0)
+            }
+        }
+
+    })
+}

convert-macros/tests/ui.rs

@@ -0,0 +1,5 @@
+#[test]
+fn ui() {
+    let t = trybuild::TestCases::new();
+    t.compile_fail("tests/ui/*.rs");
+}

convert-macros/tests/ui/invalid-encoding.rs

@@ -0,0 +1,23 @@
+use extism_convert_macros::ToBytes;
+
+#[derive(ToBytes)]
+struct MissingEncoding;
+
+#[derive(ToBytes)]
+#[encoding]
+struct EmptyAttr;
+
+#[derive(ToBytes)]
+#[encoding = "string"]
+struct EqNoParen;
+
+#[derive(ToBytes)]
+#[encoding(something, else)]
+struct NotAPath;
+
+#[derive(ToBytes)]
+#[encoding(Multiple)]
+#[encoding(Encodings)]
+struct MultipleEncodings;
+
+fn main() {}

convert-macros/tests/ui/invalid-encoding.stderr

@@ -0,0 +1,44 @@
+error: encoding needs to be specified
+
+         = try: `#[encoding(Json)]`
+ --> tests/ui/invalid-encoding.rs:3:10
+  |
+3 | #[derive(ToBytes)]
+  |          ^^^^^^^
+  |
+  = note: this error originates in the derive macro `ToBytes` (in Nightly builds, run with -Z macro-backtrace for more info)
+
+error: expected attribute arguments in parentheses: #[encoding(...)]
+
+         = note: expects a path
+         = try: `#[encoding(Json)]`
+ --> tests/ui/invalid-encoding.rs:7:3
+  |
+7 | #[encoding]
+  |   ^^^^^^^^
+
+error: expected parentheses: #[encoding(...)]
+
+         = note: expects a path
+         = try: `#[encoding(Json)]`
+  --> tests/ui/invalid-encoding.rs:11:12
+   |
+11 | #[encoding = "string"]
+   |            ^
+
+error: unexpected token
+
+         = note: expects a path
+         = try: `#[encoding(Json)]`
+  --> tests/ui/invalid-encoding.rs:15:21
+   |
+15 | #[encoding(something, else)]
+   |                     ^
+
+error: only one encoding can be specified
+
+         = try: remove `#[encoding(Encodings)]`
+  --> tests/ui/invalid-encoding.rs:20:1
+   |
+20 | #[encoding(Encodings)]
+   | ^^^^^^^^^^^^^^^^^^^^^^

convert/Cargo.toml

@@ -11,18 +11,21 @@ description = "Traits to make Rust types usable with Extism"
 
 [dependencies]
 anyhow = "1.0.75"
-base64 = "~0.21"
+base64 = "~0.22"
 bytemuck = {version = "1.14.0", optional = true }
-prost = { version = "0.12.0", optional = true }
+prost = { version = "0.13.1", optional = true }
+protobuf = { version = "3.2.0", optional = true }
 rmp-serde = { version = "1.1.2", optional = true }
 serde = "1.0.186"
 serde_json = "1.0.105"
+extism-convert-macros.workspace = true
 
 [dev-dependencies]
 serde = { version = "1.0.186", features = ["derive"] }
 
 [features]
-default = ["msgpack", "protobuf", "raw"]
+default = ["msgpack", "prost", "raw"]
 msgpack = ["rmp-serde"]
-protobuf = ["prost"]
 raw = ["bytemuck"]
+extism-path = ["extism-convert-macros/extism-path"]
+extism-pdk-path = ["extism-convert-macros/extism-pdk-path"]

convert/src/encoding.rs

@@ -11,8 +11,8 @@ use base64::Engine;
 /// extism_convert::encoding!(MyJson, serde_json::to_vec, serde_json::from_slice);
 /// ```
 ///
-/// This will create a struct `struct MyJson<T>(pub T)` and implement `ToBytes` using `serde_json::to_vec`
-/// and `FromBytesOwned` using `serde_json::from_vec`
+/// This will create a struct `struct MyJson<T>(pub T)` and implement [`ToBytes`] using [`serde_json::to_vec`]
+/// and [`FromBytesOwned`] using [`serde_json::from_slice`]
 #[macro_export]
 macro_rules! encoding {
     ($pub:vis $name:ident, $to_vec:expr, $from_slice:expr) => {
@@ -55,7 +55,7 @@ encoding!(pub Json, serde_json::to_vec, serde_json::from_slice);
 #[cfg(feature = "msgpack")]
 encoding!(pub Msgpack, rmp_serde::to_vec, rmp_serde::from_slice);
 
-impl<'a> ToBytes<'a> for serde_json::Value {
+impl ToBytes<'_> for serde_json::Value {
     type Bytes = Vec<u8>;
 
     fn to_bytes(&self) -> Result<Self::Bytes, Error> {
@@ -85,7 +85,7 @@ impl<T: AsRef<[u8]>> From<T> for Base64<T> {
     }
 }
 
-impl<'a, T: AsRef<[u8]>> ToBytes<'a> for Base64<T> {
+impl<T: AsRef<[u8]>> ToBytes<'_> for Base64<T> {
     type Bytes = String;
 
     fn to_bytes(&self) -> Result<Self::Bytes, Error> {
@@ -112,19 +112,19 @@ impl FromBytesOwned for Base64<String> {
 /// Protobuf encoding
 ///
 /// Allows for `prost` Protobuf messages to be used as arguments to Extism plugin calls
-#[cfg(feature = "protobuf")]
+#[cfg(feature = "prost")]
 #[derive(Debug)]
-pub struct Protobuf<T: prost::Message>(pub T);
+pub struct Prost<T: prost::Message>(pub T);
 
-#[cfg(feature = "protobuf")]
-impl<T: prost::Message> From<T> for Protobuf<T> {
+#[cfg(feature = "prost")]
+impl<T: prost::Message> From<T> for Prost<T> {
     fn from(data: T) -> Self {
         Self(data)
     }
 }
 
-#[cfg(feature = "protobuf")]
-impl<'a, T: prost::Message> ToBytes<'a> for Protobuf<T> {
+#[cfg(feature = "prost")]
+impl<T: prost::Message> ToBytes<'_> for Prost<T> {
     type Bytes = Vec<u8>;
 
     fn to_bytes(&self) -> Result<Self::Bytes, Error> {
@@ -132,10 +132,32 @@ impl<'a, T: prost::Message> ToBytes<'a> for Protobuf<T> {
     }
 }
 
-#[cfg(feature = "protobuf")]
-impl<T: Default + prost::Message> FromBytesOwned for Protobuf<T> {
+#[cfg(feature = "prost")]
+impl<T: Default + prost::Message> FromBytesOwned for Prost<T> {
     fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
-        Ok(Protobuf(T::decode(data)?))
+        Ok(Prost(T::decode(data)?))
+    }
+}
+
+/// Protobuf encoding
+///
+/// Allows for `rust-protobuf` Protobuf messages to be used as arguments to Extism plugin calls
+#[cfg(feature = "protobuf")]
+pub struct Protobuf<T: protobuf::Message>(pub T);
+
+#[cfg(feature = "protobuf")]
+impl<T: protobuf::Message> ToBytes<'_> for Protobuf<T> {
+    type Bytes = Vec<u8>;
+
+    fn to_bytes(&self) -> Result<Self::Bytes, Error> {
+        Ok(self.0.write_to_bytes()?)
+    }
+}
+
+#[cfg(feature = "protobuf")]
+impl<T: Default + protobuf::Message> FromBytesOwned for Protobuf<T> {
+    fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
+        Ok(Protobuf(T::parse_from_bytes(data)?))
     }
 }
 
@@ -161,30 +183,5 @@ impl<'a, T: bytemuck::Pod> FromBytes<'a> for Raw<'a, T> {
     }
 }
 
-#[cfg(all(test, feature = "raw", target_endian = "little"))]
-mod tests {
-    use crate::*;
-
-    #[test]
-    fn test_raw() {
-        #[derive(Debug, Clone, Copy, PartialEq)]
-        struct TestRaw {
-            a: i32,
-            b: f64,
-            c: bool,
-        }
-        unsafe impl bytemuck::Pod for TestRaw {}
-        unsafe impl bytemuck::Zeroable for TestRaw {}
-        let x = TestRaw {
-            a: 123,
-            b: 45678.91011,
-            c: true,
-        };
-        let raw = Raw(&x).to_bytes().unwrap();
-        let y = Raw::from_bytes(&raw).unwrap();
-        assert_eq!(&x, y.0);
-
-        let y: Result<Raw<[u8; std::mem::size_of::<TestRaw>()]>, Error> = Raw::from_bytes(&raw);
-        assert!(y.is_ok());
-    }
-}
+#[cfg(all(feature = "raw", target_endian = "big"))]
+compile_error!("The raw feature is only supported on little endian targets");

convert/src/from_bytes.rs

@@ -1,14 +1,68 @@
 use crate::*;
 
+pub use extism_convert_macros::FromBytes;
+
 /// `FromBytes` is used to define how a type should be decoded when working with
 /// Extism memory. It is used for plugin output and host function input.
+///
+/// `FromBytes` can be derived by delegating encoding to generic type implementing
+/// `FromBytes`, e.g., [`Json`], [`Msgpack`].
+///
+/// ```
+/// use extism_convert::{Json, FromBytes};
+/// use serde::Deserialize;
+///
+/// #[derive(FromBytes, Deserialize, PartialEq, Debug)]
+/// #[encoding(Json)]
+/// struct Struct {
+///     hello: String,
+/// }
+///
+/// assert_eq!(Struct::from_bytes(br#"{"hello":"hi"}"#)?, Struct { hello: "hi".into() });
+/// # Ok::<(), extism_convert::Error>(())
+/// ```
+///
+/// Custom encodings can also be used, through new-types with a single generic
+/// argument, i.e., `Type<T>(T)`, that implement `FromBytesOwned` for the struct.
+///
+/// ```
+/// use std::str::{self, FromStr};
+/// use std::convert::Infallible;
+/// use extism_convert::{Error, FromBytes, FromBytesOwned};
+///
+/// // Custom serialization using `FromStr`
+/// struct StringEnc<T>(T);
+/// impl<T: FromStr> FromBytesOwned for StringEnc<T> where Error: From<<T as FromStr>::Err> {
+///     fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
+///         Ok(Self(str::from_utf8(data)?.parse()?))
+///     }
+/// }
+///
+/// #[derive(FromBytes, PartialEq, Debug)]
+/// #[encoding(StringEnc)]
+/// struct Struct {
+///     hello: String,
+/// }
+///
+/// impl FromStr for Struct {
+///     type Err = Infallible;
+///     fn from_str(s: &str) -> Result<Self, Infallible> {
+///         Ok(Self { hello: s.to_owned() })
+///     }
+/// }
+///
+/// assert_eq!(Struct::from_bytes(b"hi")?, Struct { hello: "hi".into() });
+/// # Ok::<(), extism_convert::Error>(())
+/// ```
 pub trait FromBytes<'a>: Sized {
     /// Decode a value from a slice of bytes
     fn from_bytes(data: &'a [u8]) -> Result<Self, Error>;
 }
 
-/// `FromBytesOwned` is similar to `FromBytes` but it doesn't borrow from the input slice.
-/// `FromBytes` is automatically implemented for all types that implement `FromBytesOwned`
+/// `FromBytesOwned` is similar to [`FromBytes`] but it doesn't borrow from the input slice.
+/// [`FromBytes`] is automatically implemented for all types that implement `FromBytesOwned`.
+///
+/// `FromBytesOwned` can be derived through [`#[derive(FromBytes)]`](FromBytes).
 pub trait FromBytesOwned: Sized {
     /// Decode a value from a slice of bytes, the resulting value should not borrow the input
     /// data.
@@ -87,6 +141,16 @@ impl FromBytesOwned for u32 {
     }
 }
 
+impl FromBytesOwned for bool {
+    fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
+        if let Some(x) = data.first() {
+            Ok(*x != 0)
+        } else {
+            Err(Error::msg("Expected one byte to read boolean value"))
+        }
+    }
+}
+
 impl FromBytesOwned for () {
     fn from_bytes_owned(_: &[u8]) -> Result<Self, Error> {
         Ok(())
@@ -98,3 +162,13 @@ impl<'a, T: FromBytes<'a>> FromBytes<'a> for std::io::Cursor<T> {
         Ok(std::io::Cursor::new(T::from_bytes(data)?))
     }
 }
+
+impl<'a, T: FromBytes<'a>> FromBytes<'a> for Option<T> {
+    fn from_bytes(data: &'a [u8]) -> Result<Self, Error> {
+        if data.is_empty() {
+            return Ok(None);
+        }
+
+        T::from_bytes(data).map(Some)
+    }
+}

convert/src/lib.rs

@@ -5,6 +5,9 @@
 //! similar to [axum extractors](https://docs.rs/axum/latest/axum/extract/index.html#intro) - they are
 //! implemented as a tuple struct with a single field that is meant to be extracted using pattern matching.
 
+// Makes proc-macros able to resolve `::extism_convert` correctly
+extern crate self as extism_convert;
+
 pub use anyhow::Error;
 
 mod encoding;
@@ -18,6 +21,9 @@ pub use encoding::{Base64, Json};
 #[cfg(feature = "msgpack")]
 pub use encoding::Msgpack;
 
+#[cfg(feature = "prost")]
+pub use encoding::Prost;
+
 #[cfg(feature = "protobuf")]
 pub use encoding::Protobuf;
 

convert/src/tests.rs

@@ -20,6 +20,7 @@ fn roundtrip_json() {
 }
 
 #[test]
+#[cfg(feature = "msgpack")]
 fn roundtrip_msgpack() {
     let x = Testing {
         a: "foobar".to_string(),
@@ -37,3 +38,64 @@ fn roundtrip_base64() {
     let Base64(s): Base64<String> = FromBytes::from_bytes(bytes.as_bytes()).unwrap();
     assert_eq!(s, "this is a test");
 }
+
+#[test]
+fn rountrip_option() {
+    // `None` case
+    let e0: Option<Json<Testing>> = FromBytes::from_bytes(&[]).unwrap();
+    let b = e0.to_bytes().unwrap();
+    let e1: Option<Json<Testing>> = FromBytes::from_bytes(&b).unwrap();
+    assert!(e0.is_none());
+    assert_eq!(e0.is_none(), e1.is_none());
+
+    // `Some` case
+    let x = Testing {
+        a: "foobar".to_string(),
+        b: 123,
+        c: 456.7,
+    };
+    let bytes = Json(&x).to_bytes().unwrap();
+    let y: Option<Json<Testing>> = FromBytes::from_bytes(&bytes).unwrap();
+    let b = ToBytes::to_bytes(&y).unwrap();
+    let z: Option<Json<Testing>> = FromBytes::from_bytes(&b).unwrap();
+    assert_eq!(y.unwrap().0, z.unwrap().0);
+}
+
+#[test]
+fn check_bool() {
+    // `None` case
+    let a = true.to_bytes().unwrap();
+    let b = false.to_bytes().unwrap();
+    assert_ne!(a, b);
+
+    assert_eq!(a, [1]);
+    assert_eq!(b, [0]);
+}
+
+#[cfg(all(feature = "raw", target_endian = "little"))]
+mod raw_tests {
+    use crate::*;
+
+    #[test]
+    fn test_raw() {
+        #[derive(Debug, Clone, Copy, PartialEq)]
+        struct TestRaw {
+            a: i32,
+            b: f64,
+            c: bool,
+        }
+        unsafe impl bytemuck::Pod for TestRaw {}
+        unsafe impl bytemuck::Zeroable for TestRaw {}
+        let x = TestRaw {
+            a: 123,
+            b: 45678.91011,
+            c: true,
+        };
+        let raw = Raw(&x).to_bytes().unwrap();
+        let y = Raw::from_bytes(raw).unwrap();
+        assert_eq!(&x, y.0);
+
+        let y: Result<Raw<[u8; std::mem::size_of::<TestRaw>()]>, Error> = Raw::from_bytes(raw);
+        assert!(y.is_ok());
+    }
+}

convert/src/to_bytes.rs

@@ -1,7 +1,58 @@
 use crate::*;
 
+pub use extism_convert_macros::ToBytes;
+
 /// `ToBytes` is used to define how a type should be encoded when working with
 /// Extism memory. It is used for plugin input and host function output.
+///
+/// `ToBytes` can be derived by delegating encoding to generic type implementing
+/// `ToBytes`, e.g., [`Json`], [`Msgpack`].
+///
+/// ```
+/// use extism_convert::{Json, ToBytes};
+/// use serde::Serialize;
+///
+/// #[derive(ToBytes, Serialize)]
+/// #[encoding(Json)]
+/// struct Struct {
+///     hello: String,
+/// }
+///
+/// assert_eq!(Struct { hello: "hi".into() }.to_bytes()?, br#"{"hello":"hi"}"#);
+/// # Ok::<(), extism_convert::Error>(())
+/// ```
+///
+/// But custom types can also be used, as long as they are new-types with a single
+/// generic argument, i.e., `Type<T>(T)`, that implement `ToBytes` for the struct.
+///
+/// ```
+/// use extism_convert::{Error, ToBytes};
+///
+/// // Custom serialization using `ToString`
+/// struct StringEnc<T>(T);
+/// impl<T: ToString> ToBytes<'_> for StringEnc<&T> {
+///     type Bytes = String;
+///     
+///     fn to_bytes(&self) -> Result<String, Error> {
+///         Ok(self.0.to_string())
+///     }
+/// }
+///
+/// #[derive(ToBytes)]
+/// #[encoding(StringEnc)]
+/// struct Struct {
+///     hello: String,
+/// }
+///
+/// impl ToString for Struct {
+///    fn to_string(&self) -> String {
+///        self.hello.clone()     
+///    }
+/// }
+///
+/// assert_eq!(Struct { hello: "hi".into() }.to_bytes()?, b"hi");
+/// # Ok::<(), Error>(())
+/// ```
 pub trait ToBytes<'a> {
     /// A configurable byte slice representation, allows any type that implements `AsRef<[u8]>`
     type Bytes: AsRef<[u8]>;
@@ -10,21 +61,21 @@ pub trait ToBytes<'a> {
     fn to_bytes(&self) -> Result<Self::Bytes, Error>;
 }
 
-impl<'a> ToBytes<'a> for () {
+impl ToBytes<'_> for () {
     type Bytes = [u8; 0];
     fn to_bytes(&self) -> Result<Self::Bytes, Error> {
         Ok([])
     }
 }
 
-impl<'a> ToBytes<'a> for Vec<u8> {
+impl ToBytes<'_> for Vec<u8> {
     type Bytes = Vec<u8>;
     fn to_bytes(&self) -> Result<Self::Bytes, Error> {
         Ok(self.clone())
     }
 }
 
-impl<'a> ToBytes<'a> for String {
+impl ToBytes<'_> for String {
     type Bytes = String;
     fn to_bytes(&self) -> Result<Self::Bytes, Error> {
         Ok(self.clone())
@@ -45,7 +96,7 @@ impl<'a> ToBytes<'a> for &'a str {
     }
 }
 
-impl<'a> ToBytes<'a> for f64 {
+impl ToBytes<'_> for f64 {
     type Bytes = [u8; 8];
 
     fn to_bytes(&self) -> Result<Self::Bytes, Error> {
@@ -53,7 +104,7 @@ impl<'a> ToBytes<'a> for f64 {
     }
 }
 
-impl<'a> ToBytes<'a> for f32 {
+impl ToBytes<'_> for f32 {
     type Bytes = [u8; 4];
 
     fn to_bytes(&self) -> Result<Self::Bytes, Error> {
@@ -61,7 +112,7 @@ impl<'a> ToBytes<'a> for f32 {
     }
 }
 
-impl<'a> ToBytes<'a> for i64 {
+impl ToBytes<'_> for i64 {
     type Bytes = [u8; 8];
 
     fn to_bytes(&self) -> Result<Self::Bytes, Error> {
@@ -69,7 +120,7 @@ impl<'a> ToBytes<'a> for i64 {
     }
 }
 
-impl<'a> ToBytes<'a> for i32 {
+impl ToBytes<'_> for i32 {
     type Bytes = [u8; 4];
 
     fn to_bytes(&self) -> Result<Self::Bytes, Error> {
@@ -77,7 +128,7 @@ impl<'a> ToBytes<'a> for i32 {
     }
 }
 
-impl<'a> ToBytes<'a> for u64 {
+impl ToBytes<'_> for u64 {
     type Bytes = [u8; 8];
 
     fn to_bytes(&self) -> Result<Self::Bytes, Error> {
@@ -85,7 +136,7 @@ impl<'a> ToBytes<'a> for u64 {
     }
 }
 
-impl<'a> ToBytes<'a> for u32 {
+impl ToBytes<'_> for u32 {
     type Bytes = [u8; 4];
 
     fn to_bytes(&self) -> Result<Self::Bytes, Error> {
@@ -93,6 +144,14 @@ impl<'a> ToBytes<'a> for u32 {
     }
 }
 
+impl ToBytes<'_> for bool {
+    type Bytes = [u8; 1];
+
+    fn to_bytes(&self) -> Result<Self::Bytes, Error> {
+        Ok([*self as u8])
+    }
+}
+
 impl<'a, T: ToBytes<'a>> ToBytes<'a> for &'a T {
     type Bytes = T::Bytes;
 
@@ -100,3 +159,26 @@ impl<'a, T: ToBytes<'a>> ToBytes<'a> for &'a T {
         <T as ToBytes>::to_bytes(self)
     }
 }
+
+impl<'a, T: ToBytes<'a>> ToBytes<'a> for Option<T> {
+    type Bytes = Vec<u8>;
+
+    fn to_bytes(&self) -> Result<Self::Bytes, Error> {
+        match self {
+            Some(x) => x.to_bytes().map(|x| x.as_ref().to_vec()),
+            None => Ok(vec![]),
+        }
+    }
+}
+
+#[test]
+fn test() {
+    use extism_convert::{Json, ToBytes};
+    use serde::Serialize;
+
+    #[derive(ToBytes, Serialize)]
+    #[encoding(Json)]
+    struct Struct {
+        hello: String,
+    }
+}

example-schema.yaml

@@ -0,0 +1,28 @@
+# yaml-language-server: $schema=https://xtp.dylibso.com/assets/wasm/schema.json
+# Learn more at https://docs.xtp.dylibso.com/docs/concepts/xtp-schema
+version: v1-draft
+exports: 
+  CountVowels:
+      input: 
+          type: string
+          contentType: text/plain; charset=utf-8
+      output:
+          $ref: "#/components/schemas/VowelReport"
+          contentType: application/json
+components:
+  schemas:
+    VowelReport:
+      description: The result of counting vowels on the Vowels input.
+      properties:
+        count:
+          type: integer
+          format: int32
+          description: The count of vowels for input string.
+        total:
+          type: integer
+          format: int32
+          description: The cumulative amount of vowels counted, if this keeps state across multiple function calls.
+          nullable: true
+        vowels:
+          type: string
+          description: The set of vowels used to get the count, e.g. "aAeEiIoOuU"

kernel/Cargo.toml

@@ -5,6 +5,9 @@ edition = "2021"
 
 [dependencies]
 
+[dev-dependencies]
+wasm-bindgen-test = "0.3.39"
+
 [features]
 default = ["bounds-checking"]
 bounds-checking = []

kernel/build.sh

@@ -17,6 +17,9 @@ done
 
 cargo build --package extism-runtime-kernel --bin extism-runtime --release --target wasm32-unknown-unknown $CARGO_FLAGS
 cp target/wasm32-unknown-unknown/release/extism-runtime.wasm .
-wasm-strip extism-runtime.wasm
-mv extism-runtime.wasm ../runtime/src/extism-runtime.wasm
 
+wasm-tools parse extism-context.wat -o extism-context.wasm
+wasm-merge --enable-reference-types ./extism-runtime.wasm runtime extism-context.wasm context -o ../runtime/src/extism-runtime.wasm
+rm extism-context.wasm
+rm extism-runtime.wasm
+wasm-strip ../runtime/src/extism-runtime.wasm

kernel/extism-context.wat

@@ -0,0 +1,3 @@
+(module
+  (global (export "extism_context") (mut externref) (ref.null extern))
+)

kernel/src/bin/extism-runtime.rs

@@ -3,7 +3,7 @@
 
 pub use extism_runtime_kernel::*;
 
-#[cfg(target_arch = "wasm32")]
+#[cfg(all(target_arch = "wasm32", not(test)))]
 #[panic_handler]
 fn panic(_info: &core::panic::PanicInfo) -> ! {
     core::arch::wasm32::unreachable()

kernel/src/lib.rs

@@ -39,7 +39,7 @@
 use core::sync::atomic::*;
 
 pub type Pointer = u64;
-pub type Length = u64;
+pub type Handle = u64;
 
 /// WebAssembly page size
 const PAGE_SIZE: usize = 65536;
@@ -81,13 +81,13 @@ pub struct MemoryRoot {
     /// Offset of error block
     pub error: AtomicU64,
     /// Input position in memory
-    pub input_offset: Pointer,
+    pub input_offset: Handle,
     /// Input length
-    pub input_length: Length,
+    pub input_length: u64,
     /// Output position in memory
     pub output_offset: Pointer,
     /// Output length
-    pub output_length: Length,
+    pub output_length: u64,
     /// A pointer to the start of the first block
     pub blocks: [MemoryBlock; 0],
 }
@@ -198,15 +198,15 @@ impl MemoryRoot {
     fn pointer_in_bounds_fast(p: Pointer) -> bool {
         // Similar to `pointer_in_bounds` but less accurate on the upper bound. This uses the total memory size,
         // instead of checking `MemoryRoot::length`
-        let end = core::arch::wasm32::memory_size(0) << 16;
-        p >= core::mem::size_of::<Self>() as Pointer && p <= end as Pointer
+        let end = (core::arch::wasm32::memory_size(0) as u64) << 16;
+        p >= core::mem::size_of::<Self>() as Pointer && p <= end as u64
     }
 
     // Find a block that is free to use, this can be a new block or an existing freed block. The `self_position` argument
     // is used to avoid loading the allocators position more than once when performing an allocation.
     unsafe fn find_free_block(
         &mut self,
-        length: Length,
+        length: u64,
         self_position: u64,
     ) -> Option<&'static mut MemoryBlock> {
         // Get the first block
@@ -228,7 +228,7 @@ impl MemoryRoot {
             if status == MemoryStatus::Free as u8 && b.size >= length as usize {
                 // Split block if there is too much excess
                 if b.size - length as usize >= 128 {
-                    b.size -= length as usize;
+                    b.size -= length as usize + core::mem::size_of::<MemoryBlock>();
                     b.used = 0;
 
                     let block1 = b.data.as_mut_ptr().add(b.size) as *mut MemoryBlock;
@@ -252,30 +252,37 @@ impl MemoryRoot {
 
     /// Create a new `MemoryBlock`, when `Some(block)` is returned, `block` will contain at least enough room for `length` bytes
     /// but may be as large as `length` + `BLOCK_SPLIT_SIZE` bytes. When `None` is returned the allocation has failed.
-    pub unsafe fn alloc(&mut self, length: Length) -> Option<&'static mut MemoryBlock> {
+    pub unsafe fn alloc(&mut self, length: u64) -> Option<&'static mut MemoryBlock> {
         let self_position = self.position.load(Ordering::Acquire);
         let self_length = self.length.load(Ordering::Acquire);
-        let b = self.find_free_block(length, self_position);
-
-        // If there's a free block then re-use it
-        if let Some(b) = b {
-            b.used = length as usize;
-            b.status
-                .store(MemoryStatus::Active as u8, Ordering::Release);
-            return Some(b);
-        }
 
         // Get the current index for a new block
         let curr = self.blocks.as_ptr() as u64 + self_position;
 
         // Get the number of bytes available
         let mem_left = self_length - self_position - core::mem::size_of::<MemoryRoot>() as u64;
+        let length_with_block = length + core::mem::size_of::<MemoryBlock>() as u64;
 
         // When the allocation is larger than the number of bytes available
         // we will need to try to grow the memory
-        if length >= mem_left {
+        if length_with_block >= mem_left {
+            // If the current position is large enough to hold the length of the block being
+            // allocated then check for existing free blocks that can be re-used before
+            // growing memory
+            if length_with_block <= self_position {
+                let b = self.find_free_block(length, self_position);
+
+                // If there's a free block then re-use it
+                if let Some(b) = b {
+                    b.used = length as usize;
+                    b.status
+                        .store(MemoryStatus::Active as u8, Ordering::Release);
+                    return Some(b);
+                }
+            }
+
             // Calculate the number of pages needed to cover the remaining bytes
-            let npages = num_pages(length - mem_left);
+            let npages = num_pages(length_with_block - mem_left);
             let x = core::arch::wasm32::memory_grow(0, npages);
             if x == usize::MAX {
                 return None;
@@ -306,9 +313,26 @@ impl MemoryRoot {
         if !Self::pointer_in_bounds_fast(offs) {
             return None;
         }
-        let ptr = offs - core::mem::size_of::<MemoryBlock>() as u64;
-        let ptr = ptr as *mut MemoryBlock;
-        Some(&mut *ptr)
+
+        // Get the first block
+        let mut block = self.blocks.as_mut_ptr();
+
+        // Only loop while the block pointer is less then the current position
+        while (block as u64) < self.blocks.as_ptr() as u64 + offs {
+            let b = &mut *block;
+
+            // Get the block status, this lets us know if we are able to re-use it
+            let status = b.status.load(Ordering::Acquire);
+
+            if status == MemoryStatus::Active as u8 && b.data.as_ptr() as Pointer == offs {
+                return Some(b);
+            }
+
+            // Get the next block
+            block = b.next_ptr();
+        }
+
+        None
     }
 }
 
@@ -333,21 +357,21 @@ impl MemoryBlock {
 
 /// Allocate a block of memory and return the offset
 #[no_mangle]
-pub unsafe fn alloc(n: Length) -> Pointer {
+pub unsafe fn alloc(n: u64) -> Handle {
     if n == 0 {
         return 0;
     }
     let region = MemoryRoot::new();
     let block = region.alloc(n);
     match block {
-        Some(block) => block.data.as_mut_ptr() as Pointer,
+        Some(block) => block.data.as_mut_ptr() as Handle,
         None => 0,
     }
 }
 
 /// Free allocated memory
 #[no_mangle]
-pub unsafe fn free(p: Pointer) {
+pub unsafe fn free(p: Handle) {
     if p == 0 {
         return;
     }
@@ -365,13 +389,42 @@ pub unsafe fn free(p: Pointer) {
 }
 
 /// Get the length of an allocated memory block
+///
+/// Note: this should only be called on memory handles returned
+/// by a call to `alloc` - it will return garbage on invalid offsets
 #[no_mangle]
-pub unsafe fn length(p: Pointer) -> Length {
+pub unsafe fn length_unsafe(p: Handle) -> u64 {
     if p == 0 {
         return 0;
     }
+
+    if !MemoryRoot::pointer_in_bounds_fast(p) {
+        return 0;
+    }
+
+    let ptr = p - core::mem::size_of::<MemoryBlock>() as u64;
+    let block = &mut *(ptr as *mut MemoryBlock);
+
+    // Simplest sanity check to verify the pointer is a block
+    if block.status.load(Ordering::Acquire) != MemoryStatus::Active as u8 {
+        return 0;
+    }
+
+    block.used as u64
+}
+
+/// Get the length but returns 0 if the offset is not a valid handle.
+///
+/// Note: this function walks each node in the allocations list, which ensures correctness, but is also
+/// slow
+#[no_mangle]
+pub unsafe fn length(p: Pointer) -> u64 {
+    if p == 0 {
+        return 0;
+    }
+
     if let Some(block) = MemoryRoot::new().find_block(p) {
-        block.used as Length
+        block.used as u64
     } else {
         0
     }
@@ -399,24 +452,24 @@ pub unsafe fn load_u64(p: Pointer) -> u64 {
 
 /// Load a byte from the input data
 #[no_mangle]
-pub unsafe fn input_load_u8(p: Pointer) -> u8 {
+pub unsafe fn input_load_u8(offset: u64) -> u8 {
     let root = MemoryRoot::new();
     #[cfg(feature = "bounds-checking")]
-    if p >= root.input_length {
+    if offset >= root.input_length {
         return 0;
     }
-    *((root.input_offset + p) as *mut u8)
+    *((root.input_offset + offset) as *mut u8)
 }
 
 /// Load a u64 from the input data
 #[no_mangle]
-pub unsafe fn input_load_u64(p: Pointer) -> u64 {
+pub unsafe fn input_load_u64(offset: u64) -> u64 {
     let root = MemoryRoot::new();
     #[cfg(feature = "bounds-checking")]
-    if p + core::mem::size_of::<u64>() as Pointer > root.input_length {
+    if offset + core::mem::size_of::<u64>() as u64 > root.input_length {
         return 0;
     }
-    *((root.input_offset + p) as *mut u64)
+    *((root.input_offset + offset) as *mut u64)
 }
 
 /// Write a byte in Extism-managed memory
@@ -440,22 +493,28 @@ pub unsafe fn store_u64(p: Pointer, x: u64) {
 }
 
 /// Set the range of the input data in memory
+/// h must always be a handle so that length works on it
+/// len must match length(handle)
+/// **Note**: this function takes ownership of the handle passed in
+/// the caller should not `free` this value
 #[no_mangle]
-pub unsafe fn input_set(p: Pointer, len: Length) {
+pub unsafe fn input_set(h: Handle, len: u64) {
     let root = MemoryRoot::new();
     #[cfg(feature = "bounds-checking")]
     {
-        if !root.pointer_in_bounds(p) || !root.pointer_in_bounds(p + len - 1) {
+        if !root.pointer_in_bounds(h) || !root.pointer_in_bounds(h + len - 1) {
             return;
         }
     }
-    root.input_offset = p;
+    root.input_offset = h;
     root.input_length = len;
 }
 
 /// Set the range of the output data in memory
+/// **Note**: this function takes ownership of the handle passed in
+/// the caller should not `free` this value
 #[no_mangle]
-pub unsafe fn output_set(p: Pointer, len: Length) {
+pub unsafe fn output_set(p: Pointer, len: u64) {
     let root = MemoryRoot::new();
     #[cfg(feature = "bounds-checking")]
     {
@@ -469,25 +528,25 @@ pub unsafe fn output_set(p: Pointer, len: Length) {
 
 /// Get the input length
 #[no_mangle]
-pub fn input_length() -> Length {
+pub fn input_length() -> u64 {
     unsafe { MemoryRoot::new().input_length }
 }
 
 /// Get the input offset in Exitsm-managed memory
 #[no_mangle]
-pub fn input_offset() -> Length {
+pub fn input_offset() -> Handle {
     unsafe { MemoryRoot::new().input_offset }
 }
 
 /// Get the output length
 #[no_mangle]
-pub fn output_length() -> Length {
+pub fn output_length() -> u64 {
     unsafe { MemoryRoot::new().output_length }
 }
 
 /// Get the output offset in Extism-managed memory
 #[no_mangle]
-pub unsafe fn output_offset() -> Length {
+pub unsafe fn output_offset() -> Pointer {
     MemoryRoot::new().output_offset
 }
 
@@ -497,32 +556,60 @@ pub unsafe fn reset() {
     MemoryRoot::new().reset()
 }
 
-/// Set the error message offset
+/// Set the error message offset, the handle passed to this
+/// function should not be freed after this call
+/// **Note**: this function takes ownership of the handle passed in
+/// the caller should not `free` this value
 #[no_mangle]
-pub unsafe fn error_set(ptr: Pointer) {
+pub unsafe fn error_set(h: Handle) {
     let root = MemoryRoot::new();
 
     // Allow ERROR to be set to 0
-    if ptr == 0 {
-        root.error.store(ptr, Ordering::SeqCst);
+    if h == 0 {
+        root.error.store(h, Ordering::SeqCst);
         return;
     }
 
     #[cfg(feature = "bounds-checking")]
-    if !root.pointer_in_bounds(ptr) {
+    if !root.pointer_in_bounds(h) {
         return;
     }
-    root.error.store(ptr, Ordering::SeqCst);
+    root.error.store(h, Ordering::SeqCst);
 }
 
 /// Get the error message offset, if it's `0` then no error has been set
 #[no_mangle]
-pub unsafe fn error_get() -> Pointer {
+pub unsafe fn error_get() -> Handle {
     MemoryRoot::new().error.load(Ordering::SeqCst)
 }
 
 /// Get the position of the allocator, this can be used as an indication of how many bytes are currently in-use
 #[no_mangle]
-pub unsafe fn memory_bytes() -> Length {
+pub unsafe fn memory_bytes() -> u64 {
     MemoryRoot::new().length.load(Ordering::Acquire)
 }
+
+#[cfg(test)]
+mod test {
+    use crate::*;
+    use wasm_bindgen_test::*;
+
+    #[wasm_bindgen_test]
+    fn test_oom() {
+        let size = 1024 * 1024 * 5;
+
+        let mut last = 0;
+        for _ in 0..1024 {
+            unsafe {
+                let ptr = alloc(size);
+                last = ptr;
+                if ptr == 0 {
+                    break;
+                }
+                assert_eq!(length(ptr), size);
+            }
+        }
+
+        assert_eq!(last, 0);
+    }
+}

kernel/test.sh

@@ -0,0 +1,5 @@
+# install wasm-bindgen-cli to get wasm-bindgen-runner if it is not installed yet
+which wasm-bindgen-test-runner 1>/dev/null || cargo install -f wasm-bindgen-cli
+
+# run tests with the wasm-bindgen-runner
+CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER=wasm-bindgen-test-runner cargo test --release --target=wasm32-unknown-unknown

libextism/.clang-format

@@ -0,0 +1,2 @@
+BasedOnStyle: LLVM
+IndentWidth: 2

libextism/API.md

@@ -14,7 +14,9 @@ Create a new plugin.
 - `functions`: is an array of `ExtismFunction*`
 - `n_functions`: is the number of functions
 - `with_wasi`: enables/disables WASI
-- `errmsg`: error message during plugin creation
+- `errmsg`: error message during plugin creation, this should be freed with
+  `extism_plugin_new_error_free`
+
 
 ```c
 ExtismPlugin extism_plugin_new(const uint8_t *wasm,
@@ -24,6 +26,17 @@ ExtismPlugin extism_plugin_new(const uint8_t *wasm,
                                bool with_wasi,
                                char **errmsg);
 ```
+---
+
+### `extism_plugin_new_error_free`
+
+Frees the error message returned when creating a plugin
+
+```c
+void extism_plugin_new_error_free(char *err);
+```
+
+---
 
 ### `extism_plugin_free`
 
@@ -33,6 +46,8 @@ Remove a plugin from the registry and free associated memory.
 void extism_plugin_free(ExtismPlugin *plugin);
 ```
 
+---
+
 ### `extism_plugin_config`
 
 Update plugin config values, this will merge with the existing values.
@@ -43,6 +58,8 @@ bool extism_plugin_config(ExtismPlugin *plugin,
                           ExtismSize json_size);
 ```
 
+---
+
 ### `extism_plugin_function_exists`
 
 Returns true if `func_name` exists.
@@ -52,6 +69,8 @@ bool extism_plugin_function_exists(ExtismPlugin *plugin,
                                    const char *func_name);
 ```
 
+---
+
 ### `extism_plugin_call`
 
 Call a function.
@@ -59,6 +78,8 @@ Call a function.
 - `data`: is the input data
 - `data_len`: is the length of `data`
 
+Returns `0` when the call is successful.
+
 ```c
 int32_t extism_plugin_call(ExtismPlugin *plugin,
                            const char *func_name,
@@ -66,6 +87,28 @@ int32_t extism_plugin_call(ExtismPlugin *plugin,
                            ExtismSize data_len);
 ```
 
+---
+
+### `extism_plugin_call_with_host_context`
+
+Call a function with additional host context that can be accessed from inside host functions.
+- `func_name`: is the function to call
+- `data`: is the input data
+- `data_len`: is the length of `data`
+- `host_ctx`: an opaque pointer that can be accessed in host functions
+
+Returns `0` when the call is successful.
+
+```c
+int32_t extism_plugin_call_with_host_context(ExtismPlugin *plugin,
+                           const char *func_name,
+                           const uint8_t *data,
+                           ExtismSize data_len,
+                           void *host_ctx);
+```
+
+---
+
 ### `extism_plugin_error`
 
 Get the error associated with a `Plugin`
@@ -74,6 +117,8 @@ Get the error associated with a `Plugin`
 const char *extism_plugin_error(ExtismPlugin *plugin);
 ```
 
+---
+
 ### `extism_plugin_output_length`
 
 Get the length of a plugin's output data.
@@ -82,6 +127,8 @@ Get the length of a plugin's output data.
 ExtismSize extism_plugin_output_length(ExtismPlugin *plugin);
 ```
 
+---
+
 ### `extism_plugin_output_data`
 
 Get the plugin's output data.
@@ -90,6 +137,8 @@ Get the plugin's output data.
 const uint8_t *extism_plugin_output_data(ExtismPlugin *plugin);
 ```
 
+---
+
 ### `extism_plugin_reset`
 
 Reset the Extism runtime, this will invalidate all allocated memory.
@@ -98,6 +147,8 @@ Reset the Extism runtime, this will invalidate all allocated memory.
 bool extism_plugin_reset(ExtismPlugin *plugin);
 ```
 
+---
+
 ### `extism_log_file`
 
 Set log file and level.
@@ -106,6 +157,8 @@ Set log file and level.
 bool extism_log_file(const char *filename, const char *log_level);
 ```
 
+---
+
 ### `extism_log_custom`
 
 Enable a custom log handler, this will buffer logs until `extism_log_drain`
@@ -115,6 +168,8 @@ is called Log level should be one of: info, error, trace, debug, warn
 bool extism_log_custom(const char *log_level);
 ```
 
+---
+
 ### `extism_log_drain`
 
 Calls the provided callback function for each buffered log line.
@@ -124,6 +179,8 @@ This is only needed when `extism_log_custom` is used.
 void extism_log_drain(void (*handler)(const char *, uintptr_t));
 ```
 
+---
+
 ### `extism_version`
 
 Get the Extism version string.
@@ -132,6 +189,8 @@ Get the Extism version string.
 const char *extism_version(void);
 ```
 
+---
+
 ### `extism_current_plugin_memory`
 
 Returns a pointer to the memory of the currently running plugin
@@ -140,6 +199,19 @@ Returns a pointer to the memory of the currently running plugin
 uint8_t *extism_current_plugin_memory(ExtismCurrentPlugin *plugin);
 ```
 
+---
+
+### `extism_current_plugin_host_context`
+
+Get access to the host context, passed in using `extism_plugin_call_with_host_context`
+
+```c
+void *extism_current_plugin_host_context(ExtismCurrentPlugin *plugin);
+```
+
+---
+
+
 ### `extism_current_plugin_memory_alloc`
 
 Allocate a memory block in the currently running plugin
@@ -148,6 +220,8 @@ Allocate a memory block in the currently running plugin
 uint64_t extism_current_plugin_memory_alloc(ExtismCurrentPlugin *plugin, ExtismSize n);
 ```
 
+---
+
 ### `extism_current_plugin_memory_length`
 
 Get the length of an allocated block
@@ -156,6 +230,8 @@ Get the length of an allocated block
 ExtismSize extism_current_plugin_memory_length(ExtismCurrentPlugin *plugin, ExtismSize n);
 ```
 
+---
+
 ### `extism_current_plugin_memory_free`
 
 Free an allocated memory block
@@ -164,6 +240,8 @@ Free an allocated memory block
 void extism_current_plugin_memory_free(ExtismCurrentPlugin *plugin, uint64_t ptr);
 ```
 
+---
+
 ### `extism_function_new`
 Create a new host function
 - `name`: function name, this should be valid UTF-8
@@ -190,6 +268,8 @@ ExtismFunction *extism_function_new(const char *name,
                                     void (*free_user_data)(void *_));
 ```
 
+---
+
 ### `extism_function_set_namespace`
 
 Set the namespace of an `ExtismFunction`
@@ -198,6 +278,8 @@ Set the namespace of an `ExtismFunction`
 void extism_function_set_namespace(ExtismFunction *ptr, const char *namespace_);
 ```
 
+---
+
 ### `extism_function_free`
 
 Free an `ExtismFunction`
@@ -206,6 +288,8 @@ Free an `ExtismFunction`
 void extism_function_free(ExtismFunction *ptr);
 ```
 
+---
+
 ### `extism_plugin_cancel_handle`
 
 Get handle for plugin cancellation
@@ -214,6 +298,8 @@ Get handle for plugin cancellation
 const ExtismCancelHandle *extism_plugin_cancel_handle(const ExtismPlugin *plugin);
 ```
 
+---
+
 ### `extism_plugin_cancel`
 
 Cancel a running plugin from another thread
@@ -222,12 +308,14 @@ Cancel a running plugin from another thread
 bool extism_plugin_cancel(const ExtismCancelHandle *handle);
 ```
 
+---
+
 ## Type definitions:
 
 ### `ExtismPlugin`
 
 ```c
-typedef int32_t ExtismPlugin;
+typedef struct ExtismPlugin ExtismPlugin;
 ```
 
 ### `ExtismSize`
@@ -259,3 +347,5 @@ typedef struct ExtismCurrentPlugin ExtismCurrentPlugin;
 ```c
 typedef struct ExtismCancelHandle ExtismCancelHandle;
 ```
+
+

libextism/Makefile

@@ -4,7 +4,7 @@ build:
 
 .PHONY: static
 static:
-	$(CC) -g -o example example.c -l:libextism.a -lm
+	$(CC) -g -o example example.c -l:libextism.a -lm -lpthread
 
 # if needed, set PKG_CONFIG_PATH= to the directory with extism*.pc installed
 LDFLAGS=`pkg-config --libs extism`

libextism/README.md

@@ -56,7 +56,6 @@ Since you may not have an Extism plug-in on hand to test, let's load a demo plug
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <unistd.h>
 
 void print_plugin_output(ExtismPlugin *plugin, int32_t rc){
   if (rc != EXTISM_SUCCESS) {
@@ -64,9 +63,9 @@ void print_plugin_output(ExtismPlugin *plugin, int32_t rc){
     return;
   }
 
-  size_t outlen = extism_plugin_output_length(plugin);
+  ExtismSize outlen = extism_plugin_output_length(plugin);
   const uint8_t *out = extism_plugin_output_data(plugin);
-  write(STDOUT_FILENO, out, outlen);
+  fwrite(out, 1, outlen, stdout);
 }
 
 int main(void) {
@@ -106,9 +105,9 @@ if (rc != EXTISM_SUCCESS) {
   exit(2);
 }
 
-size_t outlen = extism_plugin_output_length(plugin);
+ExtismSize outlen = extism_plugin_output_length(plugin);
 const uint8_t *out = extism_plugin_output_data(plugin);
-write(STDOUT_FILENO, out, outlen);
+fwrite(out, 1, outlen, stdout);
 ```
 
 Will print
@@ -171,7 +170,6 @@ We want to expose two functions to our plugin, `kv_write(key: String, value: Byt
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <unistd.h>
 
 // A stubbed out KV store
 typedef struct KVStore KVStore;
@@ -184,14 +182,14 @@ extern const uint32_t fake_kv_store_get(KVStore *kv, const char *key,
 
 // Our host functions to access the fake KV store
 void kv_get(ExtismCurrentPlugin *plugin, const ExtismVal *inputs,
-            size_t ninputs, ExtismVal *outputs, size_t noutputs,
+            ExtismSize ninputs, ExtismVal *outputs, ExtismSize noutputs,
             void *userdata) {
   // Cast the userdata pointer
   KVStore *kv = (KVStore *)userdata;
 
   // Get the offset to the key in the plugin memory
   uint64_t offs = inputs[0].v.i64;
-  size_t keylen = extism_current_plugin_memory_length(plugin, offs);
+  ExtismSize keylen = extism_current_plugin_memory_length(plugin, offs);
 
   // Allocate a new block to return
   uint64_t outoffs =
@@ -205,23 +203,23 @@ void kv_get(ExtismCurrentPlugin *plugin, const ExtismVal *inputs,
   *(uint64_t *)(extism_current_plugin_memory(plugin) + outoffs) = value;
 
   // Return the offset to our allocated block
-  outputs[0].t = PTR;
+  outputs[0].t = EXTISM_PTR;
   outputs[0].v.i64 = outoffs;
 }
 
 void kv_set(ExtismCurrentPlugin *plugin, const ExtismVal *inputs,
-            size_t ninputs, ExtismVal *outputs, size_t noutputs,
+            ExtismSize ninputs, ExtismVal *outputs, ExtismSize noutputs,
             void *userdata) {
   // Cast the userdata pointer
   KVStore *kv = (KVStore *)userdata;
 
   // Get the offset to the key in the plugin memory
   uint64_t keyoffs = inputs[0].v.i64;
-  size_t keylen = extism_current_plugin_memory_length(plugin, keyoffs);
+  ExtismSize keylen = extism_current_plugin_memory_length(plugin, keyoffs);
 
   // Get the offset to the value in the plugin memory
   uint64_t valueoffs = inputs[1].v.i64;
-  size_t valuelen = extism_current_plugin_memory_length(plugin, valueoffs);
+  ExtismSize valuelen = extism_current_plugin_memory_length(plugin, valueoffs);
 
   // Set key => value
   fake_kv_store_set(
@@ -234,13 +232,13 @@ int main(void) {
   const char *manifest = "{\"wasm\": [{\"url\": "
                          "\"https://github.com/extism/plugins/releases/latest/"
                          "download/count_vowels_kvstore.wasm\"}]}";
-  const ExtismValType kv_get_inputs[] = {PTR};
-  const ExtismValType kv_get_outputs[] = {PTR};
+  const ExtismValType kv_get_inputs[] = {EXTISM_PTR};
+  const ExtismValType kv_get_outputs[] = {EXTISM_PTR};
   ExtismFunction *kv_get_fn = extism_function_new(
       "kv_get", kv_get_inputs, 1, kv_get_outputs, 1, kv_get, kv, NULL);
 
-  const ExtismValType kv_set_inputs[] = {PTR};
-  const ExtismValType kv_set_outputs[] = {PTR};
+  const ExtismValType kv_set_inputs[] = {EXTISM_PTR};
+  const ExtismValType kv_set_outputs[] = {EXTISM_PTR};
   ExtismFunction *kv_set_fn = extism_function_new(
       "kv_set", kv_set_inputs, 1, kv_set_outputs, 1, kv_set, kv, NULL);
   const ExtismFunction *functions[] = {kv_get_fn};
@@ -288,5 +286,3 @@ print_plugin_output(plugin, extism_plugin_call(plugin, "count_vowels",
 # => Writing value=6 from key=count-vowels"
 # => {"count": 3, "total": 6, "vowels": "aeiouAEIOU"}
 ```
-
-

libextism/example.c

@@ -9,7 +9,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
-void log_handler(const char *line, uintptr_t length) {
+void log_handler(const char *line, ExtismSize length) {
   fwrite(line, length, 1, stderr);
 }
 
@@ -63,8 +63,8 @@ int main(int argc, char *argv[]) {
 
   size_t len = 0;
   uint8_t *data = read_file("../wasm/code-functions.wasm", &len);
-  ExtismValType inputs[] = {PTR};
-  ExtismValType outputs[] = {PTR};
+  ExtismValType inputs[] = {EXTISM_PTR};
+  ExtismValType outputs[] = {EXTISM_PTR};
   ExtismFunction *f =
       extism_function_new("hello_world", inputs, 1, outputs, 1, hello_world,
                           "Hello, again!", free_data);

libextism/extism-static.pc.in

@@ -5,6 +5,6 @@ includedir=${prefix}/include
 Version: 1.0.0
 Name: Extism
 Description: The framework for building with WebAssembly (wasm).
-Libs: -L${libdir} -l:libextism.a
-Libs.private: -lm
+Libs: ${libdir}/libextism.a
+Libs.private: -lm -lpthread
 Cflags: -I${includedir}

libextism/extism.pc.in

@@ -6,5 +6,5 @@ Version: 1.0.0
 Name: Extism
 Description: The framework for building with WebAssembly (wasm).
 Libs: -L${libdir} -lextism
-Libs.private: -lm
+Libs.private: -lm -lpthread
 Cflags: -I${includedir}

manifest/Cargo.toml

@@ -10,7 +10,7 @@ version.workspace = true
 
 [dependencies]
 serde = { version = "1", features = ["derive"] }
-base64 = "~0.21"
+base64 = "~0.22"
 schemars = { version = "0.8", optional = true }
 serde_json = "1"
 

manifest/schema.json

@@ -1,9 +1,11 @@
 {
   "$schema": "http://json-schema.org/draft-07/schema#",
   "title": "Manifest",
+  "description": "The `Manifest` type is used to configure the runtime and specify how to load modules.",
   "type": "object",
   "properties": {
     "allowed_hosts": {
+      "description": "Specifies which hosts may be accessed via HTTP, if this is empty then no hosts may be accessed. Wildcards may be used.",
       "default": null,
       "type": [
         "array",
@@ -14,6 +16,7 @@
       }
     },
     "allowed_paths": {
+      "description": "Specifies which paths should be made available on disk when using WASI. This is a mapping from the path on disk to the path it should be available inside the plugin. For example, `\".\": \"/tmp\"` would mount the current directory as `/tmp` inside the module",
       "default": null,
       "type": [
         "object",
@@ -24,6 +27,7 @@
       }
     },
     "config": {
+      "description": "Config values are made accessible using the PDK `extism_config_get` function",
       "default": {},
       "type": "object",
       "additionalProperties": {
@@ -31,8 +35,11 @@
       }
     },
     "memory": {
+      "description": "Memory options",
       "default": {
-        "max_pages": null
+        "max_http_response_bytes": null,
+        "max_pages": null,
+        "max_var_bytes": null
       },
       "allOf": [
         {
@@ -41,6 +48,8 @@
       ]
     },
     "timeout_ms": {
+      "description": "The plugin timeout in milliseconds",
+      "default": null,
       "type": [
         "integer",
         "null"
@@ -49,6 +58,7 @@
       "minimum": 0.0
     },
     "wasm": {
+      "description": "WebAssembly modules, the `main` module should be named `main` or listed last",
       "default": [],
       "type": "array",
       "items": {
@@ -56,35 +66,63 @@
       }
     }
   },
+  "additionalProperties": false,
   "definitions": {
     "MemoryOptions": {
+      "description": "Configure memory settings",
       "type": "object",
       "properties": {
+        "max_http_response_bytes": {
+          "description": "The maximum number of bytes allowed in an HTTP response",
+          "default": null,
+          "type": [
+            "integer",
+            "null"
+          ],
+          "format": "uint64",
+          "minimum": 0.0
+        },
         "max_pages": {
+          "description": "The max number of WebAssembly pages that should be allocated",
           "type": [
             "integer",
             "null"
           ],
           "format": "uint32",
           "minimum": 0.0
+        },
+        "max_var_bytes": {
+          "description": "The maximum number of bytes allowed to be used by plugin vars. Setting this to 0 will disable Extism vars. The default value is 1mb.",
+          "default": 1048576,
+          "type": [
+            "integer",
+            "null"
+          ],
+          "format": "uint64",
+          "minimum": 0.0
         }
-      }
+      },
+      "additionalProperties": false
     },
     "Wasm": {
+      "description": "The `Wasm` type specifies how to access a WebAssembly module",
       "anyOf": [
         {
+          "description": "From disk",
           "type": "object",
           "required": [
             "path"
           ],
           "properties": {
             "hash": {
+              "description": "Module hash, if the data loaded from disk or via HTTP doesn't match an error will be raised",
               "type": [
                 "string",
                 "null"
               ]
             },
             "name": {
+              "description": "Module name, this is used by Extism to determine which is the `main` module",
               "type": [
                 "string",
                 "null"
@@ -93,45 +131,72 @@
             "path": {
               "type": "string"
             }
-          }
+          },
+          "additionalProperties": false
         },
         {
+          "description": "From memory",
           "type": "object",
           "required": [
             "data"
           ],
           "properties": {
             "data": {
-              "type": "string",
-              "format": "string"
+              "type": [
+                "string",
+                "object"
+              ],
+              "required": [
+                "len",
+                "ptr"
+              ],
+              "properties": {
+                "len": {
+                  "type": "integer",
+                  "format": "uint64",
+                  "minimum": 0.0
+                },
+                "ptr": {
+                  "type": "integer",
+                  "format": "uint64",
+                  "minimum": 0.0
+                }
+              },
+              "additionalProperties": false
             },
             "hash": {
+              "description": "Module hash, if the data loaded from disk or via HTTP doesn't match an error will be raised",
               "type": [
                 "string",
                 "null"
               ]
             },
             "name": {
+              "description": "Module name, this is used by Extism to determine which is the `main` module",
               "type": [
                 "string",
                 "null"
               ]
             }
-          }
+          },
+          "additionalProperties": false
         },
         {
+          "description": "Via HTTP",
           "type": "object",
           "required": [
             "url"
           ],
           "properties": {
             "hash": {
+              "description": "Module hash, if the data loaded from disk or via HTTP doesn't match an error will be raised",
               "type": [
                 "string",
                 "null"
               ]
             },
             "headers": {
+              "description": "Request headers",
               "default": {},
               "type": "object",
               "additionalProperties": {
@@ -139,21 +204,25 @@
               }
             },
             "method": {
+              "description": "Request method",
               "type": [
                 "string",
                 "null"
               ]
             },
             "name": {
+              "description": "Module name, this is used by Extism to determine which is the `main` module",
               "type": [
                 "string",
                 "null"
               ]
             },
             "url": {
+              "description": "The request URL",
               "type": "string"
             }
-          }
+          },
+          "additionalProperties": false
         }
       ]
     }

manifest/src/lib.rs

@@ -1,6 +1,10 @@
 use std::collections::BTreeMap;
 use std::path::{Path, PathBuf};
 
+mod local_path;
+
+pub use local_path::LocalPath;
+
 #[deprecated]
 pub type ManifestMemory = MemoryOptions;
 
@@ -10,8 +14,45 @@ pub type ManifestMemory = MemoryOptions;
 #[serde(deny_unknown_fields)]
 pub struct MemoryOptions {
     /// The max number of WebAssembly pages that should be allocated
-    #[serde(alias = "max")]
     pub max_pages: Option<u32>,
+
+    /// The maximum number of bytes allowed in an HTTP response
+    #[serde(default)]
+    pub max_http_response_bytes: Option<u64>,
+
+    /// The maximum number of bytes allowed to be used by plugin vars. Setting this to 0
+    /// will disable Extism vars. The default value is 1mb.
+    #[serde(default = "default_var_bytes")]
+    pub max_var_bytes: Option<u64>,
+}
+
+impl MemoryOptions {
+    /// Create an empty `MemoryOptions` value
+    pub fn new() -> Self {
+        Default::default()
+    }
+
+    /// Set max pages
+    pub fn with_max_pages(mut self, pages: u32) -> Self {
+        self.max_pages = Some(pages);
+        self
+    }
+
+    /// Set max HTTP response size
+    pub fn with_max_http_response_bytes(mut self, bytes: u64) -> Self {
+        self.max_http_response_bytes = Some(bytes);
+        self
+    }
+
+    /// Set max size of Extism vars
+    pub fn with_max_var_bytes(mut self, bytes: u64) -> Self {
+        self.max_var_bytes = Some(bytes);
+        self
+    }
+}
+
+fn default_var_bytes() -> Option<u64> {
+    Some(1024 * 1024)
 }
 
 /// Generic HTTP request structure
@@ -24,7 +65,6 @@ pub struct HttpRequest {
 
     /// Request headers
     #[serde(default)]
-    #[serde(alias = "header")]
     pub headers: std::collections::BTreeMap<String, String>,
 
     /// Request method
@@ -111,8 +151,8 @@ pub enum Wasm {
 
     /// From memory
     Data {
-        #[serde(with = "base64")]
-        #[cfg_attr(feature = "json_schema", schemars(schema_with = "base64_schema"))]
+        #[serde(with = "wasmdata")]
+        #[cfg_attr(feature = "json_schema", schemars(schema_with = "wasmdata_schema"))]
         data: Vec<u8>,
         #[serde(flatten)]
         meta: WasmMetadata,
@@ -195,11 +235,25 @@ impl Wasm {
     }
 }
 
+#[derive(Default, Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
+#[cfg_attr(feature = "json_schema", derive(schemars::JsonSchema))]
+#[serde(deny_unknown_fields)]
+struct DataPtrLength {
+    ptr: u64,
+    len: u64,
+}
+
 #[cfg(feature = "json_schema")]
-fn base64_schema(gen: &mut schemars::gen::SchemaGenerator) -> schemars::schema::Schema {
+fn wasmdata_schema(gen: &mut schemars::gen::SchemaGenerator) -> schemars::schema::Schema {
     use schemars::{schema::SchemaObject, JsonSchema};
     let mut schema: SchemaObject = <String>::json_schema(gen).into();
-    schema.format = Some("string".to_owned());
+    let objschema: SchemaObject = <DataPtrLength>::json_schema(gen).into();
+    let types = schemars::schema::SingleOrVec::<schemars::schema::InstanceType>::Vec(vec![
+        schemars::schema::InstanceType::String,
+        schemars::schema::InstanceType::Object,
+    ]);
+    schema.instance_type = Some(types);
+    schema.object = objschema.object;
     schema.into()
 }
 
@@ -211,6 +265,7 @@ pub struct Manifest {
     /// WebAssembly modules, the `main` module should be named `main` or listed last
     #[serde(default)]
     pub wasm: Vec<Wasm>,
+
     /// Memory options
     #[serde(default)]
     pub memory: MemoryOptions,
@@ -218,8 +273,8 @@ pub struct Manifest {
     /// Config values are made accessible using the PDK `extism_config_get` function
     #[serde(default)]
     pub config: BTreeMap<String, String>,
-    #[serde(default)]
 
+    #[serde(default)]
     /// Specifies which hosts may be accessed via HTTP, if this is empty then
     /// no hosts may be accessed. Wildcards may be used.
     pub allowed_hosts: Option<Vec<String>>,
@@ -228,23 +283,18 @@ pub struct Manifest {
     /// the path on disk to the path it should be available inside the plugin.
     /// For example, `".": "/tmp"` would mount the current directory as `/tmp` inside the module
     #[serde(default)]
-    pub allowed_paths: Option<BTreeMap<PathBuf, PathBuf>>,
+    pub allowed_paths: Option<BTreeMap<PathBuf, LocalPath>>,
 
-    /// The plugin timeout, by default this is set to 30s
-    #[serde(default = "default_timeout")]
+    /// The plugin timeout in milliseconds
+    #[serde(default)]
     pub timeout_ms: Option<u64>,
 }
 
-fn default_timeout() -> Option<u64> {
-    Some(30000)
-}
-
 impl Manifest {
     /// Create a new manifest
     pub fn new(wasm: impl IntoIterator<Item = impl Into<Wasm>>) -> Manifest {
         Manifest {
             wasm: wasm.into_iter().map(|x| x.into()).collect(),
-            timeout_ms: default_timeout(),
             ..Default::default()
         }
     }
@@ -291,16 +341,15 @@ impl Manifest {
     }
 
     /// Add a path to `allowed_paths`
-    pub fn with_allowed_path(mut self, src: impl AsRef<Path>, dest: impl AsRef<Path>) -> Self {
-        let src = src.as_ref().to_path_buf();
+    pub fn with_allowed_path(mut self, src: impl Into<LocalPath>, dest: impl AsRef<Path>) -> Self {
         let dest = dest.as_ref().to_path_buf();
         match &mut self.allowed_paths {
             Some(p) => {
-                p.insert(src, dest);
+                p.insert(dest, src.into());
             }
             None => {
                 let mut p = BTreeMap::new();
-                p.insert(src, dest);
+                p.insert(dest, src.into());
                 self.allowed_paths = Some(p);
             }
         }
@@ -309,8 +358,8 @@ impl Manifest {
     }
 
     /// Set `allowed_paths`
-    pub fn with_allowed_paths(mut self, paths: impl Iterator<Item = (PathBuf, PathBuf)>) -> Self {
-        self.allowed_paths = Some(paths.collect());
+    pub fn with_allowed_paths(mut self, paths: impl Iterator<Item = (LocalPath, PathBuf)>) -> Self {
+        self.allowed_paths = Some(paths.map(|(local, wasm)| (wasm, local)).collect());
         self
     }
 
@@ -340,10 +389,12 @@ impl Manifest {
     }
 }
 
-mod base64 {
+mod wasmdata {
+    use crate::DataPtrLength;
     use base64::{engine::general_purpose, Engine as _};
     use serde::{Deserialize, Serialize};
     use serde::{Deserializer, Serializer};
+    use std::slice;
 
     pub fn serialize<S: Serializer>(v: &Vec<u8>, s: S) -> Result<S::Ok, S::Error> {
         let base64 = general_purpose::STANDARD.encode(v.as_slice());
@@ -351,21 +402,33 @@ mod base64 {
     }
 
     pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Vec<u8>, D::Error> {
-        let base64 = String::deserialize(d)?;
-        general_purpose::STANDARD
-            .decode(base64.as_bytes())
-            .map_err(serde::de::Error::custom)
+        #[derive(Deserialize)]
+        #[serde(untagged)]
+        enum WasmDataTypes {
+            String(String),
+            DataPtrLength(DataPtrLength),
+        }
+        Ok(match WasmDataTypes::deserialize(d)? {
+            WasmDataTypes::String(string) => general_purpose::STANDARD
+                .decode(string.as_bytes())
+                .map_err(serde::de::Error::custom)?,
+            WasmDataTypes::DataPtrLength(ptrlen) => {
+                let slice =
+                    unsafe { slice::from_raw_parts(ptrlen.ptr as *const u8, ptrlen.len as usize) };
+                slice.to_vec()
+            }
+        })
     }
 }
 
-impl<'a> From<Manifest> for std::borrow::Cow<'a, [u8]> {
+impl From<Manifest> for std::borrow::Cow<'_, [u8]> {
     fn from(m: Manifest) -> Self {
         let s = serde_json::to_vec(&m).unwrap();
         std::borrow::Cow::Owned(s)
     }
 }
 
-impl<'a> From<&Manifest> for std::borrow::Cow<'a, [u8]> {
+impl From<&Manifest> for std::borrow::Cow<'_, [u8]> {
     fn from(m: &Manifest) -> Self {
         let s = serde_json::to_vec(&m).unwrap();
         std::borrow::Cow::Owned(s)

manifest/src/local_path.rs

@@ -0,0 +1,119 @@
+use std::path::{Path, PathBuf};
+
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
+#[cfg_attr(feature = "json_schema", derive(schemars::JsonSchema))]
+pub enum LocalPath {
+    ReadOnly(PathBuf),
+    ReadWrite(PathBuf),
+}
+
+impl LocalPath {
+    pub fn as_path(&self) -> &Path {
+        match self {
+            LocalPath::ReadOnly(p) => p.as_path(),
+            LocalPath::ReadWrite(p) => p.as_path(),
+        }
+    }
+}
+
+impl From<&str> for LocalPath {
+    fn from(value: &str) -> Self {
+        if let Some(s) = value.strip_prefix("ro:") {
+            LocalPath::ReadOnly(PathBuf::from(s))
+        } else {
+            LocalPath::ReadWrite(PathBuf::from(value))
+        }
+    }
+}
+
+impl From<String> for LocalPath {
+    fn from(value: String) -> Self {
+        LocalPath::from(value.as_str())
+    }
+}
+
+impl From<PathBuf> for LocalPath {
+    fn from(value: PathBuf) -> Self {
+        LocalPath::ReadWrite(value)
+    }
+}
+
+impl From<&Path> for LocalPath {
+    fn from(value: &Path) -> Self {
+        LocalPath::ReadWrite(value.to_path_buf())
+    }
+}
+
+impl serde::Serialize for LocalPath {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        match self {
+            LocalPath::ReadOnly(path) => {
+                let s = match path.to_str() {
+                    Some(s) => s,
+                    None => {
+                        return Err(serde::ser::Error::custom(
+                            "Path contains invalid UTF-8 characters",
+                        ))
+                    }
+                };
+
+                format!("ro:{s}").serialize(serializer)
+            }
+            LocalPath::ReadWrite(path) => path.serialize(serializer),
+        }
+    }
+}
+
+struct LocalPathVisitor;
+
+impl serde::de::Visitor<'_> for LocalPathVisitor {
+    type Value = LocalPath;
+
+    fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+        formatter.write_str("path string")
+    }
+
+    fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
+    where
+        E: serde::de::Error,
+    {
+        Ok(From::from(v))
+    }
+
+    fn visit_string<E>(self, v: String) -> Result<Self::Value, E>
+    where
+        E: serde::de::Error,
+    {
+        Ok(From::from(v))
+    }
+
+    fn visit_bytes<E>(self, v: &[u8]) -> Result<Self::Value, E>
+    where
+        E: serde::de::Error,
+    {
+        std::str::from_utf8(v)
+            .map(From::from)
+            .map_err(|_| serde::de::Error::invalid_value(serde::de::Unexpected::Bytes(v), &self))
+    }
+
+    fn visit_byte_buf<E>(self, v: Vec<u8>) -> Result<Self::Value, E>
+    where
+        E: serde::de::Error,
+    {
+        String::from_utf8(v).map(From::from).map_err(|e| {
+            serde::de::Error::invalid_value(serde::de::Unexpected::Bytes(&e.into_bytes()), &self)
+        })
+    }
+}
+
+impl<'de> serde::Deserialize<'de> for LocalPath {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::de::Deserializer<'de>,
+    {
+        deserializer.deserialize_string(LocalPathVisitor)
+    }
+}

runtime/Cargo.toml

@@ -9,20 +9,21 @@ repository.workspace = true
 version.workspace = true
 
 [dependencies]
-wasmtime = ">= 14.0.0, < 16.0.0"
-wasmtime-wasi = ">= 14.0.0, < 16.0.0"
+wasmtime = {version = ">= 27.0.0, < 31.0.0"}
+wasi-common = {version = ">= 27.0.0, < 31.0.0"}
+wiggle = {version = ">= 27.0.0, < 31.0.0"}
 anyhow = "1"
 serde = {version = "1", features = ["derive"]}
 serde_json = "1"
 toml = "0.8"
 sha2 = "0.10"
 tracing = "0.1"
-tracing-subscriber = {version = "0.3", features = ["std", "env-filter", "fmt"]}
+tracing-subscriber = {version = "0.3.18", features = ["std", "env-filter", "fmt"]}
 url = "2"
 glob = "0.3"
-ureq = {version = "2.5", optional=true}
+ureq = {version = "3.0", optional=true}
 extism-manifest = { workspace = true }
-extism-convert = { workspace = true }
+extism-convert = { workspace = true, features = ["extism-path"] }
 uuid = { version = "1", features = ["v4"] }
 libc = "0.2"
 
@@ -33,10 +34,12 @@ register-filesystem = [] # enables wasm to be loaded from disk
 http = ["ureq"]          # enables extism_http_request
 
 [build-dependencies]
-cbindgen = "0.26"
+cbindgen = { version = "0.29", default-features = false }
 
 [dev-dependencies]
-criterion = "0.5.1"
+criterion = "0.6.0"
+quickcheck = "1"
+rand = "0.9.0"
 
 [[bench]]
 name = "bench"

runtime/README.md

@@ -12,7 +12,7 @@ To use the `extism` crate, you can add it to your Cargo file:
 
 ```toml
 [dependencies]
-extism = "^1.0.0-rc3"
+extism = "1.4.1"
 ```
 
 ## Environment variables
@@ -24,7 +24,7 @@ There are a few environment variables that can be used for debugging purposes:
 - `EXTISM_COREDUMP=extism.core`: write [coredump](https://github.com/WebAssembly/tool-conventions/blob/main/Coredump.md) to a file when a WebAssembly function traps
 - `EXTISM_DEBUG=1`: generate debug information
 - `EXTISM_PROFILE=perf|jitdump|vtune`: enable Wasmtime profiling
-- `EXTISM_CACHE_CONFIG=path/to/config.toml`: enable Wasmtime cache, see [the docs](https://docs.wasmtime.dev/cli-cache.html) for details about configuration. Setting this to an empty string will disable caching.
+- `EXTISM_CACHE_CONFIG=path/to/config.toml`: enable Wasmtime cache, details [here](#wasmtime-caching)
 
 > *Note*: The debug and coredump info will only be written if the plug-in has an error.
 
@@ -88,7 +88,7 @@ println!("{:?}", res);
 
 ### Plug-in State
 
-Plug-ins may be stateful or stateless. Plug-ins can maintain state b/w calls by the use of variables. Our count vowels plug-in remembers the total number of vowels it's ever counted in the "total" key in the result. You can see this by making subsequent calls to the export:
+Plug-ins may be stateful or stateless. Plug-ins can maintain state between calls by the use of variables. Our count vowels plug-in remembers the total number of vowels it's ever counted in the "total" key in the result. You can see this by making subsequent calls to the export:
 
 ```rust
 let res = plugin.call::<&str, &str>("count_vowels", "Hello, world!").unwrap();
@@ -201,7 +201,7 @@ fn main() {
 }
 ```
 
-> *Note*: In order to write host functions you should get familiar with the methods on the [Extism::CurrentPlugin](https://docs.rs/extism/latest/extism/struct.CurrentPlugin.html) and [Extism::CurrentPlugin](https://docs.rs/extism/latest/extism/struct.UserData.html) types.
+> *Note*: In order to write host functions you should get familiar with the methods on the [CurrentPlugin](https://docs.rs/extism/latest/extism/struct.CurrentPlugin.html) and [UserData](https://docs.rs/extism/latest/extism/enum.UserData.html) types.
 
 Now we can invoke the event:
 
@@ -219,5 +219,52 @@ println!("{}", res);
 # => {"count": 3, "total": 6, "vowels": "aeiouAEIOU"}
 ```
 
+### Logging
 
+Plug-ins can't directly print anything to the console. They can however use Extism's built-in logging functionality, for example the [`log!` macro in the rust-pdk](https://docs.rs/extism-pdk/1.4.0/extism_pdk/macro.log.html) or the [`logInfo` function in the haskell-pdk](https://hackage.haskell.org/package/extism-pdk-1.2.0.0/docs/Extism-PDK.html#v:logInfo).
 
+Inside your host application, the rust-sdk emits these as [tracing](https://github.com/tokio-rs/tracing) events. The simplest way to make the logged messages visible is by adding the `tracing_subscriber` dependency to your crate and then initializing a tracing subscriber at the top of your main function:
+
+```rust
+tracing_subscriber::fmt::init();
+```
+
+### Wasmtime Caching
+
+To enable or disable caching for plugin compilation, you need to provide a configuration file that will be used by the [wasmtime crate](https://github.com/bytecodealliance/wasmtime).
+
+For more information and values that can be used for configuring caching, take a look at [the docs](https://docs.wasmtime.dev/cli-cache.html).
+
+> *Note*: As of now extism uses wasmtime [`version = ">= 27.0.0, < 31.0.0"`](https://github.com/extism/extism/blob/v1.11.1/runtime/Cargo.toml#L12), but the `enabled` key requirement [was removed](https://github.com/bytecodealliance/wasmtime/pull/10859) from `wasmtime` and its documentation, this could explain the `failed to parse config file` error you might encounter without it.
+
+An example configuration for caching would be:
+
+```toml
+[cache]
+enabled = true # This value is required
+directory = "/some/path"
+```
+
+You can :
+- [Create a global `wasmtime` configuration file](#using-a-configuration-file) in `$HOME/.config/wasmtime/config.toml`.
+- [Set the `EXTISM_CACHE_CONFIG` environment variable](#using-an-environment-variable)
+- [Set the configuration file path using `PluginBuilder`](#using-pluginbuilder)
+
+#### Using a configuration file
+
+The [wasmtime](https://github.com/bytecodealliance/wasmtime) crate, by default, will look for a configuration file in your systems' default configuration directory (for example on UNIX systems: `$HOME/.config/wasmtime/config.toml`),
+for more [information on this behaviour](`https://docs.rs/wasmtime/31.0.0/wasmtime/struct.Config.html#method.cache_config_load_default`).
+
+#### Using an environment variable
+
+You can set the `EXTISM_CACHE_CONFIG=path/to/config.toml` environment variable to set the path of the configuration file used by [wasmtime](https://github.com/bytecodealliance/wasmtime).
+Setting the variable to an empty string will disable caching (it won't load any configuration file).
+
+> *Note*: If the environment variable is not set, `wasmtime` will still try to read from a configuration file that may exist in your system's default configuration folder (e.g. `$HOME/.config/wasmtime/config.toml`).
+
+The environment variable does not override the path you might have set using `PluginBuilder`. will only be checked for if you did not specify a cache configuration path in `PluginBuilder`.
+
+#### Using PluginBuilder
+
+If you use a [PluginBuilder](https://docs.rs/extism/latest/extism/struct.PluginBuilder.html), you can set the `wasmtime` configuration path using the [with_cache_config](https://docs.rs/extism/latest/extism/struct.PluginBuilder.html#method.with_cache_config) method.
+This will override the `EXTISM_CACHE_CONFIG` environment variable if it's set, so you could have a "global" and per plugin configuration if needed.

runtime/benches/bench.rs

@@ -6,6 +6,7 @@ const COUNT_VOWELS: &[u8] = include_bytes!("../../wasm/code.wasm");
 const REFLECT: &[u8] = include_bytes!("../../wasm/reflect.wasm");
 const ECHO: &[u8] = include_bytes!("../../wasm/echo.wasm");
 const CONSUME: &[u8] = include_bytes!("../../wasm/consume.wasm");
+const ALLOCATIONS: &[u8] = include_bytes!("../../wasm/allocations.wasm");
 
 host_fn!(hello_world (a: String) -> String { Ok(a) });
 
@@ -35,6 +36,46 @@ pub fn create_plugin(c: &mut Criterion) {
     });
 }
 
+pub fn create_compiled(c: &mut Criterion) {
+    let mut g = c.benchmark_group("create");
+    g.noise_threshold(1.0);
+    g.significance_level(0.2);
+    g.bench_function("create_compiled", |b| {
+        b.iter(|| {
+            let plugin = PluginBuilder::new(COUNT_VOWELS).with_wasi(true);
+            let _compiled = CompiledPlugin::new(plugin).unwrap();
+        })
+    });
+}
+
+pub fn create_plugin_compiled(c: &mut Criterion) {
+    let mut g = c.benchmark_group("create");
+    g.noise_threshold(1.0);
+    g.significance_level(0.2);
+    let plugin = PluginBuilder::new(COUNT_VOWELS).with_wasi(true);
+    let compiled = CompiledPlugin::new(plugin).unwrap();
+    g.bench_function("create_plugin_compiled", |b| {
+        b.iter(|| {
+            let _plugin = Plugin::new_from_compiled(&compiled).unwrap();
+        })
+    });
+}
+
+pub fn create_plugin_no_cache(c: &mut Criterion) {
+    let mut g = c.benchmark_group("create");
+    g.noise_threshold(1.0);
+    g.significance_level(0.2);
+    g.bench_function("create_plugin_no_cache", |b| {
+        b.iter(|| {
+            let _plugin = PluginBuilder::new(COUNT_VOWELS)
+                .with_cache_disabled()
+                .with_wasi(true)
+                .build()
+                .unwrap();
+        })
+    });
+}
+
 #[derive(Debug, serde::Deserialize, PartialEq)]
 struct Count {
     count: u32,
@@ -153,6 +194,17 @@ pub fn reflect(c: &mut Criterion) {
     }
 }
 
+pub fn allocations(c: &mut Criterion) {
+    let mut g = c.benchmark_group("allocations");
+
+    let mut plugin = PluginBuilder::new(ALLOCATIONS).build().unwrap();
+    g.bench_function("allocations", |b| {
+        b.iter(|| {
+            plugin.call::<_, ()>("allocations", "").unwrap();
+        })
+    });
+}
+
 // This is an apples-to-apples comparison of a linked wasm "reflect" function to our host "reflect"
 // function.
 pub fn reflect_linked(c: &mut Criterion) {
@@ -245,12 +297,16 @@ pub fn reflect_linked(c: &mut Criterion) {
 
 criterion_group!(
     benches,
+    allocations,
     consume,
     echo,
     reflect,
     reflect_linked,
     basic,
     create_plugin,
+    create_plugin_compiled,
+    create_plugin_no_cache,
+    create_compiled,
     count_vowels
 );
 criterion_main!(benches);

runtime/build.rs

@@ -7,7 +7,7 @@ fn main() {
 #define EXTISM_SUCCESS 0
 
 /** An alias for I64 to signify an Extism pointer */
-#define PTR I64
+#define EXTISM_PTR ExtismValType_I64
 ";
     if let Ok(x) = cbindgen::Builder::new()
         .with_crate(".")
@@ -24,6 +24,7 @@ fn main() {
         .rename_item("CurrentPlugin", "ExtismCurrentPlugin")
         .rename_item("CancelHandle", "ExtismCancelHandle")
         .rename_item("Plugin", "ExtismPlugin")
+        .rename_item("CompiledPlugin", "ExtismCompiledPlugin")
         .rename_item("Function", "ExtismFunction")
         .with_style(cbindgen::Style::Type)
         .generate()

runtime/examples/fs.rs

@@ -0,0 +1,36 @@
+use extism::*;
+fn main() {
+    let url = Wasm::file("../wasm/read_write.wasm");
+    let manifest = Manifest::new([url])
+        // This will fail because we're using a readonly path (specified with the `ro:` prefix)
+        // to overwrite the data file, remove `ro:` from the path on the following line
+        .with_allowed_path("ro:src/tests/data".to_string(), "/data")
+        .with_config_key("path", "/data/data.txt");
+
+    let mut plugin = PluginBuilder::new(manifest)
+        .with_wasi(true)
+        .build()
+        .unwrap();
+
+    println!("trying to read file: ");
+
+    let res = plugin.call::<&str, &str>("try_read", "").unwrap();
+
+    println!("{:?}", res);
+
+    println!("-----------------------------------------------------");
+
+    // If the allowed path is readonly then writing back to the file should fail
+    println!("trying to write file: ");
+    let line = format!(
+        "Hello World at {:?}\n",
+        std::time::SystemTime::now()
+            .duration_since(std::time::SystemTime::UNIX_EPOCH)
+            .unwrap()
+    );
+    let res2 = plugin.call::<&str, &str>("try_write", &line).unwrap();
+
+    println!("{:?}", res2);
+
+    println!("done!");
+}

runtime/examples/linking.rs

@@ -2,6 +2,8 @@ use extism::*;
 
 fn main() {
     let manifest = Manifest::new([
+        // upper.wat provides an export called `host_reflect` that takes a string
+        // and returns the same string uppercased
         Wasm::File {
             // See https://github.com/extism/plugins/blob/main/upper.wat
             path: "../wasm/upper.wasm".into(),
@@ -10,6 +12,9 @@ fn main() {
                 hash: None,
             },
         },
+        // reflect expects host_reflect to be imported: https://github.com/extism/plugins/blob/e5578bbbdd87f9936a0a8d36df629768b2eff6bb/reflect/src/lib.rs#L5
+        // Extism will link the export from upper.wat to the import of reflect.rs at runtime so it
+        // can call it
         Wasm::File {
             // See https://github.com/extism/plugins/tree/main/reflect
             path: "../wasm/reflect.wasm".into(),

runtime/extism.h

@@ -10,7 +10,7 @@
 #define EXTISM_SUCCESS 0
 
 /** An alias for I64 to signify an Extism pointer */
-#define PTR I64
+#define EXTISM_PTR ExtismValType_I64
 
 
 /**
@@ -20,31 +20,31 @@ typedef enum {
   /**
    * Signed 32 bit integer.
    */
-  I32,
+  ExtismValType_I32,
   /**
    * Signed 64 bit integer.
    */
-  I64,
+  ExtismValType_I64,
   /**
    * Floating point 32 bit integer.
    */
-  F32,
+  ExtismValType_F32,
   /**
    * Floating point 64 bit integer.
    */
-  F64,
+  ExtismValType_F64,
   /**
    * A 128 bit number.
    */
-  V128,
+  ExtismValType_V128,
   /**
    * A reference to a Wasm function.
    */
-  FuncRef,
+  ExtismValType_FuncRef,
   /**
    * A reference to opaque data in the Wasm instance.
    */
-  ExternRef,
+  ExtismValType_ExternRef,
 } ExtismValType;
 
 /**
@@ -52,6 +52,8 @@ typedef enum {
  */
 typedef struct ExtismCancelHandle ExtismCancelHandle;
 
+typedef struct ExtismCompiledPlugin ExtismCompiledPlugin;
+
 /**
  * CurrentPlugin stores data that is available to the caller in PDK functions, this should
  * only be accessed from inside a host function
@@ -97,6 +99,11 @@ typedef void (*ExtismFunctionType)(ExtismCurrentPlugin *plugin,
                                    ExtismSize n_outputs,
                                    void *data);
 
+/**
+ * Log drain callback
+ */
+typedef void (*ExtismLogDrainFunctionType)(const char *data, ExtismSize size);
+
 
 
 #ifdef __cplusplus
@@ -108,6 +115,12 @@ extern "C" {
  */
 const uint8_t *extism_plugin_id(ExtismPlugin *plugin);
 
+/**
+ * Get the current plugin's associated host context data. Returns null if call was made without
+ * host context.
+ */
+void *extism_current_plugin_host_context(ExtismCurrentPlugin *plugin);
+
 /**
  * Returns a pointer to the memory of the currently running plugin
  * NOTE: this should only be called from host functions.
@@ -143,7 +156,7 @@ void extism_current_plugin_memory_free(ExtismCurrentPlugin *plugin, ExtismMemory
  * - `n_outputs`: number of return types
  * - `func`: the function to call
  * - `user_data`: a pointer that will be passed to the function when it's called
- *    this value should live as long as the function exists
+ *   this value should live as long as the function exists
  * - `free_user_data`: a callback to release the `user_data` value when the resulting
  *   `ExtismFunction` is freed.
  *
@@ -168,6 +181,21 @@ void extism_function_free(ExtismFunction *f);
  */
 void extism_function_set_namespace(ExtismFunction *ptr, const char *namespace_);
 
+/**
+ * Pre-compile an Extism plugin
+ */
+ExtismCompiledPlugin *extism_compiled_plugin_new(const uint8_t *wasm,
+                                                 ExtismSize wasm_size,
+                                                 const ExtismFunction **functions,
+                                                 ExtismSize n_functions,
+                                                 bool with_wasi,
+                                                 char **errmsg);
+
+/**
+ * Free `ExtismCompiledPlugin`
+ */
+void extism_compiled_plugin_free(ExtismCompiledPlugin *plugin);
+
 /**
  * Create a new plugin with host functions, the functions passed to this function no longer need to be manually freed using
  *
@@ -184,13 +212,34 @@ ExtismPlugin *extism_plugin_new(const uint8_t *wasm,
                                 bool with_wasi,
                                 char **errmsg);
 
+/**
+ * Create a new plugin from an `ExtismCompiledPlugin`
+ */
+ExtismPlugin *extism_plugin_new_from_compiled(const ExtismCompiledPlugin *compiled, char **errmsg);
+
+/**
+ * Create a new plugin and set the number of instructions a plugin is allowed to execute
+ */
+ExtismPlugin *extism_plugin_new_with_fuel_limit(const uint8_t *wasm,
+                                                ExtismSize wasm_size,
+                                                const ExtismFunction **functions,
+                                                ExtismSize n_functions,
+                                                bool with_wasi,
+                                                uint64_t fuel_limit,
+                                                char **errmsg);
+
+/**
+ * Enable HTTP response headers in plugins using `extism:host/env::http_request`
+ */
+void extism_plugin_allow_http_response_headers(ExtismPlugin *plugin);
+
 /**
  * Free the error returned by `extism_plugin_new`, errors returned from `extism_plugin_error` don't need to be freed
  */
 void extism_plugin_new_error_free(char *err);
 
 /**
- * Remove a plugin from the registry and free associated memory
+ * Free `ExtismPlugin`
  */
 void extism_plugin_free(ExtismPlugin *plugin);
 
@@ -226,6 +275,20 @@ int32_t extism_plugin_call(ExtismPlugin *plugin,
                            const uint8_t *data,
                            ExtismSize data_len);
 
+/**
+ * Call a function with host context.
+ *
+ * `func_name`: is the function to call
+ * `data`: is the input data
+ * `data_len`: is the length of `data`
+ * `host_context`: a pointer to context data that will be available in host functions
+ */
+int32_t extism_plugin_call_with_host_context(ExtismPlugin *plugin,
+                                             const char *func_name,
+                                             const uint8_t *data,
+                                             ExtismSize data_len,
+                                             void *host_context);
+
 /**
  * Get the error associated with a `Plugin`
  */
@@ -264,7 +327,7 @@ bool extism_log_custom(const char *log_level);
  * Calls the provided callback function for each buffered log line.
  * This is only needed when `extism_log_custom` is used.
  */
-void extism_log_drain(void (*handler)(const char*, uintptr_t));
+void extism_log_drain(ExtismLogDrainFunctionType handler);
 
 /**
  * Reset the Extism runtime, this will invalidate all allocated memory
@@ -277,5 +340,5 @@ bool extism_plugin_reset(ExtismPlugin *plugin);
 const char *extism_version(void);
 
 #ifdef __cplusplus
-} // extern "C"
-#endif // __cplusplus
+}  // extern "C"
+#endif  // __cplusplus

runtime/src/current_plugin.rs

@@ -1,3 +1,5 @@
+use anyhow::Context;
+
 use crate::*;
 
 /// CurrentPlugin stores data that is available to the caller in PDK functions, this should
@@ -12,9 +14,11 @@ pub struct CurrentPlugin {
     pub(crate) linker: *mut wasmtime::Linker<CurrentPlugin>,
     pub(crate) wasi: Option<Wasi>,
     pub(crate) http_status: u16,
+    pub(crate) http_headers: Option<std::collections::BTreeMap<String, String>>,
     pub(crate) available_pages: Option<u32>,
     pub(crate) memory_limiter: Option<MemoryLimiter>,
     pub(crate) id: uuid::Uuid,
+    pub(crate) start_time: std::time::Instant,
 }
 
 unsafe impl Send for CurrentPlugin {}
@@ -52,7 +56,12 @@ impl wasmtime::ResourceLimiter for MemoryLimiter {
         Ok(true)
     }
 
-    fn table_growing(&mut self, _current: u32, desired: u32, maximum: Option<u32>) -> Result<bool> {
+    fn table_growing(
+        &mut self,
+        _current: usize,
+        desired: usize,
+        maximum: Option<usize>,
+    ) -> Result<bool> {
         if let Some(max) = maximum {
             return Ok(desired <= max);
         }
@@ -62,6 +71,11 @@ impl wasmtime::ResourceLimiter for MemoryLimiter {
 }
 
 impl CurrentPlugin {
+    /// Gets `Plugin`'s ID
+    pub fn id(&self) -> uuid::Uuid {
+        self.id
+    }
+
     /// Get a `MemoryHandle` from a memory offset
     pub fn memory_handle(&mut self, offs: u64) -> Option<MemoryHandle> {
         if offs == 0 {
@@ -154,10 +168,10 @@ impl CurrentPlugin {
     }
 
     pub fn memory_bytes_mut(&mut self, handle: MemoryHandle) -> Result<&mut [u8], Error> {
-        let (linker, mut store) = self.linker_and_store();
-        if let Some(mem) = linker.get(&mut store, EXTISM_ENV_MODULE, "memory") {
+        let (linker, store) = self.linker_and_store();
+        if let Some(mem) = linker.get(&mut *store, EXTISM_ENV_MODULE, "memory") {
             let mem = mem.into_memory().unwrap();
-            let ptr = unsafe { mem.data_ptr(&store).add(handle.offset() as usize) };
+            let ptr = unsafe { mem.data_ptr(&*store).add(handle.offset() as usize) };
             if ptr.is_null() {
                 return Ok(&mut []);
             }
@@ -168,10 +182,10 @@ impl CurrentPlugin {
     }
 
     pub fn memory_bytes(&mut self, handle: MemoryHandle) -> Result<&[u8], Error> {
-        let (linker, mut store) = self.linker_and_store();
-        if let Some(mem) = linker.get(&mut store, EXTISM_ENV_MODULE, "memory") {
+        let (linker, store) = self.linker_and_store();
+        if let Some(mem) = linker.get(&mut *store, EXTISM_ENV_MODULE, "memory") {
             let mem = mem.into_memory().unwrap();
-            let ptr = unsafe { mem.data_ptr(&store).add(handle.offset() as usize) };
+            let ptr = unsafe { mem.data_ptr(&*store).add(handle.offset() as usize) };
             if ptr.is_null() {
                 return Ok(&[]);
             }
@@ -181,6 +195,30 @@ impl CurrentPlugin {
         anyhow::bail!("{} unable to locate extism memory", self.id)
     }
 
+    pub fn host_context<T: 'static>(&mut self) -> Result<&mut T, Error> {
+        let (linker, store) = self.linker_and_store();
+        let Some(Extern::Global(xs)) = linker.get(&mut *store, EXTISM_ENV_MODULE, "extism_context")
+        else {
+            anyhow::bail!("unable to locate an extism kernel global: extism_context",)
+        };
+
+        let Val::ExternRef(Some(xs)) = xs.get(&mut *store) else {
+            anyhow::bail!("expected extism_context to be an externref value",)
+        };
+
+        if let Some(d) = xs.data_mut(&mut *store)? {
+            match d.downcast_mut::<Box<dyn std::any::Any + Send + Sync>>() {
+                Some(xs) => match xs.downcast_mut::<T>() {
+                    Some(xs) => Ok(xs),
+                    None => anyhow::bail!("could not downcast extism_context inner value"),
+                },
+                None => anyhow::bail!("could not downcast extism_context"),
+            }
+        } else {
+            anyhow::bail!("extism_context not found")
+        }
+    }
+
     pub fn memory_alloc(&mut self, n: u64) -> Result<MemoryHandle, Error> {
         if n == 0 {
             return Ok(MemoryHandle {
@@ -191,9 +229,13 @@ impl CurrentPlugin {
         let (linker, mut store) = self.linker_and_store();
         let output = &mut [Val::I64(0)];
         if let Some(f) = linker.get(&mut store, EXTISM_ENV_MODULE, "alloc") {
-            f.into_func()
-                .unwrap()
-                .call(&mut store, &[Val::I64(n as i64)], output)?;
+            catch_out_of_fuel!(
+                &store,
+                f.into_func()
+                    .unwrap()
+                    .call(&mut *store, &[Val::I64(n as i64)], output)
+                    .context("failed to allocate extism memory")
+            )?;
         } else {
             anyhow::bail!("{} unable to allocate memory", self.id);
         }
@@ -215,11 +257,15 @@ impl CurrentPlugin {
 
     /// Free a block of Extism plugin memory
     pub fn memory_free(&mut self, handle: MemoryHandle) -> Result<(), Error> {
-        let (linker, mut store) = self.linker_and_store();
-        if let Some(f) = linker.get(&mut store, EXTISM_ENV_MODULE, "free") {
-            f.into_func()
-                .unwrap()
-                .call(&mut store, &[Val::I64(handle.offset as i64)], &mut [])?;
+        let (linker, store) = self.linker_and_store();
+        if let Some(f) = linker.get(&mut *store, EXTISM_ENV_MODULE, "free") {
+            catch_out_of_fuel!(
+                &store,
+                f.into_func()
+                    .unwrap()
+                    .call(&mut *store, &[Val::I64(handle.offset as i64)], &mut [])
+                    .context("failed to free extism memory")
+            )?;
         } else {
             anyhow::bail!("unable to locate an extism kernel function: free",)
         }
@@ -227,12 +273,16 @@ impl CurrentPlugin {
     }
 
     pub fn memory_length(&mut self, offs: u64) -> Result<u64, Error> {
-        let (linker, mut store) = self.linker_and_store();
+        let (linker, store) = self.linker_and_store();
         let output = &mut [Val::I64(0)];
-        if let Some(f) = linker.get(&mut store, EXTISM_ENV_MODULE, "length") {
-            f.into_func()
-                .unwrap()
-                .call(&mut store, &[Val::I64(offs as i64)], output)?;
+        if let Some(f) = linker.get(&mut *store, EXTISM_ENV_MODULE, "length") {
+            catch_out_of_fuel!(
+                &store,
+                f.into_func()
+                    .unwrap()
+                    .call(&mut *store, &[Val::I64(offs as i64)], output)
+                    .context("failed to get length of extism memory handle")
+            )?;
         } else {
             anyhow::bail!("unable to locate an extism kernel function: length",)
         }
@@ -246,6 +296,30 @@ impl CurrentPlugin {
         Ok(len)
     }
 
+    pub fn memory_length_unsafe(&mut self, offs: u64) -> Result<u64, Error> {
+        let (linker, store) = self.linker_and_store();
+        let output = &mut [Val::I64(0)];
+        if let Some(f) = linker.get(&mut *store, EXTISM_ENV_MODULE, "length_unsafe") {
+            catch_out_of_fuel!(
+                &store,
+                f.into_func()
+                    .unwrap()
+                    .call(&mut *store, &[Val::I64(offs as i64)], output)
+                    .context("failed to get length of extism memory using length_unsafe")
+            )?;
+        } else {
+            anyhow::bail!("unable to locate an extism kernel function: length_unsafe",)
+        }
+        let len = output[0].unwrap_i64() as u64;
+        trace!(
+            plugin = self.id.to_string(),
+            "memory_length_unsafe({}) = {}",
+            offs,
+            len
+        );
+        Ok(len)
+    }
+
     /// Access a plugin's variables
     pub fn vars(&self) -> &std::collections::BTreeMap<String, Vec<u8>> {
         &self.vars
@@ -265,35 +339,51 @@ impl CurrentPlugin {
         manifest: extism_manifest::Manifest,
         wasi: bool,
         available_pages: Option<u32>,
+        allow_http_response_headers: bool,
         id: uuid::Uuid,
     ) -> Result<Self, Error> {
         let wasi = if wasi {
-            let auth = wasmtime_wasi::ambient_authority();
-            let mut ctx = wasmtime_wasi::WasiCtxBuilder::new();
-            for (k, v) in manifest.config.iter() {
-                ctx.env(k, v)?;
-            }
+            let auth = wasi_common::sync::ambient_authority();
+            let random = wasi_common::sync::random_ctx();
+            let clocks = wasi_common::sync::clocks_ctx();
+            let sched = wasi_common::sync::sched_ctx();
+            let table = wasi_common::Table::new();
+            let ctx = wasi_common::WasiCtx::new(random, clocks, sched, table);
 
             if let Some(a) = &manifest.allowed_paths {
                 for (k, v) in a.iter() {
-                    let d = wasmtime_wasi::Dir::open_ambient_dir(k, auth)?;
-                    ctx.preopened_dir(d, v)?;
+                    let readonly = matches!(v, extism_manifest::LocalPath::ReadOnly(_));
+
+                    let dir_path = v.as_path();
+
+                    let dir = wasi_common::sync::dir::Dir::from_cap_std(
+                        wasi_common::sync::Dir::open_ambient_dir(dir_path, auth)?,
+                    );
+
+                    let file: Box<dyn wasi_common::dir::WasiDir> = if readonly {
+                        Box::new(readonly_dir::ReadOnlyDir::new(dir))
+                    } else {
+                        Box::new(dir)
+                    };
+
+                    ctx.push_preopened_dir(file, k)?;
                 }
             }
 
             // Enable WASI output, typically used for debugging purposes
             if std::env::var("EXTISM_ENABLE_WASI_OUTPUT").is_ok() {
-                ctx.inherit_stdout().inherit_stderr();
+                ctx.set_stderr(Box::new(wasi_common::sync::stdio::stderr()));
+                ctx.set_stdout(Box::new(wasi_common::sync::stdio::stdout()));
             }
 
-            Some(Wasi { ctx: ctx.build() })
+            Some(Wasi { ctx })
         } else {
             None
         };
 
         let memory_limiter = if let Some(pgs) = available_pages {
             let n = pgs as usize * 65536;
-            Some(crate::current_plugin::MemoryLimiter {
+            Some(MemoryLimiter {
                 max_bytes: n,
                 bytes_left: n,
             })
@@ -311,6 +401,12 @@ impl CurrentPlugin {
             available_pages,
             memory_limiter,
             id,
+            start_time: std::time::Instant::now(),
+            http_headers: if allow_http_response_headers {
+                Some(BTreeMap::new())
+            } else {
+                None
+            },
         })
     }
 
@@ -355,12 +451,12 @@ impl CurrentPlugin {
     /// Clear the current plugin error
     pub fn clear_error(&mut self) {
         trace!(plugin = self.id.to_string(), "CurrentPlugin::clear_error");
-        let (linker, mut store) = self.linker_and_store();
-        if let Some(f) = linker.get(&mut store, EXTISM_ENV_MODULE, "error_set") {
+        let (linker, store) = self.linker_and_store();
+        if let Some(f) = linker.get(&mut *store, EXTISM_ENV_MODULE, "error_set") {
             let res = f
                 .into_func()
                 .unwrap()
-                .call(&mut store, &[Val::I64(0)], &mut []);
+                .call(&mut *store, &[Val::I64(0)], &mut []);
             if let Err(e) = res {
                 error!(
                     plugin = self.id.to_string(),
@@ -381,23 +477,22 @@ impl CurrentPlugin {
             offset: offs,
             length,
         });
-        match s {
-            Ok(s) => Some(s),
-            Err(_) => None,
-        }
+        s.ok()
     }
 
     #[doc(hidden)]
     pub fn set_error(&mut self, s: impl AsRef<str>) -> Result<(u64, u64), Error> {
         let s = s.as_ref();
         debug!(plugin = self.id.to_string(), "set error: {:?}", s);
-        let handle = self.current_plugin_mut().memory_new(s)?;
-        let (linker, mut store) = self.linker_and_store();
-        if let Some(f) = linker.get(&mut store, EXTISM_ENV_MODULE, "error_set") {
-            f.into_func().unwrap().call(
-                &mut store,
-                &[Val::I64(handle.offset() as i64)],
-                &mut [],
+        let handle = self.memory_new(s)?;
+        let (linker, store) = self.linker_and_store();
+        if let Some(f) = linker.get(&mut *store, EXTISM_ENV_MODULE, "error_set") {
+            catch_out_of_fuel!(
+                &store,
+                f.into_func()
+                    .unwrap()
+                    .call(&mut *store, &[Val::I64(handle.offset() as i64)], &mut [])
+                    .context("failed to set extism error")
             )?;
             Ok((handle.offset(), s.len() as u64))
         } else {
@@ -406,10 +501,13 @@ impl CurrentPlugin {
     }
 
     pub(crate) fn get_error_position(&mut self) -> (u64, u64) {
-        let (linker, mut store) = self.linker_and_store();
+        let (linker, store) = self.linker_and_store();
         let output = &mut [Val::I64(0)];
-        if let Some(f) = linker.get(&mut store, EXTISM_ENV_MODULE, "error_get") {
-            if let Err(e) = f.into_func().unwrap().call(&mut store, &[], output) {
+        if let Some(f) = linker.get(&mut *store, EXTISM_ENV_MODULE, "error_get") {
+            if let Err(e) = catch_out_of_fuel!(
+                &store,
+                f.into_func().unwrap().call(&mut *store, &[], output)
+            ) {
                 error!(
                     plugin = self.id.to_string(),
                     "unable to call extism:host/env::error_get: {:?}", e
@@ -421,6 +519,18 @@ impl CurrentPlugin {
         let length = self.memory_length(offs).unwrap_or_default();
         (offs, length)
     }
+
+    /// Returns the remaining time before a plugin will timeout, or
+    /// `None` if no timeout is configured in the manifest
+    pub fn time_remaining(&self) -> Option<std::time::Duration> {
+        if let Some(x) = &self.manifest.timeout_ms {
+            let elapsed = &self.start_time.elapsed().as_millis();
+            let ms_left = x.saturating_sub(*elapsed as u64);
+            return Some(std::time::Duration::from_millis(ms_left));
+        }
+
+        None
+    }
 }
 
 impl Internal for CurrentPlugin {
@@ -432,14 +542,6 @@ impl Internal for CurrentPlugin {
         unsafe { &mut *self.store }
     }
 
-    fn linker(&self) -> &Linker<CurrentPlugin> {
-        unsafe { &*self.linker }
-    }
-
-    fn linker_mut(&mut self) -> &mut Linker<CurrentPlugin> {
-        unsafe { &mut *self.linker }
-    }
-
     fn linker_and_store(&mut self) -> (&mut Linker<CurrentPlugin>, &mut Store<CurrentPlugin>) {
         unsafe { (&mut *self.linker, &mut *self.store) }
     }

runtime/src/function.rs

@@ -4,6 +4,7 @@ use wasmtime::Caller;
 use crate::{error, trace, CurrentPlugin, Error};
 
 /// An enumeration of all possible value types in WebAssembly.
+/// cbindgen:prefix-with-name
 #[derive(Debug, Clone, Hash, Eq, PartialEq)]
 #[repr(C)]
 pub enum ValType {
@@ -37,8 +38,13 @@ impl From<wasmtime::ValType> for ValType {
             F32 => ValType::F32,
             F64 => ValType::F64,
             V128 => ValType::V128,
-            FuncRef => ValType::FuncRef,
-            ExternRef => ValType::ExternRef,
+            Ref(t) => {
+                if t.heap_type().is_func() {
+                    ValType::FuncRef
+                } else {
+                    ValType::ExternRef
+                }
+            }
         }
     }
 }
@@ -52,8 +58,8 @@ impl From<ValType> for wasmtime::ValType {
             F32 => wasmtime::ValType::F32,
             F64 => wasmtime::ValType::F64,
             V128 => wasmtime::ValType::V128,
-            FuncRef => wasmtime::ValType::FuncRef,
-            ExternRef => wasmtime::ValType::ExternRef,
+            FuncRef => wasmtime::ValType::FUNCREF,
+            ExternRef => wasmtime::ValType::EXTERNREF,
         }
     }
 }
@@ -71,7 +77,9 @@ pub struct CPtr {
 /// UserDataHandle is an untyped version of `UserData` that is stored inside `Function` to keep a live reference.
 #[derive(Clone)]
 pub(crate) enum UserDataHandle {
+    #[allow(dead_code)]
     C(Arc<CPtr>),
+    #[allow(dead_code)]
     Rust(Arc<std::sync::Mutex<dyn std::any::Any>>),
 }
 
@@ -168,8 +176,8 @@ pub struct Function {
     /// Module name
     pub(crate) namespace: Option<String>,
 
-    /// Function type
-    pub(crate) ty: wasmtime::FuncType,
+    pub(crate) params: Vec<ValType>,
+    pub(crate) results: Vec<ValType>,
 
     /// Function handle
     pub(crate) f: Arc<FunctionInner>,
@@ -182,8 +190,8 @@ impl Function {
     /// Create a new host function
     pub fn new<T: 'static, F>(
         name: impl Into<String>,
-        args: impl IntoIterator<Item = ValType>,
-        returns: impl IntoIterator<Item = ValType>,
+        params: impl IntoIterator<Item = ValType>,
+        results: impl IntoIterator<Item = ValType>,
         user_data: UserData<T>,
         f: F,
     ) -> Function
@@ -195,13 +203,13 @@ impl Function {
     {
         let data = user_data.clone();
         let name = name.into();
-        let args = args.into_iter().map(wasmtime::ValType::from);
-        let returns = returns.into_iter().map(wasmtime::ValType::from);
-        let ty = wasmtime::FuncType::new(args, returns);
-        trace!("Creating function {name}: type={ty:?}");
+        let params = params.into_iter().collect();
+        let results = results.into_iter().collect();
+        trace!("Creating function {name}: params={params:?}, results={results:?}");
         Function {
             name,
-            ty,
+            params,
+            results,
             f: Arc::new(
                 move |mut caller: Caller<_>, inp: &[Val], outp: &mut [Val]| {
                     let x = data.clone();
@@ -216,6 +224,22 @@ impl Function {
         }
     }
 
+    pub(crate) fn ty(&self, engine: &wasmtime::Engine) -> wasmtime::FuncType {
+        wasmtime::FuncType::new(
+            engine,
+            self.params
+                .iter()
+                .cloned()
+                .map(wasmtime::ValType::from)
+                .collect::<Vec<_>>(),
+            self.results
+                .iter()
+                .cloned()
+                .map(wasmtime::ValType::from)
+                .collect::<Vec<_>>(),
+        )
+    }
+
     /// Host function name
     pub fn name(&self) -> &str {
         &self.name
@@ -239,9 +263,14 @@ impl Function {
         self
     }
 
-    /// Get function type
-    pub fn ty(&self) -> &wasmtime::FuncType {
-        &self.ty
+    /// Get param types
+    pub fn params(&self) -> &[ValType] {
+        &self.params
+    }
+
+    /// Get result types
+    pub fn results(&self) -> &[ValType] {
+        &self.results
     }
 }
 
@@ -250,13 +279,13 @@ impl Function {
 /// For example, the following defines a host function named `add_newline` that takes a
 /// string parameter and returns a string result:
 /// ```rust
-/// extism::host_fn!(add_newline(_user_data: (), a: String) -> String { Ok(a + "\n") });
+/// extism::host_fn!(add_newline(_user_data: (); a: String) -> String { Ok(a + "\n") });
 /// ```
 /// A few things worth noting:
 /// - The function always returns a `Result` that wraps the specified return type
 /// - If a first parameter and type are passed (`_user_data` above) followed by a semicolon it will be
-///    the name of the `UserData` parameter and can be used from inside the function
-//     definition.
+///   the name of the `UserData` parameter and can be used from inside the function
+//    definition.
 #[macro_export]
 macro_rules! host_fn {
     ($pub:vis $name: ident  ($($arg:ident : $argty:ty),*) $(-> $ret:ty)? $b:block) => {

runtime/src/internal.rs

@@ -3,7 +3,7 @@ use crate::*;
 /// WASI context
 pub struct Wasi {
     /// wasi
-    pub ctx: wasmtime_wasi::WasiCtx,
+    pub ctx: wasi_common::WasiCtx,
 }
 
 /// InternalExt provides a unified way of acessing `memory`, `store` and `internal` values
@@ -12,10 +12,6 @@ pub(crate) trait Internal {
 
     fn store_mut(&mut self) -> &mut Store<CurrentPlugin>;
 
-    fn linker(&self) -> &Linker<CurrentPlugin>;
-
-    fn linker_mut(&mut self) -> &mut Linker<CurrentPlugin>;
-
     fn linker_and_store(&mut self) -> (&mut Linker<CurrentPlugin>, &mut Store<CurrentPlugin>);
 
     fn current_plugin(&self) -> &CurrentPlugin {

runtime/src/lib.rs

@@ -1,8 +1,23 @@
+// Makes proc-macros able to resolve `::extism` correctly
+extern crate self as extism;
+
+macro_rules! catch_out_of_fuel {
+    ($store: expr, $x:expr) => {{
+        let y = $x;
+        if y.is_err() && $store.get_fuel().is_ok_and(|x| x == 0) {
+            Err(Error::msg("plugin ran out of fuel"))
+        } else {
+            y
+        }
+    }};
+}
+
 pub(crate) use extism_convert::*;
 pub(crate) use std::collections::BTreeMap;
 use std::str::FromStr;
 pub(crate) use wasmtime::*;
 
+#[doc(inline)]
 pub use extism_convert as convert;
 
 pub use anyhow::Error;
@@ -14,6 +29,8 @@ pub(crate) mod manifest;
 pub(crate) mod pdk;
 mod plugin;
 mod plugin_builder;
+mod pool;
+mod readonly_dir;
 mod timer;
 
 /// Extism C API
@@ -23,8 +40,11 @@ pub use current_plugin::CurrentPlugin;
 pub use extism_convert::{FromBytes, FromBytesOwned, ToBytes};
 pub use extism_manifest::{Manifest, Wasm, WasmMetadata};
 pub use function::{Function, UserData, Val, ValType, PTR};
-pub use plugin::{CancelHandle, Plugin, WasmInput, EXTISM_ENV_MODULE, EXTISM_USER_MODULE};
+pub use plugin::{
+    CancelHandle, CompiledPlugin, Plugin, WasmInput, EXTISM_ENV_MODULE, EXTISM_USER_MODULE,
+};
 pub use plugin_builder::{DebugOptions, PluginBuilder};
+pub use pool::{Pool, PoolBuilder, PoolPlugin};
 
 pub(crate) use internal::{Internal, Wasi};
 pub(crate) use timer::{Timer, TimerAction};
@@ -73,11 +93,16 @@ pub fn set_log_callback<F: 'static + Clone + Fn(&str)>(
     filter: impl AsRef<str>,
 ) -> Result<(), Error> {
     let filter = filter.as_ref();
-    let cfg = tracing_subscriber::FmtSubscriber::builder().with_env_filter(
-        tracing_subscriber::EnvFilter::builder()
-            .with_default_directive(tracing::Level::ERROR.into())
-            .parse_lossy(filter),
-    );
+    let is_level = tracing::Level::from_str(filter).is_ok();
+    let cfg = tracing_subscriber::FmtSubscriber::builder().with_env_filter({
+        let x = tracing_subscriber::EnvFilter::builder()
+            .with_default_directive(tracing::Level::ERROR.into());
+        if is_level {
+            x.parse_lossy(format!("extism={}", filter))
+        } else {
+            x.parse_lossy(filter)
+        }
+    });
     let w = LogFunction { func };
     cfg.with_ansi(false)
         .with_writer(move || w.clone())

runtime/src/manifest.rs

@@ -47,9 +47,13 @@ fn to_module(engine: &Engine, wasm: &extism_manifest::Wasm) -> Result<(String, M
             let name = meta.name.as_deref().unwrap_or(MAIN_KEY).to_string();
 
             // Load file
-            let mut buf = Vec::new();
-            let mut file = std::fs::File::open(path)?;
-            file.read_to_end(&mut buf)?;
+            let buf = std::fs::read(path).map_err(|err| {
+                Error::msg(format!(
+                    "Unable to load Wasm file \"{}\": {}",
+                    path.display(),
+                    err.kind()
+                ))
+            })?;
 
             check_hash(&meta.hash, &buf)?;
             Ok((name, Module::new(engine, buf)?))
@@ -82,14 +86,16 @@ fn to_module(engine: &Engine, wasm: &extism_manifest::Wasm) -> Result<(String, M
             #[cfg(feature = "register-http")]
             {
                 // Setup request
-                let mut req = ureq::request(method.as_deref().unwrap_or("GET"), url);
+                let mut req = ureq::http::request::Builder::new()
+                    .method(method.as_deref().unwrap_or("GET").to_uppercase().as_str())
+                    .uri(url);
 
                 for (k, v) in headers.iter() {
-                    req = req.set(k, v);
+                    req = req.header(k, v);
                 }
 
                 // Fetch WASM code
-                let mut r = req.call()?.into_reader();
+                let mut r = ureq::run(req.body(())?)?.into_body().into_reader();
                 let mut data = Vec::new();
                 r.read_to_end(&mut data)?;
 
@@ -117,10 +123,17 @@ pub(crate) fn load(
     match input {
         WasmInput::Data(data) => {
             let has_magic = data.len() >= 4 && data[0..4] == WASM_MAGIC;
-            let is_wat = data.starts_with(b"(module") || data.starts_with(b";;");
+            let s = std::str::from_utf8(&data);
+            let is_wat = s.is_ok_and(|s| {
+                let s = s.trim_start();
+                let starts_with_module = s.len() > 2
+                    && data[0] == b'('   // First character is `(`
+                    && s[1..].trim_start().starts_with("module"); // Then `module` (after any whitespace)
+                starts_with_module || s.starts_with(";;") || s.starts_with("(;")
+            });
             if !has_magic && !is_wat {
                 trace!("Loading manifest");
-                if let Ok(s) = std::str::from_utf8(&data) {
+                if let Ok(s) = s {
                     let t = if let Ok(t) = toml::from_str::<extism_manifest::Manifest>(s) {
                         trace!("Manifest is TOML");
                         modules(engine, &t, &mut mods)?;

runtime/src/pdk.rs

@@ -20,6 +20,8 @@ macro_rules! args {
 /// Get a configuration value
 /// Params: i64 (offset)
 /// Returns: i64 (offset)
+/// **Note**: this function takes ownership of the handle passed in
+/// the caller should not `free` this value
 pub(crate) fn config_get(
     mut caller: Caller<CurrentPlugin>,
     input: &[Val],
@@ -38,6 +40,7 @@ pub(crate) fn config_get(
     };
     let val = data.manifest.config.get(key);
     let ptr = val.map(|x| (x.len(), x.as_ptr()));
+    data.memory_free(handle)?;
     let mem = match ptr {
         Some((len, ptr)) => {
             let bytes = unsafe { std::slice::from_raw_parts(ptr, len) };
@@ -55,6 +58,9 @@ pub(crate) fn config_get(
 /// Get a variable
 /// Params: i64 (offset)
 /// Returns: i64 (offset)
+/// **Note**: this function takes ownership of the handle passed in
+/// the caller should not `free` this value, but the return value
+/// will need to be freed
 pub(crate) fn var_get(
     mut caller: Caller<CurrentPlugin>,
     input: &[Val],
@@ -73,6 +79,8 @@ pub(crate) fn var_get(
     };
     let val = data.vars.get(key);
     let ptr = val.map(|x| (x.len(), x.as_ptr()));
+    data.memory_free(handle)?;
+
     let mem = match ptr {
         Some((len, ptr)) => {
             let bytes = unsafe { std::slice::from_raw_parts(ptr, len) };
@@ -90,6 +98,8 @@ pub(crate) fn var_get(
 /// Set a variable, if the value offset is 0 then the provided key will be removed
 /// Params: i64 (key offset), i64 (value offset)
 /// Returns: none
+/// **Note**: this function takes ownership of the handles passed in
+/// the caller should not `free` these values
 pub(crate) fn var_set(
     mut caller: Caller<CurrentPlugin>,
     input: &[Val],
@@ -97,25 +107,19 @@ pub(crate) fn var_set(
 ) -> Result<(), Error> {
     let data: &mut CurrentPlugin = caller.data_mut();
 
-    let mut size = 0;
-    for v in data.vars.values() {
-        size += v.len();
+    if data.manifest.memory.max_var_bytes.is_some_and(|x| x == 0) {
+        anyhow::bail!("Vars are disabled by this host")
     }
 
     let voffset = args!(input, 1, i64) as u64;
-
-    // If the store is larger than 100MB then stop adding things
-    if size > 1024 * 1024 * 100 && voffset != 0 {
-        return Err(Error::msg("Variable store is full"));
-    }
-
     let key_offs = args!(input, 0, i64) as u64;
+
+    let key_handle = match data.memory_handle(key_offs) {
+        Some(h) => h,
+        None => anyhow::bail!("invalid handle offset for var key: {key_offs}"),
+    };
     let key = {
-        let handle = match data.memory_handle(key_offs) {
-            Some(h) => h,
-            None => anyhow::bail!("invalid handle offset for var key: {key_offs}"),
-        };
-        let key = data.memory_str(handle)?;
+        let key = data.memory_str(key_handle)?;
         let key_len = key.len();
         let key_ptr = key.as_ptr();
         unsafe { std::str::from_utf8_unchecked(std::slice::from_raw_parts(key_ptr, key_len)) }
@@ -124,6 +128,7 @@ pub(crate) fn var_set(
     // Remove if the value offset is 0
     if voffset == 0 {
         data.vars.remove(key);
+        data.memory_free(key_handle)?;
         return Ok(());
     }
 
@@ -132,8 +137,27 @@ pub(crate) fn var_set(
         None => anyhow::bail!("invalid handle offset for var value: {voffset}"),
     };
 
+    let mut size = std::mem::size_of::<String>()
+        + std::mem::size_of::<Vec<u8>>()
+        + key.len()
+        + handle.length as usize;
+
+    for (k, v) in data.vars.iter() {
+        size += k.len();
+        size += v.len();
+        size += std::mem::size_of::<String>() + std::mem::size_of::<Vec<u8>>();
+    }
+
+    // If the store is larger than the configured size, or 1mb by default, then stop adding things
+    if size > data.manifest.memory.max_var_bytes.unwrap_or(1024 * 1024) as usize && voffset != 0 {
+        return Err(Error::msg("Variable store is full"));
+    }
+
     let value = data.memory_bytes(handle)?.to_vec();
 
+    data.memory_free(handle)?;
+    data.memory_free(key_handle)?;
+
     // Insert the value from memory into the `vars` map
     data.vars.insert(key.to_string(), value);
 
@@ -143,6 +167,9 @@ pub(crate) fn var_set(
 /// Make an HTTP request
 /// Params: i64 (offset to JSON encoded HttpRequest), i64 (offset to body or 0)
 /// Returns: i64 (offset)
+/// **Note**: this function takes ownership of the handles passed in
+/// the caller should not `free` these values, the result will need to
+/// be freed.
 pub(crate) fn http_request(
     #[allow(unused_mut)] mut caller: Caller<CurrentPlugin>,
     input: &[Val],
@@ -156,6 +183,7 @@ pub(crate) fn http_request(
             Some(h) => h,
             None => anyhow::bail!("http_request input is invalid: {http_req_offset}"),
         };
+        data.memory_free(handle)?;
         let req: extism_manifest::HttpRequest = serde_json::from_slice(data.memory_bytes(handle)?)?;
         output[0] = Val::I64(0);
         anyhow::bail!(
@@ -166,12 +194,16 @@ pub(crate) fn http_request(
 
     #[cfg(feature = "http")]
     {
+        data.http_headers.iter_mut().for_each(|x| x.clear());
+        data.http_status = 0;
+
         use std::io::Read;
         let handle = match data.memory_handle(http_req_offset) {
             Some(h) => h,
             None => anyhow::bail!("invalid handle offset for http request: {http_req_offset}"),
         };
         let req: extism_manifest::HttpRequest = serde_json::from_slice(data.memory_bytes(handle)?)?;
+        data.memory_free(handle)?;
 
         let body_offset = args!(input, 1, i64) as u64;
 
@@ -201,12 +233,22 @@ pub(crate) fn http_request(
             )));
         }
 
-        let mut r = ureq::request(req.method.as_deref().unwrap_or("GET"), &req.url);
+        let mut r = ureq::http::request::Builder::new()
+            .method(
+                req.method
+                    .as_deref()
+                    .unwrap_or("GET")
+                    .to_uppercase()
+                    .as_str(),
+            )
+            .uri(&req.url);
 
         for (k, v) in req.headers.iter() {
-            r = r.set(k, v);
+            r = r.header(k, v);
         }
 
+        // Set HTTP timeout to respect the manifest timeout
+        let timeout = data.time_remaining();
         let res = if body_offset > 0 {
             let handle = match data.memory_handle(body_offset) {
                 Some(h) => h,
@@ -215,31 +257,63 @@ pub(crate) fn http_request(
                 }
             };
             let buf: &[u8] = data.memory_bytes(handle)?;
-            r.send_bytes(buf)
+            let agent = ureq::agent();
+            let config = agent.configure_request(r.body(buf)?);
+            let req = config.timeout_global(timeout).build();
+            ureq::run(req)
         } else {
-            r.call()
+            let agent = ureq::agent();
+            let config = agent.configure_request(r.body(())?);
+            let req = config.timeout_global(timeout).build();
+            ureq::run(req)
         };
 
+        if let Some(handle) = data.memory_handle(body_offset) {
+            data.memory_free(handle)?;
+        }
+
         let reader = match res {
             Ok(res) => {
-                data.http_status = res.status();
-                Some(res.into_reader())
+                if let Some(headers) = &mut data.http_headers {
+                    for (name, h) in res.headers() {
+                        if let Ok(h) = h.to_str() {
+                            headers.insert(name.as_str().to_string(), h.to_string());
+                        }
+                    }
+                }
+                data.http_status = res.status().as_u16();
+                Some(res.into_body().into_reader())
             }
             Err(e) => {
-                if let Some(res) = e.into_response() {
-                    data.http_status = res.status();
-                    Some(res.into_reader())
-                } else {
+                // Catch timeout and return
+                if let Some(d) = data.time_remaining() {
+                    if matches!(e, ureq::Error::Timeout(_)) && d.as_nanos() == 0 {
+                        anyhow::bail!("timeout");
+                    }
+                }
+                let msg = e.to_string();
+                if let ureq::Error::StatusCode(res) = e {
+                    data.http_status = res;
                     None
+                } else {
+                    return Err(Error::msg(msg));
                 }
             }
         };
 
         if let Some(reader) = reader {
             let mut buf = Vec::new();
-            reader
-                .take(1024 * 1024 * 50) // TODO: make this limit configurable
-                .read_to_end(&mut buf)?;
+            let max = if let Some(max) = &data.manifest.memory.max_http_response_bytes {
+                reader.take(*max + 1).read_to_end(&mut buf)?;
+                *max
+            } else {
+                reader.take(1024 * 1024 * 50 + 1).read_to_end(&mut buf)?;
+                1024 * 1024 * 50
+            };
+
+            if buf.len() > max as usize {
+                anyhow::bail!("HTTP response exceeds the configured maximum number of bytes: {max}")
+            }
 
             let mem = data.memory_new(&buf)?;
             output[0] = Val::I64(mem.offset() as i64);
@@ -264,6 +338,24 @@ pub(crate) fn http_status_code(
     Ok(())
 }
 
+/// Get the HTTP response headers from the last HTTP request
+/// Params: none
+/// Returns: i64 (offset)
+pub(crate) fn http_headers(
+    mut caller: Caller<CurrentPlugin>,
+    _input: &[Val],
+    output: &mut [Val],
+) -> Result<(), Error> {
+    let data: &mut CurrentPlugin = caller.data_mut();
+    if let Some(h) = &data.http_headers {
+        let headers = serde_json::to_string(h)?;
+        data.memory_set_val(&mut output[0], headers)?;
+    } else {
+        output[0] = Val::I64(0);
+    }
+    Ok(())
+}
+
 pub fn log(
     level: tracing::Level,
     mut caller: Caller<CurrentPlugin>,
@@ -271,13 +363,22 @@ pub fn log(
     _output: &mut [Val],
 ) -> Result<(), Error> {
     let data: &mut CurrentPlugin = caller.data_mut();
+
     let offset = args!(input, 0, i64) as u64;
 
+    // Check if the current log level should be logged
+    let global_log_level = tracing::level_filters::LevelFilter::current();
+    if global_log_level == tracing::level_filters::LevelFilter::OFF || level > global_log_level {
+        if let Some(handle) = data.memory_handle(offset) {
+            data.memory_free(handle)?;
+        }
+        return Ok(());
+    }
+
     let handle = match data.memory_handle(offset) {
         Some(h) => h,
         None => anyhow::bail!("invalid handle offset for log message: {offset}"),
     };
-
     let id = data.id.to_string();
     let buf = data.memory_str(handle);
 
@@ -301,12 +402,16 @@ pub fn log(
         },
         Err(_) => tracing::error!(plugin = id, "unable to log message: {:?}", buf),
     }
+
+    data.memory_free(handle)?;
     Ok(())
 }
 
 /// Write to logs (warning)
 /// Params: i64 (offset)
 /// Returns: none
+/// **Note**: this function takes ownership of the handle passed in
+/// the caller should not `free` this value
 pub(crate) fn log_warn(
     caller: Caller<CurrentPlugin>,
     input: &[Val],
@@ -318,6 +423,8 @@ pub(crate) fn log_warn(
 /// Write to logs (info)
 /// Params: i64 (offset)
 /// Returns: none
+/// **Note**: this function takes ownership of the handle passed in
+/// the caller should not `free` this value
 pub(crate) fn log_info(
     caller: Caller<CurrentPlugin>,
     input: &[Val],
@@ -329,6 +436,8 @@ pub(crate) fn log_info(
 /// Write to logs (debug)
 /// Params: i64 (offset)
 /// Returns: none
+/// **Note**: this function takes ownership of the handle passed in
+/// the caller should not `free` this value
 pub(crate) fn log_debug(
     caller: Caller<CurrentPlugin>,
     input: &[Val],
@@ -340,6 +449,8 @@ pub(crate) fn log_debug(
 /// Write to logs (error)
 /// Params: i64 (offset)
 /// Returns: none
+/// **Note**: this function takes ownership of the handle passed in
+/// the caller should not `free` this value
 pub(crate) fn log_error(
     caller: Caller<CurrentPlugin>,
     input: &[Val],
@@ -347,3 +458,46 @@ pub(crate) fn log_error(
 ) -> Result<(), Error> {
     log(tracing::Level::ERROR, caller, input, _output)
 }
+
+/// Write to logs (trace)
+/// Params: i64 (offset)
+/// Returns: none
+/// **Note**: this function takes ownership of the handle passed in
+/// the caller should not `free` this value
+pub(crate) fn log_trace(
+    caller: Caller<CurrentPlugin>,
+    input: &[Val],
+    _output: &mut [Val],
+) -> Result<(), Error> {
+    log(tracing::Level::TRACE, caller, input, _output)
+}
+
+/// Get the log level
+/// Params: none
+/// Returns: i32 (log level)
+pub(crate) fn get_log_level(
+    mut _caller: Caller<CurrentPlugin>,
+    _input: &[Val],
+    output: &mut [Val],
+) -> Result<(), Error> {
+    let level = tracing::level_filters::LevelFilter::current();
+    if level == tracing::level_filters::LevelFilter::OFF {
+        output[0] = Val::I32(i32::MAX)
+    } else {
+        output[0] = Val::I32(log_level_to_int(
+            level.into_level().unwrap_or(tracing::Level::ERROR),
+        ));
+    }
+    Ok(())
+}
+
+/// Convert log level to integer
+pub(crate) const fn log_level_to_int(level: tracing::Level) -> i32 {
+    match level {
+        tracing::Level::TRACE => 0,
+        tracing::Level::DEBUG => 1,
+        tracing::Level::INFO => 2,
+        tracing::Level::WARN => 3,
+        tracing::Level::ERROR => 4,
+    }
+}

runtime/src/plugin_builder.rs

@@ -33,12 +33,21 @@ impl Default for DebugOptions {
 }
 
 /// PluginBuilder is used to configure and create `Plugin` instances
+#[derive(Clone)]
 pub struct PluginBuilder<'a> {
-    source: WasmInput<'a>,
-    wasi: bool,
-    functions: Vec<Function>,
-    debug_options: DebugOptions,
-    cache_config: Option<Option<PathBuf>>,
+    pub(crate) source: WasmInput<'a>,
+    pub(crate) config: Option<wasmtime::Config>,
+    pub(crate) options: PluginBuilderOptions,
+}
+
+#[derive(Clone)]
+pub(crate) struct PluginBuilderOptions {
+    pub(crate) wasi: bool,
+    pub(crate) functions: Vec<Function>,
+    pub(crate) debug_options: DebugOptions,
+    pub(crate) cache_config: Option<Option<PathBuf>>,
+    pub(crate) fuel: Option<u64>,
+    pub(crate) http_response_headers: bool,
 }
 
 impl<'a> PluginBuilder<'a> {
@@ -46,16 +55,21 @@ impl<'a> PluginBuilder<'a> {
     pub fn new(plugin: impl Into<WasmInput<'a>>) -> Self {
         PluginBuilder {
             source: plugin.into(),
-            wasi: false,
-            functions: vec![],
-            debug_options: DebugOptions::default(),
-            cache_config: None,
+            config: None,
+            options: PluginBuilderOptions {
+                wasi: false,
+                functions: vec![],
+                debug_options: DebugOptions::default(),
+                cache_config: None,
+                fuel: None,
+                http_response_headers: false,
+            },
         }
     }
 
     /// Enables WASI if the argument is set to `true`
     pub fn with_wasi(mut self, wasi: bool) -> Self {
-        self.wasi = wasi;
+        self.options.wasi = wasi;
         self
     }
 
@@ -74,7 +88,8 @@ impl<'a> PluginBuilder<'a> {
             + Sync
             + Send,
     {
-        self.functions
+        self.options
+            .functions
             .push(Function::new(name, args, returns, user_data, f));
         self
     }
@@ -95,67 +110,97 @@ impl<'a> PluginBuilder<'a> {
             + Sync
             + Send,
     {
-        self.functions
+        self.options
+            .functions
             .push(Function::new(name, args, returns, user_data, f).with_namespace(namespace));
         self
     }
 
     /// Add multiple host functions
     pub fn with_functions(mut self, f: impl IntoIterator<Item = Function>) -> Self {
-        self.functions.extend(f);
+        self.options.functions.extend(f);
         self
     }
 
     /// Set profiling strategy
     pub fn with_profiling_strategy(mut self, p: wasmtime::ProfilingStrategy) -> Self {
-        self.debug_options.profiling_strategy = p;
+        self.options.debug_options.profiling_strategy = p;
         self
     }
 
     /// Enable Wasmtime coredump on trap
     pub fn with_coredump(mut self, path: impl Into<std::path::PathBuf>) -> Self {
-        self.debug_options.coredump = Some(path.into());
+        self.options.debug_options.coredump = Some(path.into());
         self
     }
 
     /// Enable Extism memory dump when plugin calls return an error
     pub fn with_memdump(mut self, path: impl Into<std::path::PathBuf>) -> Self {
-        self.debug_options.memdump = Some(path.into());
+        self.options.debug_options.memdump = Some(path.into());
         self
     }
 
     /// Compile with debug info
     pub fn with_debug_info(mut self) -> Self {
-        self.debug_options.debug_info = true;
+        self.options.debug_options.debug_info = true;
         self
     }
 
     /// Configure debug options
     pub fn with_debug_options(mut self, options: DebugOptions) -> Self {
-        self.debug_options = options;
+        self.options.debug_options = options;
         self
     }
 
     /// Set wasmtime compilation cache config path
     pub fn with_cache_config(mut self, dir: impl Into<PathBuf>) -> Self {
-        self.cache_config = Some(Some(dir.into()));
+        self.options.cache_config = Some(Some(dir.into()));
         self
     }
 
     /// Turn wasmtime compilation caching off
     pub fn with_cache_disabled(mut self) -> Self {
-        self.cache_config = Some(None);
+        self.options.cache_config = Some(None);
+        self
+    }
+
+    /// Limit the number of instructions that can be executed
+    pub fn with_fuel_limit(mut self, fuel: u64) -> Self {
+        self.options.fuel = Some(fuel);
+        self
+    }
+
+    /// Configure an initial wasmtime config to be passed to the plugin
+    ///
+    /// **Warning**: some values might be overwritten by the Extism runtime. In particular:
+    /// - async_support
+    /// - epoch_interruption
+    /// - debug_info
+    /// - coredump_on_trap
+    /// - profiler
+    /// - wasm_tail_call
+    /// - wasm_function_references
+    /// - wasm_gc
+    ///
+    /// See the implementation details of [PluginBuilder::build] and [Plugin::build_new] to verify which values are overwritten.
+    pub fn with_wasmtime_config(mut self, config: wasmtime::Config) -> Self {
+        self.config = Some(config);
+        self
+    }
+
+    /// Enables `http_response_headers`, which allows for plugins to access response headers when using `extism:host/env::http_request`
+    pub fn with_http_response_headers(mut self, allow: bool) -> Self {
+        self.options.http_response_headers = allow;
         self
     }
 
     /// Generate a new plugin with the configured settings
     pub fn build(self) -> Result<Plugin, Error> {
-        Plugin::build_new(
-            self.source,
-            self.functions,
-            self.wasi,
-            self.debug_options,
-            self.cache_config,
-        )
+        Plugin::new_from_compiled(&CompiledPlugin::new(self)?)
+    }
+
+    /// Build new `CompiledPlugin`
+    pub fn compile(self) -> Result<CompiledPlugin, Error> {
+        CompiledPlugin::new(self)
     }
 }

runtime/src/pool.rs

@@ -0,0 +1,174 @@
+use crate::{Error, FromBytesOwned, Plugin, ToBytes};
+
+// `PoolBuilder` is used to configure and create `Pool`s
+#[derive(Debug, Clone)]
+pub struct PoolBuilder {
+    /// Max number of concurrent instances for a plugin - by default this is set to
+    /// the output of `std::thread::available_parallelism`
+    pub max_instances: usize,
+}
+
+impl PoolBuilder {
+    /// Create a `PoolBuilder` with default values
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    /// Set the max number of parallel instances
+    pub fn with_max_instances(mut self, n: usize) -> Self {
+        self.max_instances = n;
+        self
+    }
+
+    /// Create a new `Pool` with the given configuration
+    pub fn build<F: 'static + Fn() -> Result<Plugin, Error>>(self, source: F) -> Pool {
+        Pool::new_from_builder(source, self)
+    }
+}
+
+impl Default for PoolBuilder {
+    fn default() -> Self {
+        PoolBuilder {
+            max_instances: std::thread::available_parallelism()
+                .expect("available parallelism")
+                .into(),
+        }
+    }
+}
+
+/// `PoolPlugin` is used by the pool to track the number of live instances of a particular plugin
+#[derive(Clone, Debug)]
+pub struct PoolPlugin(std::rc::Rc<std::cell::RefCell<Plugin>>);
+
+impl PoolPlugin {
+    fn new(plugin: Plugin) -> Self {
+        Self(std::rc::Rc::new(std::cell::RefCell::new(plugin)))
+    }
+
+    /// Access the underlying plugin
+    pub fn plugin(&self) -> std::cell::RefMut<Plugin> {
+        self.0.borrow_mut()
+    }
+
+    /// Helper to call a plugin function on the underlying plugin
+    pub fn call<'a, Input: ToBytes<'a>, Output: FromBytesOwned>(
+        &self,
+        name: impl AsRef<str>,
+        input: Input,
+    ) -> Result<Output, Error> {
+        self.plugin().call(name.as_ref(), input)
+    }
+
+    /// Helper to get the underlying plugin's ID
+    pub fn id(&self) -> uuid::Uuid {
+        self.plugin().id
+    }
+}
+
+type PluginSource = dyn Fn() -> Result<Plugin, Error>;
+
+struct PoolInner {
+    plugin_source: Box<PluginSource>,
+    instances: Vec<PoolPlugin>,
+}
+
+unsafe impl Send for PoolInner {}
+unsafe impl Sync for PoolInner {}
+
+/// `Pool` manages threadsafe access to a limited number of instances of multiple plugins
+#[derive(Clone)]
+pub struct Pool {
+    config: PoolBuilder,
+    inner: std::sync::Arc<std::sync::Mutex<PoolInner>>,
+}
+
+unsafe impl Send for Pool {}
+unsafe impl Sync for Pool {}
+
+impl Pool {
+    /// Create a new pool with the default configuration
+    pub fn new<F: 'static + Fn() -> Result<Plugin, Error>>(source: F) -> Self {
+        Pool {
+            config: Default::default(),
+            inner: std::sync::Arc::new(std::sync::Mutex::new(PoolInner {
+                plugin_source: Box::new(source),
+                instances: Default::default(),
+            })),
+        }
+    }
+
+    /// Create a new pool configured using a `PoolBuilder`
+    pub fn new_from_builder<F: 'static + Fn() -> Result<Plugin, Error>>(
+        source: F,
+        builder: PoolBuilder,
+    ) -> Self {
+        Pool {
+            config: builder,
+            inner: std::sync::Arc::new(std::sync::Mutex::new(PoolInner {
+                plugin_source: Box::new(source),
+                instances: Default::default(),
+            })),
+        }
+    }
+
+    fn find_available(&self) -> Result<Option<PoolPlugin>, Error> {
+        let pool = self.inner.lock().unwrap();
+        for instance in pool.instances.iter() {
+            if std::rc::Rc::strong_count(&instance.0) == 1 {
+                return Ok(Some(instance.clone()));
+            }
+        }
+        Ok(None)
+    }
+
+    /// Get the number of live instances for a plugin
+    pub fn count(&self) -> usize {
+        self.inner.lock().unwrap().instances.len()
+    }
+
+    /// Get access to a plugin, this will create a new instance if needed (and allowed by the specified
+    /// max_instances). `Ok(None)` is returned if the timeout is reached before an available plugin could be
+    /// acquired
+    pub fn get(&self, timeout: std::time::Duration) -> Result<Option<PoolPlugin>, Error> {
+        let start = std::time::Instant::now();
+        let max = self.config.max_instances;
+        if let Some(avail) = self.find_available()? {
+            return Ok(Some(avail));
+        }
+
+        {
+            let mut pool = self.inner.lock().unwrap();
+            if pool.instances.len() < max {
+                let plugin = (*pool.plugin_source)()?;
+                let instance = PoolPlugin::new(plugin);
+                pool.instances.push(instance);
+                return Ok(Some(pool.instances.last().unwrap().clone()));
+            }
+        }
+
+        loop {
+            if let Ok(Some(x)) = self.find_available() {
+                return Ok(Some(x));
+            }
+            if std::time::Instant::now() - start > timeout {
+                return Ok(None);
+            }
+
+            std::thread::sleep(std::time::Duration::from_millis(100));
+        }
+    }
+
+    /// Access a plugin in a callback function. This calls `Pool::get` then the provided
+    /// callback. `Ok(None)` is returned if the timeout is reached before an available
+    /// plugin could be acquired
+    pub fn with_plugin<T>(
+        &self,
+        timeout: std::time::Duration,
+        f: impl FnOnce(&mut Plugin) -> Result<T, Error>,
+    ) -> Result<Option<T>, Error> {
+        if let Some(plugin) = self.get(timeout)? {
+            return f(&mut plugin.plugin()).map(Some);
+        }
+        Ok(None)
+    }
+}

runtime/src/readonly_dir.rs

@@ -0,0 +1,109 @@
+use crate::*;
+
+use wasi_common::{Error, ErrorExt};
+
+pub struct ReadOnlyDir<D: wasi_common::WasiDir> {
+    inner: std::sync::Arc<D>,
+}
+
+impl<D: wasi_common::WasiDir> ReadOnlyDir<D> {
+    pub fn new(inner: D) -> Self {
+        ReadOnlyDir {
+            inner: std::sync::Arc::new(inner),
+        }
+    }
+}
+
+#[wiggle::async_trait]
+impl<D: wasi_common::WasiDir> wasi_common::WasiDir for ReadOnlyDir<D> {
+    fn as_any(&self) -> &dyn std::any::Any {
+        self.inner.as_any()
+    }
+
+    async fn open_file(
+        &self,
+        symlink_follow: bool,
+        path: &str,
+        oflags: wasi_common::file::OFlags,
+        read: bool,
+        write: bool,
+        fdflags: wasi_common::file::FdFlags,
+    ) -> Result<wasi_common::dir::OpenResult, Error> {
+        if write {
+            return Err(Error::not_supported());
+        }
+        self.inner
+            .open_file(symlink_follow, path, oflags, read, false, fdflags)
+            .await
+    }
+
+    async fn create_dir(&self, _path: &str) -> Result<(), Error> {
+        Err(Error::not_supported())
+    }
+
+    async fn readdir(
+        &self,
+        cursor: wasi_common::dir::ReaddirCursor,
+    ) -> Result<
+        Box<dyn Iterator<Item = Result<wasi_common::dir::ReaddirEntity, Error>> + Send>,
+        Error,
+    > {
+        self.inner.readdir(cursor).await
+    }
+
+    async fn symlink(&self, _old_path: &str, _new_path: &str) -> Result<(), Error> {
+        Err(Error::not_supported())
+    }
+
+    async fn remove_dir(&self, _path: &str) -> Result<(), Error> {
+        Err(Error::not_supported())
+    }
+
+    async fn unlink_file(&self, _path: &str) -> Result<(), Error> {
+        Err(Error::not_supported())
+    }
+
+    async fn read_link(&self, path: &str) -> Result<std::path::PathBuf, Error> {
+        self.inner.read_link(path).await
+    }
+
+    async fn get_filestat(&self) -> Result<wasi_common::file::Filestat, Error> {
+        self.inner.get_filestat().await
+    }
+
+    async fn get_path_filestat(
+        &self,
+        path: &str,
+        follow_symlinks: bool,
+    ) -> Result<wasi_common::file::Filestat, Error> {
+        self.inner.get_path_filestat(path, follow_symlinks).await
+    }
+
+    async fn rename(
+        &self,
+        _path: &str,
+        _dest_dir: &dyn wasi_common::WasiDir,
+        _dest_path: &str,
+    ) -> Result<(), Error> {
+        Err(wasi_common::Error::not_supported())
+    }
+
+    async fn hard_link(
+        &self,
+        _path: &str,
+        _target_dir: &dyn wasi_common::WasiDir,
+        _target_path: &str,
+    ) -> Result<(), Error> {
+        Err(wasi_common::Error::not_supported())
+    }
+
+    async fn set_times(
+        &self,
+        _path: &str,
+        _atime: std::option::Option<wasi_common::SystemTimeSpec>,
+        _mtime: std::option::Option<wasi_common::SystemTimeSpec>,
+        _follow_symlinks: bool,
+    ) -> Result<(), Error> {
+        Err(wasi_common::Error::not_supported())
+    }
+}

runtime/src/sdk.rs

@@ -1,6 +1,6 @@
 #![allow(clippy::missing_safety_doc)]
 
-use std::os::raw::c_char;
+use std::{os::raw::c_char, ptr::null_mut};
 
 use crate::*;
 
@@ -11,6 +11,12 @@ pub struct ExtismFunction(std::cell::Cell<Option<Function>>);
 /// The return code used to specify a successful plugin call
 pub static EXTISM_SUCCESS: i32 = 0;
 
+fn make_error_msg(s: String) -> Vec<u8> {
+    let mut s = s.into_bytes();
+    s.push(0);
+    s
+}
+
 /// A union type for host function argument/return values
 #[repr(C)]
 pub union ValUnion {
@@ -38,33 +44,36 @@ pub type ExtismFunctionType = extern "C" fn(
     data: *mut std::ffi::c_void,
 );
 
-impl From<&wasmtime::Val> for ExtismVal {
-    fn from(value: &wasmtime::Val) -> Self {
-        match value.ty() {
-            wasmtime::ValType::I32 => ExtismVal {
+/// Log drain callback
+pub type ExtismLogDrainFunctionType = extern "C" fn(data: *const std::ffi::c_char, size: Size);
+
+impl ExtismVal {
+    fn from_val(value: &wasmtime::Val, ctx: impl AsContext) -> Result<Self, Error> {
+        match value.ty(ctx)? {
+            wasmtime::ValType::I32 => Ok(ExtismVal {
                 t: ValType::I32,
                 v: ValUnion {
                     i32: value.unwrap_i32(),
                 },
-            },
-            wasmtime::ValType::I64 => ExtismVal {
+            }),
+            wasmtime::ValType::I64 => Ok(ExtismVal {
                 t: ValType::I64,
                 v: ValUnion {
                     i64: value.unwrap_i64(),
                 },
-            },
-            wasmtime::ValType::F32 => ExtismVal {
+            }),
+            wasmtime::ValType::F32 => Ok(ExtismVal {
                 t: ValType::F32,
                 v: ValUnion {
                     f32: value.unwrap_f32(),
                 },
-            },
-            wasmtime::ValType::F64 => ExtismVal {
+            }),
+            wasmtime::ValType::F64 => Ok(ExtismVal {
                 t: ValType::F64,
                 v: ValUnion {
                     f64: value.unwrap_f64(),
                 },
-            },
+            }),
             t => todo!("{}", t),
         }
     }
@@ -81,6 +90,24 @@ pub unsafe extern "C" fn extism_plugin_id(plugin: *mut Plugin) -> *const u8 {
     plugin.id.as_bytes().as_ptr()
 }
 
+/// Get the current plugin's associated host context data. Returns null if call was made without
+/// host context.
+#[no_mangle]
+pub unsafe extern "C" fn extism_current_plugin_host_context(
+    plugin: *mut CurrentPlugin,
+) -> *mut std::ffi::c_void {
+    if plugin.is_null() {
+        return std::ptr::null_mut();
+    }
+
+    let plugin = &mut *plugin;
+    if let Ok(CVoidContainer(ptr)) = plugin.host_context::<CVoidContainer>() {
+        *ptr
+    } else {
+        std::ptr::null_mut()
+    }
+}
+
 /// Returns a pointer to the memory of the currently running plugin
 /// NOTE: this should only be called from host functions.
 #[no_mangle]
@@ -153,7 +180,7 @@ pub unsafe extern "C" fn extism_current_plugin_memory_free(
 /// - `n_outputs`: number of return types
 /// - `func`: the function to call
 /// - `user_data`: a pointer that will be passed to the function when it's called
-///    this value should live as long as the function exists
+///   this value should live as long as the function exists
 /// - `free_user_data`: a callback to release the `user_data` value when the resulting
 ///   `ExtismFunction` is freed.
 ///
@@ -197,7 +224,11 @@ pub unsafe extern "C" fn extism_function_new(
         output_types.clone(),
         user_data,
         move |plugin, inputs, outputs, user_data| {
-            let inputs: Vec<_> = inputs.iter().map(ExtismVal::from).collect();
+            let store = &*plugin.store;
+            let inputs: Vec<_> = inputs
+                .iter()
+                .map(|x| ExtismVal::from_val(x, store).unwrap())
+                .collect();
             let mut output_tmp: Vec<_> = output_types
                 .iter()
                 .map(|t| ExtismVal {
@@ -206,12 +237,28 @@ pub unsafe extern "C" fn extism_function_new(
                 })
                 .collect();
 
+            // We cannot simply "get" the Vec's storage pointer because
+            // the underlying storage might be invalid when the Vec is empty.
+            // In that case, we return (null, 0).
+
+            let (inputs_ptr, inputs_len) = if inputs.is_empty() {
+                (core::ptr::null(), 0 as Size)
+            } else {
+                (inputs.as_ptr(), inputs.len() as Size)
+            };
+
+            let (output_ptr, output_len) = if output_tmp.is_empty() {
+                (null_mut(), 0 as Size)
+            } else {
+                (output_tmp.as_mut_ptr(), output_tmp.len() as Size)
+            };
+
             func(
                 plugin,
-                inputs.as_ptr(),
-                inputs.len() as Size,
-                output_tmp.as_mut_ptr(),
-                output_tmp.len() as Size,
+                inputs_ptr,
+                inputs_len,
+                output_ptr,
+                output_len,
                 user_data.as_ptr(),
             );
 
@@ -219,8 +266,8 @@ pub unsafe extern "C" fn extism_function_new(
                 match tmp.t {
                     ValType::I32 => *out = Val::I32(tmp.v.i32),
                     ValType::I64 => *out = Val::I64(tmp.v.i64),
-                    ValType::F32 => *out = Val::F32(tmp.v.f32 as u32),
-                    ValType::F64 => *out = Val::F64(tmp.v.f64 as u64),
+                    ValType::F32 => *out = Val::F32(tmp.v.f32.to_bits()),
+                    ValType::F64 => *out = Val::F64(tmp.v.f64.to_bits()),
                     _ => todo!(),
                 }
             }
@@ -255,6 +302,79 @@ pub unsafe extern "C" fn extism_function_set_namespace(
     }
 }
 
+/// Pre-compile an Extism plugin
+#[no_mangle]
+pub unsafe extern "C" fn extism_compiled_plugin_new(
+    wasm: *const u8,
+    wasm_size: Size,
+    functions: *mut *const ExtismFunction,
+    n_functions: Size,
+    with_wasi: bool,
+    errmsg: *mut *mut std::ffi::c_char,
+) -> *mut CompiledPlugin {
+    trace!("Call to extism_plugin_new with wasm pointer {:?}", wasm);
+    let data = std::slice::from_raw_parts(wasm, wasm_size as usize);
+
+    let mut builder = PluginBuilder::new(data).with_wasi(with_wasi);
+
+    if !functions.is_null() {
+        let funcs = (0..n_functions)
+            .map(|i| unsafe { *functions.add(i as usize) })
+            .map(|ptr| {
+                if ptr.is_null() {
+                    return Err("Cannot pass null pointer");
+                }
+
+                let ExtismFunction(func) = &*ptr;
+                let Some(func) = func.take() else {
+                    return Err("Function cannot be registered with multiple different Plugins");
+                };
+
+                Ok(func)
+            })
+            .collect::<Result<Vec<_>, _>>()
+            .unwrap_or_else(|e| {
+                if !errmsg.is_null() {
+                    let e = std::ffi::CString::new(e.to_string()).unwrap();
+                    *errmsg = e.into_raw();
+                }
+                Vec::new()
+            });
+
+        if funcs.len() != n_functions as usize {
+            return std::ptr::null_mut();
+        }
+
+        builder = builder.with_functions(funcs);
+    }
+
+    CompiledPlugin::new(builder)
+        .map(|v| Box::into_raw(Box::new(v)))
+        .unwrap_or_else(|e| {
+            if !errmsg.is_null() {
+                let e = std::ffi::CString::new(format!(
+                    "Unable to compile Extism plugin: {}",
+                    e.root_cause(),
+                ))
+                .unwrap();
+                *errmsg = e.into_raw();
+            }
+            std::ptr::null_mut()
+        })
+}
+
+/// Free `ExtismCompiledPlugin`
+#[no_mangle]
+pub unsafe extern "C" fn extism_compiled_plugin_free(plugin: *mut CompiledPlugin) {
+    if plugin.is_null() {
+        return;
+    }
+
+    let plugin = Box::from_raw(plugin);
+    trace!("called extism_compiled_plugin_free");
+    drop(plugin)
+}
+
 /// Create a new plugin with host functions, the functions passed to this function no longer need to be manually freed using
 ///
 /// `wasm`: is a WASM module (wat or wasm) or a JSON encoded manifest
@@ -271,36 +391,70 @@ pub unsafe extern "C" fn extism_plugin_new(
     with_wasi: bool,
     errmsg: *mut *mut std::ffi::c_char,
 ) -> *mut Plugin {
-    trace!("Call to extism_plugin_new with wasm pointer {:?}", wasm);
     let data = std::slice::from_raw_parts(wasm, wasm_size as usize);
-    let mut funcs = vec![];
-
-    if !functions.is_null() {
-        for i in 0..n_functions {
-            unsafe {
-                let f = *functions.add(i as usize);
-                if f.is_null() {
-                    continue;
+    let funcs = if functions.is_null() {
+        vec![]
+    } else {
+        let funcs = (0..n_functions)
+            .map(|i| unsafe { *functions.add(i as usize) })
+            .map(|ptr| {
+                if ptr.is_null() {
+                    return Err("Cannot pass null pointer");
                 }
-                if let Some(f) = (*f).0.take() {
-                    funcs.push(f);
-                } else {
-                    let e = std::ffi::CString::new(
-                        "Function cannot be registered with multiple different Plugins",
-                    )
-                    .unwrap();
+
+                let ExtismFunction(func) = &*ptr;
+                let Some(func) = func.take() else {
+                    return Err("Function cannot be registered with multiple different Plugins");
+                };
+
+                Ok(func)
+            })
+            .collect::<Result<Vec<_>, _>>()
+            .unwrap_or_else(|e| {
+                if !errmsg.is_null() {
+                    let e = std::ffi::CString::new(e.to_string()).unwrap();
                     *errmsg = e.into_raw();
                 }
-            }
-        }
-    }
+                Vec::new()
+            });
 
-    let plugin = Plugin::new(data, funcs, with_wasi);
+        if funcs.len() != n_functions as usize {
+            return std::ptr::null_mut();
+        }
+
+        funcs
+    };
+
+    Plugin::new(data, funcs, with_wasi)
+        .map(|v| Box::into_raw(Box::new(v)))
+        .unwrap_or_else(|e| {
+            if !errmsg.is_null() {
+                let e = std::ffi::CString::new(format!(
+                    "Unable to compile Extism plugin: {}",
+                    e.root_cause(),
+                ))
+                .unwrap();
+                *errmsg = e.into_raw();
+            }
+            std::ptr::null_mut()
+        })
+}
+
+/// Create a new plugin from an `ExtismCompiledPlugin`
+#[no_mangle]
+pub unsafe extern "C" fn extism_plugin_new_from_compiled(
+    compiled: *const CompiledPlugin,
+    errmsg: *mut *mut std::ffi::c_char,
+) -> *mut Plugin {
+    let plugin = Plugin::new_from_compiled(&*compiled);
     match plugin {
         Err(e) => {
             if !errmsg.is_null() {
-                let e = std::ffi::CString::new(format!("Unable to create Extism plugin: {}", e))
-                    .unwrap();
+                let e = std::ffi::CString::new(format!(
+                    "Unable to create Extism plugin: {}",
+                    e.root_cause(),
+                ))
+                .unwrap();
                 *errmsg = e.into_raw();
             }
             std::ptr::null_mut()
@@ -309,6 +463,100 @@ pub unsafe extern "C" fn extism_plugin_new(
     }
 }
 
+/// Create a new plugin and set the number of instructions a plugin is allowed to execute
+#[no_mangle]
+pub unsafe extern "C" fn extism_plugin_new_with_fuel_limit(
+    wasm: *const u8,
+    wasm_size: Size,
+    functions: *mut *const ExtismFunction,
+    n_functions: Size,
+    with_wasi: bool,
+    fuel_limit: u64,
+    errmsg: *mut *mut std::ffi::c_char,
+) -> *mut Plugin {
+    trace!(
+        "Call to extism_plugin_new_with_fuel_limit with wasm pointer {:?}",
+        wasm
+    );
+    let data = std::slice::from_raw_parts(wasm, wasm_size as usize);
+    let funcs = if functions.is_null() {
+        vec![]
+    } else {
+        let funcs = (0..n_functions)
+            .map(|i| unsafe { *functions.add(i as usize) })
+            .map(|ptr| {
+                if ptr.is_null() {
+                    return Err("Cannot pass null pointer");
+                }
+
+                let ExtismFunction(func) = &*ptr;
+                let Some(func) = func.take() else {
+                    return Err("Function cannot be registered with multiple different Plugins");
+                };
+
+                Ok(func)
+            })
+            .collect::<Result<Vec<_>, _>>()
+            .unwrap_or_else(|e| {
+                if !errmsg.is_null() {
+                    let e = std::ffi::CString::new(e.to_string()).unwrap();
+                    *errmsg = e.into_raw();
+                }
+                Vec::new()
+            });
+
+        if funcs.len() != n_functions as usize {
+            return std::ptr::null_mut();
+        }
+
+        funcs
+    };
+
+    let compiled = match CompiledPlugin::new(
+        PluginBuilder::new(data)
+            .with_functions(funcs)
+            .with_wasi(with_wasi)
+            .with_fuel_limit(fuel_limit),
+    ) {
+        Ok(x) => x,
+        Err(e) => {
+            if !errmsg.is_null() {
+                let e = std::ffi::CString::new(format!(
+                    "Unable to compile Extism plugin: {}",
+                    e.root_cause(),
+                ))
+                .unwrap();
+                *errmsg = e.into_raw();
+            }
+            return std::ptr::null_mut();
+        }
+    };
+
+    let plugin = Plugin::new_from_compiled(&compiled);
+
+    match plugin {
+        Err(e) => {
+            if !errmsg.is_null() {
+                let e = std::ffi::CString::new(format!(
+                    "Unable to create Extism plugin: {}",
+                    e.root_cause(),
+                ))
+                .unwrap();
+                *errmsg = e.into_raw();
+            }
+            std::ptr::null_mut()
+        }
+        Ok(p) => Box::into_raw(Box::new(p)),
+    }
+}
+
+/// Enable HTTP response headers in plugins using `extism:host/env::http_request`
+#[no_mangle]
+pub unsafe extern "C" fn extism_plugin_allow_http_response_headers(plugin: *mut Plugin) {
+    let plugin = &mut *plugin;
+    plugin.store.data_mut().http_headers = Some(BTreeMap::new());
+}
+
 /// Free the error returned by `extism_plugin_new`, errors returned from `extism_plugin_error` don't need to be freed
 #[no_mangle]
 pub unsafe extern "C" fn extism_plugin_new_error_free(err: *mut std::ffi::c_char) {
@@ -318,7 +566,7 @@ pub unsafe extern "C" fn extism_plugin_new_error_free(err: *mut std::ffi::c_char
     drop(std::ffi::CString::from_raw(err))
 }
 
-/// Remove a plugin from the registry and free associated memory
+/// Free `ExtismPlugin`
 #[no_mangle]
 pub unsafe extern "C" fn extism_plugin_free(plugin: *mut Plugin) {
     if plugin.is_null() {
@@ -369,8 +617,6 @@ pub unsafe extern "C" fn extism_plugin_config(
         return false;
     }
     let plugin = &mut *plugin;
-    let _lock = plugin.instance.clone();
-    let mut lock = _lock.lock().unwrap();
 
     trace!(
         plugin = plugin.id.to_string(),
@@ -381,25 +627,11 @@ pub unsafe extern "C" fn extism_plugin_config(
     let json: std::collections::BTreeMap<String, Option<String>> =
         match serde_json::from_slice(data) {
             Ok(x) => x,
-            Err(e) => {
-                return plugin.return_error(&mut lock, e, false);
+            Err(_) => {
+                return false;
             }
         };
 
-    let wasi = &mut plugin.current_plugin_mut().wasi;
-    if let Some(Wasi { ctx, .. }) = wasi {
-        for (k, v) in json.iter() {
-            match v {
-                Some(v) => {
-                    let _ = ctx.push_env(k, v);
-                }
-                None => {
-                    let _ = ctx.push_env(k, "");
-                }
-            }
-        }
-    }
-
     let id = plugin.id;
     let config = &mut plugin.current_plugin_mut().manifest.config;
     for (k, v) in json.into_iter() {
@@ -429,9 +661,6 @@ pub unsafe extern "C" fn extism_plugin_function_exists(
         return false;
     }
     let plugin = &mut *plugin;
-    let _lock = plugin.instance.clone();
-    let mut lock = _lock.lock().unwrap();
-
     let name = std::ffi::CStr::from_ptr(func_name);
     trace!(
         plugin = plugin.id.to_string(),
@@ -441,8 +670,8 @@ pub unsafe extern "C" fn extism_plugin_function_exists(
 
     let name = match name.to_str() {
         Ok(x) => x,
-        Err(e) => {
-            return plugin.return_error(&mut lock, e, false);
+        Err(_) => {
+            return false;
         }
     };
 
@@ -461,6 +690,31 @@ pub unsafe extern "C" fn extism_plugin_call(
     func_name: *const c_char,
     data: *const u8,
     data_len: Size,
+) -> i32 {
+    extism_plugin_call_with_host_context(plugin, func_name, data, data_len, std::ptr::null_mut())
+}
+
+#[derive(Clone)]
+#[repr(transparent)]
+struct CVoidContainer(*mut std::ffi::c_void);
+
+// "You break it, you buy it."
+unsafe impl Send for CVoidContainer {}
+unsafe impl Sync for CVoidContainer {}
+
+/// Call a function with host context.
+///
+/// `func_name`: is the function to call
+/// `data`: is the input data
+/// `data_len`: is the length of `data`
+/// `host_context`: a pointer to context data that will be available in host functions
+#[no_mangle]
+pub unsafe extern "C" fn extism_plugin_call_with_host_context(
+    plugin: *mut Plugin,
+    func_name: *const c_char,
+    data: *const u8,
+    data_len: Size,
+    host_context: *mut std::ffi::c_void,
 ) -> i32 {
     if plugin.is_null() {
         return -1;
@@ -473,7 +727,10 @@ pub unsafe extern "C" fn extism_plugin_call(
     let name = std::ffi::CStr::from_ptr(func_name);
     let name = match name.to_str() {
         Ok(name) => name,
-        Err(e) => return plugin.return_error(&mut lock, e, -1),
+        Err(e) => {
+            plugin.error_msg = Some(make_error_msg(e.to_string()));
+            return -1;
+        }
     };
 
     trace!(
@@ -482,10 +739,17 @@ pub unsafe extern "C" fn extism_plugin_call(
         name
     );
     let input = std::slice::from_raw_parts(data, data_len as usize);
-    let res = plugin.raw_call(&mut lock, name, input);
-
+    let r = if host_context.is_null() {
+        None
+    } else {
+        Some(CVoidContainer(host_context))
+    };
+    let res = plugin.raw_call(&mut lock, name, input, r);
     match res {
-        Err((e, rc)) => plugin.return_error(&mut lock, e, rc),
+        Err((e, rc)) => {
+            plugin.error_msg = Some(make_error_msg(e.to_string()));
+            rc
+        }
         Ok(x) => x,
     }
 }
@@ -508,14 +772,26 @@ pub unsafe extern "C" fn extism_plugin_error(plugin: *mut Plugin) -> *const c_ch
     let _lock = _lock.lock().unwrap();
 
     if plugin.output.error_offset == 0 {
+        if let Some(err) = &plugin.error_msg {
+            return err.as_ptr() as *const _;
+        }
         trace!(plugin = plugin.id.to_string(), "error is NULL");
         return std::ptr::null();
     }
 
-    plugin
+    let offs = plugin.output.error_offset;
+
+    let ptr = plugin.current_plugin_mut().memory_ptr().add(offs as usize) as *const _;
+
+    let len = plugin
         .current_plugin_mut()
-        .memory_ptr()
-        .add(plugin.output.error_offset as usize) as *const _
+        .memory_length(offs)
+        .unwrap_or_default();
+
+    let mut data = std::slice::from_raw_parts(ptr, len as usize).to_vec();
+    data.push(0);
+    plugin.error_msg = Some(data);
+    plugin.error_msg.as_ref().unwrap().as_ptr() as *const _
 }
 
 /// Get the length of a plugin's output data
@@ -590,7 +866,6 @@ pub unsafe extern "C" fn extism_log_file(
 fn set_log_file(log_file: impl Into<std::path::PathBuf>, filter: &str) -> Result<(), Error> {
     let log_file = log_file.into();
     let s = log_file.to_str();
-
     let is_level = tracing::Level::from_str(filter).is_ok();
     let cfg = tracing_subscriber::FmtSubscriber::builder().with_env_filter({
         let x = tracing_subscriber::EnvFilter::builder()
@@ -624,7 +899,7 @@ fn set_log_file(log_file: impl Into<std::path::PathBuf>, filter: &str) -> Result
     Ok(())
 }
 
-static mut LOG_BUFFER: Option<LogBuffer> = None;
+static LOG_BUFFER: std::sync::Mutex<Option<LogBuffer>> = std::sync::Mutex::new(None);
 
 /// Enable a custom log handler, this will buffer logs until `extism_log_drain` is called
 /// Log level should be one of: info, error, trace, debug, warn
@@ -641,6 +916,7 @@ pub unsafe extern "C" fn extism_log_custom(log_level: *const c_char) -> bool {
     } else {
         "error"
     };
+
     set_log_buffer(level).is_ok()
 }
 
@@ -655,8 +931,8 @@ unsafe fn set_log_buffer(filter: &str) -> Result<(), Error> {
             x.parse_lossy(filter)
         }
     });
-    LOG_BUFFER = Some(LogBuffer::default());
-    let buf = LOG_BUFFER.clone().unwrap();
+    *LOG_BUFFER.lock().unwrap() = Some(LogBuffer::default());
+    let buf = LOG_BUFFER.lock().unwrap().clone().unwrap();
     cfg.with_ansi(false)
         .with_writer(move || buf.clone())
         .try_init()
@@ -667,11 +943,11 @@ unsafe fn set_log_buffer(filter: &str) -> Result<(), Error> {
 #[no_mangle]
 /// Calls the provided callback function for each buffered log line.
 /// This is only needed when `extism_log_custom` is used.
-pub unsafe extern "C" fn extism_log_drain(handler: extern "C" fn(*const std::ffi::c_char, usize)) {
-    if let Some(buf) = &mut LOG_BUFFER {
+pub unsafe extern "C" fn extism_log_drain(handler: ExtismLogDrainFunctionType) {
+    if let Some(buf) = LOG_BUFFER.lock().unwrap().as_mut() {
         if let Ok(mut buf) = buf.buffer.lock() {
             for (line, len) in buf.drain(..) {
-                handler(line.as_ptr(), len);
+                handler(line.as_ptr(), len as u64);
             }
         }
     }

runtime/src/tests/data/data.txt

@@ -0,0 +1 @@
+hello world!

runtime/src/tests/issues.rs

@@ -1,6 +1,7 @@
 use crate::*;
 
 const WASM_EMPTY: &[u8] = include_bytes!("../../../wasm/empty.wasm");
+const WASM_UNREACHABLE: &[u8] = include_bytes!("../../../wasm/unreachable.wasm");
 
 // https://github.com/extism/extism/issues/620
 #[test]
@@ -26,3 +27,31 @@ host_fn!(
         Ok(path.display().to_string())
     }
 );
+
+// https://github.com/extism/extism/issues/775
+#[test]
+fn test_issue_775() {
+    // Load and build plugin
+    let url = Wasm::data(WASM_UNREACHABLE);
+    let manifest = Manifest::new([url]);
+    let mut plugin = PluginBuilder::new(manifest)
+        .with_wasi(true)
+        .build()
+        .unwrap();
+    // Call test method
+    let lock = plugin.instance.clone();
+    let mut lock = lock.lock().unwrap();
+    let res = plugin.raw_call(&mut lock, "do_unreachable", b"", None::<()>);
+    let p = match res {
+        Err(e) => {
+            if e.1 == 0 {
+                Err(e.1)
+            } else {
+                Ok(e.1)
+            }
+        }
+        Ok(code) => Err(code),
+    }
+    .unwrap();
+    println!("{}", p);
+}

runtime/src/tests/kernel.rs

@@ -1,23 +1,34 @@
 use crate::*;
+use quickcheck::*;
 
 const KERNEL: &[u8] = include_bytes!("../extism-runtime.wasm");
 
-fn extism_alloc<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance, n: u64) -> u64 {
+fn extism_alloc<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, n: u64) -> u64 {
     let out_alloc = &mut [Val::I64(0)];
     instance
-        .get_func(&mut store, "alloc")
+        .get_func(&mut *store, "alloc")
         .unwrap()
-        .call(&mut store, &[Val::I64(n as i64)], out_alloc)
+        .call(store, &[Val::I64(n as i64)], out_alloc)
         .unwrap();
     out_alloc[0].unwrap_i64() as u64
 }
 
-fn extism_length<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) -> u64 {
+fn extism_length<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) -> u64 {
     let out = &mut [Val::I64(0)];
     instance
-        .get_func(&mut store, "length")
+        .get_func(&mut *store, "length")
         .unwrap()
-        .call(&mut store, &[Val::I64(p as i64)], out)
+        .call(store, &[Val::I64(p as i64)], out)
+        .unwrap();
+    out[0].unwrap_i64() as u64
+}
+
+fn extism_length_unsafe<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) -> u64 {
+    let out = &mut [Val::I64(0)];
+    instance
+        .get_func(&mut *store, "length_unsafe")
+        .unwrap()
+        .call(store, &[Val::I64(p as i64)], out)
         .unwrap();
     out[0].unwrap_i64() as u64
 }
@@ -32,122 +43,96 @@ fn extism_load_u8<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance
     out[0].unwrap_i32() as u8
 }
 
-fn extism_load_u64<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) -> u64 {
+fn extism_load_u64<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) -> u64 {
     let out = &mut [Val::I32(0)];
     instance
-        .get_func(&mut store, "load_u64")
+        .get_func(&mut *store, "load_u64")
         .unwrap()
-        .call(&mut store, &[Val::I64(p as i64)], out)
+        .call(store, &[Val::I64(p as i64)], out)
         .unwrap();
     out[0].unwrap_i64() as u64
 }
 
-fn extism_input_load_u8<T>(
-    mut store: &mut wasmtime::Store<T>,
-    instance: &mut Instance,
-    p: u64,
-) -> u8 {
+fn extism_input_load_u8<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) -> u8 {
     let out = &mut [Val::I32(0)];
     instance
-        .get_func(&mut store, "input_load_u8")
+        .get_func(&mut *store, "input_load_u8")
         .unwrap()
-        .call(&mut store, &[Val::I64(p as i64)], out)
+        .call(store, &[Val::I64(p as i64)], out)
         .unwrap();
     out[0].unwrap_i32() as u8
 }
 
 fn extism_input_load_u64<T>(
-    mut store: &mut wasmtime::Store<T>,
+    store: &mut wasmtime::Store<T>,
     instance: &mut Instance,
     p: u64,
 ) -> u64 {
     let out = &mut [Val::I32(0)];
     instance
-        .get_func(&mut store, "input_load_u64")
+        .get_func(&mut *store, "input_load_u64")
         .unwrap()
-        .call(&mut store, &[Val::I64(p as i64)], out)
+        .call(store, &[Val::I64(p as i64)], out)
         .unwrap();
     out[0].unwrap_i64() as u64
 }
 
-fn extism_store_u8<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64, x: u8) {
+fn extism_store_u8<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64, x: u8) {
     instance
-        .get_func(&mut store, "store_u8")
+        .get_func(&mut *store, "store_u8")
         .unwrap()
-        .call(
-            &mut store,
-            &[Val::I64(p as i64), Val::I32(x as i32)],
-            &mut [],
-        )
+        .call(store, &[Val::I64(p as i64), Val::I32(x as i32)], &mut [])
         .unwrap();
 }
 
-fn extism_store_u64<T>(
-    mut store: &mut wasmtime::Store<T>,
-    instance: &mut Instance,
-    p: u64,
-    x: u64,
-) {
+fn extism_store_u64<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64, x: u64) {
     instance
-        .get_func(&mut store, "store_u64")
+        .get_func(&mut *store, "store_u64")
         .unwrap()
-        .call(
-            &mut store,
-            &[Val::I64(p as i64), Val::I64(x as i64)],
-            &mut [],
-        )
+        .call(store, &[Val::I64(p as i64), Val::I64(x as i64)], &mut [])
         .unwrap();
 }
 
-fn extism_free<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) {
+fn extism_free<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) {
     instance
-        .get_func(&mut store, "free")
+        .get_func(&mut *store, "free")
         .unwrap()
-        .call(&mut store, &[Val::I64(p as i64)], &mut [])
+        .call(store, &[Val::I64(p as i64)], &mut [])
         .unwrap();
 }
 
-fn extism_error_set<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) {
+fn extism_error_set<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) {
     instance
-        .get_func(&mut store, "error_set")
+        .get_func(&mut *store, "error_set")
         .unwrap()
-        .call(&mut store, &[Val::I64(p as i64)], &mut [])
+        .call(store, &[Val::I64(p as i64)], &mut [])
         .unwrap();
 }
 
-fn extism_error_get<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance) -> u64 {
+fn extism_error_get<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance) -> u64 {
     let out = &mut [Val::I64(0)];
     instance
-        .get_func(&mut store, "error_get")
+        .get_func(&mut *store, "error_get")
         .unwrap()
-        .call(&mut store, &[], out)
+        .call(store, &[], out)
         .unwrap();
 
     out[0].unwrap_i64() as u64
 }
 
-fn extism_reset<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance) {
+fn extism_reset<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance) {
     instance
-        .get_func(&mut store, "reset")
+        .get_func(&mut *store, "reset")
         .unwrap()
-        .call(&mut store, &[], &mut [])
+        .call(store, &[], &mut [])
         .unwrap();
 }
 
-fn extism_input_set<T>(
-    mut store: &mut wasmtime::Store<T>,
-    instance: &mut Instance,
-    p: u64,
-    l: u64,
-) {
+fn extism_input_set<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64, l: u64) {
     instance
-        .get_func(&mut store, "input_set")
+        .get_func(&mut *store, "input_set")
         .unwrap()
-        .call(
-            &mut store,
-            &[Val::I64(p as i64), Val::I64(l as i64)],
-            &mut [],
-        )
+        .call(store, &[Val::I64(p as i64), Val::I64(l as i64)], &mut [])
         .unwrap();
 }
 
@@ -168,11 +153,32 @@ fn test_kernel_allocations() {
     // Test allocations
     assert_eq!(extism_alloc(&mut store, instance, 0), 0);
 
+    // 512 bytes, test block re-use + splitting
+    let p = extism_alloc(&mut store, instance, 65535);
+    let first_alloc = p;
+    assert_eq!(extism_length(&mut store, instance, p), 65535);
+    assert_eq!(extism_length(&mut store, instance, p + 1), 0);
+    assert_eq!(extism_length(&mut store, instance, p + 2), 0);
+    assert_eq!(extism_length(&mut store, instance, p + 3), 0);
+    assert_eq!(extism_length(&mut store, instance, p + 4), 0);
+    extism_free(&mut store, instance, p);
+
+    // Should re-use the previous block
+    let q = extism_alloc(&mut store, instance, 65535);
+    assert_eq!(q, p);
+    assert_eq!(extism_length(&mut store, instance, q), 65535);
+    extism_free(&mut store, instance, q);
+
+    let r = extism_alloc(&mut store, instance, 65535 - 24);
+    assert_eq!(r, q);
+    assert_eq!(extism_length(&mut store, instance, q), 65535 - 24);
+    extism_free(&mut store, instance, r);
+
     // 1 byte
     let p = extism_alloc(&mut store, instance, 1);
-    let first_alloc = p;
     assert!(p > 0);
     assert_eq!(extism_length(&mut store, instance, p), 1);
+    assert_eq!(extism_length_unsafe(&mut store, instance, p), 1);
     extism_free(&mut store, instance, p);
 
     // 2 bytes
@@ -180,39 +186,17 @@ fn test_kernel_allocations() {
     assert!(x > 0);
     assert!(x != p);
     assert_eq!(extism_length(&mut store, instance, x), 2);
+    assert_eq!(extism_length_unsafe(&mut store, instance, x), 2);
     extism_free(&mut store, instance, x);
 
     for i in 0..64 {
         let p = extism_alloc(&mut store, instance, 64 - i);
         assert!(p > 0);
         assert_eq!(extism_length(&mut store, instance, p), 64 - i);
+        assert_eq!(extism_length_unsafe(&mut store, instance, p), 64 - i);
         extism_free(&mut store, instance, p);
-
-        // should re-use the last allocation
-        let q = extism_alloc(&mut store, instance, 64 - i);
-        assert_eq!(p, q);
-        assert_eq!(extism_length(&mut store, instance, q), 64 - i);
-        extism_free(&mut store, instance, q);
     }
 
-    // 512 bytes, test block re-use + splitting
-    let p = extism_alloc(&mut store, instance, 512);
-    assert_eq!(extism_length(&mut store, instance, p), 512);
-    extism_free(&mut store, instance, p);
-
-    // 128 bytes, should be split off the 512 byte block
-    let q = extism_alloc(&mut store, instance, 128);
-    assert!(p <= q && q < p + 512);
-    assert_eq!(extism_length(&mut store, instance, q), 128);
-    extism_free(&mut store, instance, q);
-
-    // 128 bytes, same as above
-    let r = extism_alloc(&mut store, instance, 128);
-    assert!(p <= r && r < p + 512);
-    assert!(r > p);
-    assert_eq!(extism_length(&mut store, instance, q), 128);
-    extism_free(&mut store, instance, q);
-
     // 100 pages
     let p = extism_alloc(&mut store, instance, 6553600);
     assert!(p > 0);
@@ -297,3 +281,158 @@ fn test_load_input() {
     // Out of bounds should return 0
     assert_eq!(extism_input_load_u64(&mut store, instance, 123457), 0);
 }
+
+#[test]
+fn test_failed_quickcheck1() {
+    let (mut store, mut instance) = init_kernel_test();
+    let allocs = [
+        20622, 23162, 58594, 32421, 25928, 44611, 26318, 24455, 5798, 60202, 42126, 64928, 57832,
+        50888, 63256, 37562, 46334, 47985, 60836, 28132, 65535, 37800, 33150, 48768, 38457, 57249,
+        5734, 58587, 26294, 26653, 24519, 1,
+    ];
+
+    extism_reset(&mut store, &mut instance);
+    for a in allocs {
+        println!("Alloc: {a}");
+        let n = extism_alloc(&mut store, &mut instance, a);
+        if n == 0 {
+            continue;
+        }
+        assert_eq!(a, extism_length(&mut store, &mut instance, n));
+    }
+}
+
+#[test]
+fn test_failed_quickcheck2() {
+    let (mut store, mut instance) = init_kernel_test();
+    let allocs = [352054710, 1248853976, 2678441931, 14567928];
+
+    extism_reset(&mut store, &mut instance);
+    for a in allocs {
+        println!("Alloc: {a}");
+        let n = extism_alloc(&mut store, &mut instance, a);
+        if n == 0 {
+            continue;
+        }
+        assert_eq!(a, extism_length(&mut store, &mut instance, n));
+    }
+}
+
+quickcheck! {
+    fn check_alloc(amounts: Vec<u16>) -> bool {
+        let (mut store, mut instance) = init_kernel_test();
+        let instance = &mut instance;
+        for a in amounts {
+            let ptr = extism_alloc(&mut store, instance, a as u64);
+            if ptr == 0 || ptr == u64::MAX {
+                continue
+            }
+            if extism_length(&mut store, instance, ptr) != a as u64 {
+                return false
+            }
+        }
+
+        true
+    }
+}
+
+quickcheck! {
+    fn check_large_alloc(amounts: Vec<u32>) -> bool {
+        let (mut store, mut instance) = init_kernel_test();
+        let instance = &mut instance;
+        for a in amounts {
+            let ptr = extism_alloc(&mut store, instance, a as u64);
+            if ptr == 0 {
+                continue
+            }
+            let len = extism_length_unsafe(&mut store, instance, ptr);
+            if len != a as u64 {
+                return false
+            }
+        }
+
+        true
+    }
+}
+
+quickcheck! {
+    fn check_alloc_with_frees(amounts: Vec<u16>) -> bool {
+        let (mut store, mut instance) = init_kernel_test();
+        let instance = &mut instance;
+        let mut prev = 0;
+        for a in amounts {
+            let ptr = extism_alloc(&mut store, instance, a as u64);
+            if ptr == 0 {
+                continue
+            }
+            if extism_length(&mut store, instance, ptr) != a as u64  {
+                return false
+            }
+
+            if a % 2 == 0 {
+                extism_free(&mut store, instance, ptr);
+            } else if a % 3 == 0 {
+                extism_free(&mut store, instance, prev);
+            }
+
+            prev = ptr;
+        }
+
+        true
+    }
+}
+
+quickcheck! {
+    fn check_large_alloc_with_frees(amounts: Vec<u32>) -> bool {
+        let (mut store, mut instance) = init_kernel_test();
+        let instance = &mut instance;
+        let mut prev = 0;
+        for a in amounts {
+            let ptr = extism_alloc(&mut store, instance, a as u64);
+            if ptr == 0 || ptr == u64::MAX {
+                continue
+            }
+            if extism_length(&mut store, instance, ptr) != a as u64 {
+                return false
+            }
+            if a % 2 == 0 {
+                extism_free(&mut store, instance, ptr);
+            } else if a % 3 == 0 {
+                extism_free(&mut store, instance, prev);
+            }
+
+            prev = ptr;
+
+        }
+
+        true
+    }
+}
+
+quickcheck! {
+    fn check_alloc_with_load_and_store(amounts: Vec<u16>) -> bool {
+        use rand::Rng;
+        let mut rng = rand::rng();
+        let (mut store, mut instance) = init_kernel_test();
+        let instance = &mut instance;
+        for a in amounts {
+            let ptr = extism_alloc(&mut store, instance, a as u64);
+            if ptr == 0 || ptr == u64::MAX {
+                continue
+            }
+            if extism_length(&mut store, instance, ptr) != a as u64 {
+                return false
+            }
+
+            for _ in 0..16 {
+                let i = rng.random_range(ptr..ptr+a as u64);
+                extism_store_u8(&mut store, instance, i, i as u8);
+                if extism_load_u8(&mut store, instance, i as u64) != i as u8 {
+                    return false
+                }
+            }
+        }
+
+        true
+    }
+}

runtime/src/tests/mod.rs

@@ -1,3 +1,4 @@
 mod issues;
 mod kernel;
+mod pool;
 mod runtime;

runtime/src/tests/pool.rs

@@ -0,0 +1,48 @@
+use crate::*;
+
+fn run_thread(p: Pool, i: u64) -> std::thread::JoinHandle<()> {
+    std::thread::spawn(move || {
+        std::thread::sleep(std::time::Duration::from_millis(i));
+        let s: String = p
+            .get(std::time::Duration::from_secs(1))
+            .unwrap()
+            .unwrap()
+            .call("count_vowels", "abc")
+            .unwrap();
+        println!("{}", s);
+    })
+}
+
+#[test]
+fn test_threads() {
+    for i in 1..=3 {
+        let data = include_bytes!("../../../wasm/code.wasm");
+        let plugin_builder =
+            extism::PluginBuilder::new(extism::Manifest::new([extism::Wasm::data(data)]))
+                .with_wasi(true);
+        let pool: Pool = PoolBuilder::new()
+            .with_max_instances(i)
+            .build(move || plugin_builder.clone().build());
+
+        let threads = vec![
+            run_thread(pool.clone(), 1000),
+            run_thread(pool.clone(), 1000),
+            run_thread(pool.clone(), 1000),
+            run_thread(pool.clone(), 1000),
+            run_thread(pool.clone(), 1000),
+            run_thread(pool.clone(), 1000),
+            run_thread(pool.clone(), 500),
+            run_thread(pool.clone(), 500),
+            run_thread(pool.clone(), 500),
+            run_thread(pool.clone(), 500),
+            run_thread(pool.clone(), 500),
+            run_thread(pool.clone(), 0),
+        ];
+
+        for t in threads {
+            t.join().unwrap();
+        }
+
+        assert!(pool.count() <= i);
+    }
+}

runtime/src/tests/runtime.rs

@@ -1,5 +1,7 @@
+use extism_manifest::{HttpRequest, MemoryOptions};
+
 use crate::*;
-use std::{io::Write, time::Instant};
+use std::{collections::HashMap, io::Write, time::Instant};
 
 const WASM: &[u8] = include_bytes!("../../../wasm/code-functions.wasm");
 const WASM_NO_FUNCTIONS: &[u8] = include_bytes!("../../../wasm/code.wasm");
@@ -7,6 +9,8 @@ const WASM_LOOP: &[u8] = include_bytes!("../../../wasm/loop.wasm");
 const WASM_GLOBALS: &[u8] = include_bytes!("../../../wasm/globals.wasm");
 const WASM_REFLECT: &[u8] = include_bytes!("../../../wasm/reflect.wasm");
 const WASM_HTTP: &[u8] = include_bytes!("../../../wasm/http.wasm");
+const WASM_HTTP_HEADERS: &[u8] = include_bytes!("../../../wasm/http_headers.wasm");
+const WASM_FS: &[u8] = include_bytes!("../../../wasm/read_write.wasm");
 
 host_fn!(pub hello_world (a: String) -> String { Ok(a) });
 
@@ -39,11 +43,12 @@ pub struct Count {
 
 #[test]
 fn it_works() {
-    tracing_subscriber::fmt()
+    let log = tracing_subscriber::fmt()
         .with_ansi(false)
         .with_env_filter("extism=debug")
         .with_writer(std::fs::File::create("test.log").unwrap())
-        .init();
+        .try_init()
+        .is_ok();
 
     let wasm_start = Instant::now();
 
@@ -143,8 +148,10 @@ fn it_works() {
     println!("wasm function call (avg, N = {}): {:?}", num_tests, avg);
 
     // Check that log file was written to
-    let meta = std::fs::metadata("test.log").unwrap();
-    assert!(meta.len() > 0);
+    if log {
+        let meta = std::fs::metadata("test.log").unwrap();
+        assert!(meta.len() > 0);
+    }
 }
 
 #[test]
@@ -235,6 +242,68 @@ fn test_timeout() {
     assert!(err == "timeout");
 }
 
+#[test]
+fn test_fuel() {
+    let manifest = Manifest::new([extism_manifest::Wasm::data(WASM_LOOP)]);
+    let mut plugin = PluginBuilder::new(manifest)
+        .with_wasi(true)
+        .with_fuel_limit(1)
+        .build()
+        .unwrap();
+    for _ in 0..10001 {
+        let output: Result<&[u8], Error> = plugin.call("loop_forever", "abc123");
+        let err = output.unwrap_err().root_cause().to_string();
+        println!("Fuel limited plugin exited with error: {:?}", &err);
+        assert!(err.contains("fuel"));
+    }
+}
+
+#[test]
+fn test_fuel_consumption() {
+    let manifest = Manifest::new([extism_manifest::Wasm::data(WASM_LOOP)]);
+    let mut plugin = PluginBuilder::new(manifest)
+        .with_wasi(true)
+        .with_fuel_limit(10000)
+        .build()
+        .unwrap();
+
+    let output: Result<&[u8], Error> = plugin.call("loop_forever", "abc123");
+    assert!(output.is_err());
+
+    let fuel_consumed = plugin.fuel_consumed().unwrap();
+    println!("Fuel consumed: {}", fuel_consumed);
+    assert!(fuel_consumed > 0);
+}
+
+#[test]
+#[cfg(feature = "http")]
+fn test_http_timeout() {
+    let f = Function::new(
+        "hello_world",
+        [PTR],
+        [PTR],
+        UserData::default(),
+        hello_world,
+    );
+
+    let manifest = Manifest::new([extism_manifest::Wasm::data(WASM_HTTP)])
+        .with_timeout(std::time::Duration::from_millis(1))
+        .with_allowed_host("www.extism.org");
+    let mut plugin = Plugin::new(manifest, [f], true).unwrap();
+
+    let start = std::time::Instant::now();
+    let output: Result<&[u8], Error> =
+        plugin.call("http_request", r#"{"url": "https://www.extism.org"}"#);
+    let end = std::time::Instant::now();
+    let time = end - start;
+    let err = output.unwrap_err().root_cause().to_string();
+    println!(
+        "Timed out plugin ran for {:?}, with error: {:?}",
+        time, &err
+    );
+    assert!(err == "timeout");
+}
+
 typed_plugin!(pub TestTypedPluginGenerics {
     count_vowels<T: FromBytes<'a>>(&str) -> T
 });
@@ -283,8 +352,10 @@ fn test_multiple_instantiations() {
 #[test]
 fn test_globals() {
     let mut plugin = Plugin::new(WASM_GLOBALS, [], true).unwrap();
-    for i in 0..100000 {
-        let Json(count) = plugin.call::<_, Json<Count>>("globals", "").unwrap();
+    for i in 0..100001 {
+        let Json(count) = plugin
+            .call_with_host_context::<_, Json<Count>, _>("globals", "", ())
+            .unwrap();
         assert_eq!(count.count, i);
     }
 }
@@ -302,6 +373,42 @@ fn test_toml_manifest() {
     assert_eq!(count.get("count").unwrap().as_i64().unwrap(), 1);
 }
 
+#[test]
+fn test_call_with_host_context() {
+    #[derive(Clone)]
+    struct Foo {
+        message: String,
+    }
+
+    let f = Function::new(
+        "host_reflect",
+        [PTR],
+        [PTR],
+        UserData::default(),
+        |current_plugin, _val, ret, _user_data: UserData<()>| {
+            let foo = current_plugin.host_context::<Foo>()?.clone();
+            let hnd = current_plugin.memory_new(foo.message)?;
+            ret[0] = current_plugin.memory_to_val(hnd);
+            Ok(())
+        },
+    );
+
+    let mut plugin = Plugin::new(WASM_REFLECT, [f], true).unwrap();
+
+    let message = "hello world";
+    let output: String = plugin
+        .call_with_host_context(
+            "reflect",
+            "anything, really",
+            Foo {
+                message: message.to_string(),
+            },
+        )
+        .unwrap();
+
+    assert_eq!(output, message);
+}
+
 #[test]
 fn test_fuzz_reflect_plugin() {
     // assert!(set_log_file("stdout", Some(log::Level::Trace)));
@@ -357,7 +464,7 @@ fn hello_world_set_error(
     _user_data: UserData<()>,
 ) -> Result<(), Error> {
     plugin.set_error("TEST")?;
-    outputs[0] = inputs[0].clone();
+    outputs[0] = inputs[0];
     Ok(())
 }
 
@@ -454,7 +561,7 @@ fn hello_world_user_data(
     let mut data = data.lock().unwrap();
     let s = _plugin.memory_get_val(&inputs[0])?;
     data.write_all(s)?;
-    outputs[0] = inputs[0].clone();
+    outputs[0] = inputs[0];
     Ok(())
 }
 
@@ -572,3 +679,165 @@ fn test_disable_cache() {
 
     assert!(t < t1);
 }
+
+#[test]
+fn test_manifest_ptr_len() {
+    let manifest = serde_json::json!({
+        "wasm" : [
+            {
+                "data" : {
+                    "ptr" : WASM_NO_FUNCTIONS.as_ptr() as u64,
+                    "len" : WASM_NO_FUNCTIONS.len()
+                }
+            }
+        ]
+    });
+    let mut plugin = Plugin::new(manifest.to_string().as_bytes(), [], true).unwrap();
+    let output = plugin.call("count_vowels", "abc123").unwrap();
+    let count: serde_json::Value = serde_json::from_slice(output).unwrap();
+    assert_eq!(count.get("count").unwrap().as_i64().unwrap(), 1);
+}
+
+#[test]
+fn test_no_vars() {
+    let data = br#"
+(module
+    (import "extism:host/env" "var_set" (func $var_set (param i64 i64)))
+    (import "extism:host/env" "input_offset" (func $input_offset (result i64)))
+    (func (export "test") (result i32)
+        (call $input_offset)
+        (call $input_offset)
+        (call $var_set)
+        (i32.const 0)
+    )
+)
+    "#;
+    let manifest = Manifest::new([Wasm::data(data)])
+        .with_memory_options(MemoryOptions::new().with_max_var_bytes(1));
+    let mut plugin = Plugin::new(manifest, [], true).unwrap();
+    let output: Result<(), Error> = plugin.call("test", b"A".repeat(1024));
+    assert!(output.is_err());
+    let output: Result<(), Error> = plugin.call("test", vec![]);
+    assert!(output.is_ok());
+}
+
+#[test]
+fn test_linking() {
+    let manifest = Manifest::new([
+        Wasm::Data {
+            data: br#"
+                (module
+                    (import "wasi_snapshot_preview1" "random_get" (func $random (param i32 i32) (result i32)))
+                    (import "extism:host/env" "alloc" (func $alloc (param i64) (result i64)))
+                    (import "extism:host/user" "hello" (func $hello))
+                    (global $counter (mut i32) (i32.const 0))
+                    (func $start (export "_start")
+                        (global.set $counter (i32.add (global.get $counter) (i32.const 1)))
+                    )
+                    (func (export "read_counter") (result i32)
+                        (global.get $counter)
+                    )
+                    (start $start)
+                )
+            "#.to_vec(),
+            meta: WasmMetadata {
+                name: Some("commander".to_string()),
+                hash: None,
+            },
+        },
+        Wasm::Data {
+            data: br#"
+                (module
+                    (import "commander" "_start" (func $commander_start))
+                    (import "commander" "read_counter" (func $commander_read_counter (result i32)))
+                    (import "extism:host/env" "store_u64" (func $store_u64 (param i64 i64)))
+                    (import "extism:host/env" "alloc" (func $alloc (param i64) (result i64)))
+                    (import "extism:host/user" "hello" (func $hello))
+                    (import "extism:host/env" "output_set" (func $output_set (param i64 i64)))
+                    (func (export "run") (result i32)
+                        (local $output i64)
+                        (local.set $output (call $alloc (i64.const 8)))
+
+                        (call $commander_start)
+                        (call $commander_start)
+                        (call $commander_start)
+                        (call $commander_start)
+                        (call $hello)
+                        (call $store_u64 (local.get $output) (i64.extend_i32_u (call $commander_read_counter)))
+                        (call $output_set (local.get $output) (i64.const 8))
+                        i32.const 0
+                    )
+                )
+            "#.to_vec(),
+            meta: WasmMetadata {
+                name: Some("main".to_string()),
+                hash: None,
+            },
+        },
+    ]);
+    let mut plugin = PluginBuilder::new(manifest)
+        .with_wasi(true)
+        .with_function("hello", [], [], UserData::new(()), |_, _, _, _| {
+            eprintln!("hello!");
+            Ok(())
+        })
+        .build()
+        .unwrap();
+
+    for _ in 0..5 {
+        assert_eq!(plugin.call::<&str, i64>("run", "Hello, world!").unwrap(), 1);
+    }
+}
+
+#[test]
+fn test_readonly_dirs() {
+    let wasm = Wasm::data(WASM_FS);
+    let manifest = Manifest::new([wasm])
+        .with_allowed_path("ro:src/tests/data".to_string(), "/data")
+        .with_config_key("path", "/data/data.txt");
+
+    let mut plugin = PluginBuilder::new(manifest)
+        .with_wasi(true)
+        .build()
+        .unwrap();
+
+    let res = plugin.call::<&str, &str>("try_read", "").unwrap();
+    assert_eq!(res, "hello world!");
+
+    let line = "hello world 2";
+    let res2 = plugin.call::<&str, &str>("try_write", line);
+    assert!(
+        res2.is_err(),
+        "Expected try_write to fail, but it succeeded."
+    );
+}
+
+#[test]
+#[cfg(feature = "http")]
+fn test_http_response_headers() {
+    let mut plugin = PluginBuilder::new(
+        Manifest::new([Wasm::data(WASM_HTTP_HEADERS)]).with_allowed_host("extism.org"),
+    )
+    .with_http_response_headers(true)
+    .build()
+    .unwrap();
+    let req = HttpRequest::new("https://extism.org");
+    let Json(res): Json<HashMap<String, String>> = plugin.call("http_get", Json(req)).unwrap();
+    println!("{:?}", res);
+    assert_eq!(res["content-type"], "text/html; charset=utf-8");
+}
+
+#[test]
+#[cfg(feature = "http")]
+fn test_http_response_headers_disabled() {
+    let mut plugin = PluginBuilder::new(
+        Manifest::new([Wasm::data(WASM_HTTP_HEADERS)]).with_allowed_host("extism.org"),
+    )
+    .with_http_response_headers(false)
+    .build()
+    .unwrap();
+    let req = HttpRequest::new("https://extism.org");
+    let Json(res): Json<HashMap<String, String>> = plugin.call("http_get", Json(req)).unwrap();
+    println!("{:?}", res);
+    assert!(res.is_empty());
+}

runtime/src/timer.rs

@@ -22,18 +22,18 @@ pub(crate) struct Timer {
 
 #[cfg(not(target_family = "windows"))]
 extern "C" fn cleanup_timer() {
-    let mut timer = match unsafe { TIMER.lock() } {
+    let mut timer = match TIMER.lock() {
         Ok(x) => x,
         Err(e) => e.into_inner(),
     };
     drop(timer.take());
 }
 
-static mut TIMER: std::sync::Mutex<Option<Timer>> = std::sync::Mutex::new(None);
+static TIMER: std::sync::Mutex<Option<Timer>> = std::sync::Mutex::new(None);
 
 impl Timer {
     pub(crate) fn tx() -> std::sync::mpsc::Sender<TimerAction> {
-        let mut timer = match unsafe { TIMER.lock() } {
+        let mut timer = match TIMER.lock() {
             Ok(x) => x,
             Err(e) => e.into_inner(),
         };
@@ -92,25 +92,39 @@ impl Timer {
             loop {
                 if plugins.is_empty() {
                     if let Ok(x) = rx.recv() {
-                        handle!(x)
+                        handle!(x);
                     }
                 }
 
-                plugins = plugins
-                    .into_iter()
-                    .filter(|(_k, (engine, end))| {
-                        if let Some(end) = end {
-                            let now = std::time::Instant::now();
-                            if end <= &now {
-                                engine.increment_epoch();
-                                return false;
+                let mut timeout: Option<std::time::Duration> = None;
+
+                plugins.retain(|_k, (engine, end)| {
+                    if let Some(end) = end {
+                        let now = std::time::Instant::now();
+                        if *end <= now {
+                            engine.increment_epoch();
+                            return false;
+                        } else {
+                            let time_left =
+                                (*end - now).saturating_sub(std::time::Duration::from_millis(1));
+                            if let Some(t) = &timeout {
+                                if time_left < *t {
+                                    timeout = Some(time_left);
+                                }
+                            } else {
+                                timeout = Some(time_left);
                             }
                         }
-                        true
-                    })
-                    .collect();
+                    }
 
-                for x in rx.try_iter() {
+                    true
+                });
+
+                if let Some(timeout) = timeout {
+                    if let Ok(x) = rx.recv_timeout(timeout) {
+                        handle!(x)
+                    }
+                } else if let Ok(x) = rx.recv() {
                     handle!(x)
                 }
             }