feat: expose building a CompiledPlugin with a fuel limit (#883)

Expose a new method `extism_compiled_plugin_new_with_fuel_limit` to
build a CompiledPlugin with a fuel limit.

Fixes #882

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @zshipko

.github/dependabot.yml

@@ -9,33 +9,8 @@ updates:
     directory: "/" # Location of package manifests
     schedule:
       interval: "weekly"
+    ignore:
+      - dependency-name: "wasmtime"
+      - dependency-name: "wasi-common"
+      - dependency-name: "wiggle"
 
-  - package-ecosystem: "pip"
-    directory: "python"
-    schedule:
-      interval: "weekly"
-
-  - package-ecosystem: "mix"
-    directory: "elixir"
-    schedule:
-      interval: "weekly"
-
-  - package-ecosystem: "npm"
-    directory: "node"
-    schedule:
-      interval: "weekly"
-
-  - package-ecosystem: "composer"
-    directory: "/"
-    schedule:
-      interval: "weekly"
-
-  - package-ecosystem: "bundler"
-    directory: "ruby"
-    schedule:
-      interval: "weekly"
-
-  - package-ecosystem: "gomod"
-    directory: "/"
-    schedule:
-      interval: "weekly"

.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-on: 
+on:
   pull_request:
     paths:
       - .github/actions/extism/**
@@ -52,7 +52,7 @@ jobs:
         shell: bash
         run: cargo build --release -p ${{ env.LIBEXTISM_CRATE }}
       - name: Upload artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: libextism-${{ matrix.os }}
           path: |
@@ -116,4 +116,9 @@ jobs:
           path: target/**
           key: ${{ runner.os }}-target-${{ github.sha }}
       - run: cargo install cargo-criterion
-      - run: cargo criterion
+      - run:
+          cargo criterion
+      - run: |
+          git fetch
+          git checkout main
+          cargo criterion

.github/workflows/kernel.yml

@@ -21,6 +21,9 @@ jobs:
           target: wasm32-unknown-unknown
       - uses: Swatinem/rust-cache@v2
 
+      - name: install wasm-tools
+        uses: bytecodealliance/actions/wasm-tools/setup@v1
+
       - name: Install deps
         run: |
           sudo apt install wabt --yes
@@ -39,6 +42,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         if: ${{ env.GIT_EXIT_CODE }} != 0
         with:
+          author: "zshipko <zshipko@users.noreply.github.com>"
           title: "update(kernel): extism-runtime.wasm in ${{ github.event.pull_request.head.ref }}"
           body: "Automated PR to update `runtime/src/extism-runtime.wasm` in PR #${{ github.event.number }}"
           base: "${{ github.event.pull_request.head.ref }}"

.github/workflows/release-convert.yaml

@@ -1,26 +0,0 @@
-on:
-  workflow_dispatch:
-
-name: Release extism-convert
-
-jobs:
-  release-convert:
-    name: release-extism-convert
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Setup Rust env
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: minimal
-          override: true
-          target: ${{ matrix.target }}
-
-      - name: Release Rust Convert Crate
-        env:
-          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_API_TOKEN }}
-        run: |
-          cargo publish --manifest-path convert/Cargo.toml

.github/workflows/release-dotnet-native.yaml

@@ -1,4 +1,6 @@
 on:
+  release:
+    types: [published, edited]
   workflow_dispatch:
 
 name: Release .NET Native NuGet Packages
@@ -18,10 +20,13 @@ jobs:
         uses: actions/setup-dotnet@v3.0.3
         with:
            dotnet-version: 7.x
-      - uses: dawidd6/action-download-artifact@v2
-        with:
-          workflow: release.yml
-          name: release-artifacts
+      - name: download release
+        run: |
+          tag='${{ github.ref }}'
+          tag="${tag/refs\/tags\//}"
+          gh release download "$tag" -p 'libextism-*.tar.gz'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Extract Archive
         run: |      
           extract_archive() {
@@ -54,16 +59,16 @@ jobs:
               fi
           }
           
-          extract_archive "libextism-x86_64-pc-windows-msvc-*.tar.gz" "dotnet/nuget/runtimes/win-x64/native/"
-          extract_archive "libextism-aarch64-apple-darwin-*.tar.gz" "dotnet/nuget/runtimes/osx-arm64/native/"
-          extract_archive "libextism-x86_64-apple-darwin-*.tar.gz" "dotnet/nuget/runtimes/osx-x64/native/"
-          extract_archive "libextism-x86_64-unknown-linux-gnu-*.tar.gz" "dotnet/nuget/runtimes/linux-x64/native/"
-          extract_archive "libextism-aarch64-unknown-linux-gnu-*.tar.gz" "dotnet/nuget/runtimes/linux-arm64/native/"
-          extract_archive "libextism-aarch64-unknown-linux-musl-*.tar.gz" "dotnet/nuget/runtimes/linux-musl-arm64/native/"
+          extract_archive "libextism-x86_64-pc-windows-msvc-*.tar.gz" "nuget/runtimes/win-x64/native/"
+          extract_archive "libextism-aarch64-apple-darwin-*.tar.gz" "nuget/runtimes/osx-arm64/native/"
+          extract_archive "libextism-x86_64-apple-darwin-*.tar.gz" "nuget/runtimes/osx-x64/native/"
+          extract_archive "libextism-x86_64-unknown-linux-gnu-*.tar.gz" "nuget/runtimes/linux-x64/native/"
+          extract_archive "libextism-aarch64-unknown-linux-gnu-*.tar.gz" "nuget/runtimes/linux-arm64/native/"
+          extract_archive "libextism-aarch64-unknown-linux-musl-*.tar.gz" "nuget/runtimes/linux-musl-arm64/native/"
 
       - name: Pack NuGet packages
         run: |
-          find ./dotnet/nuget -type f -name "*.csproj" -exec dotnet pack {} -o release-artifacts \;
+          find ./nuget -type f -name "*.csproj" -exec dotnet pack {} -o release-artifacts \;
 
       - name: Publish NuGet packages
         run: |

.github/workflows/release-manifest.yaml

@@ -1,26 +0,0 @@
-on:
-  workflow_dispatch:
-
-name: Release extism-manifest
-
-jobs:
-  release-manifest:
-    name: release-extism-manifest
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Setup Rust env
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: minimal
-          override: true
-          target: ${{ matrix.target }}
-
-      - name: Release Rust Manifest Crate
-        env:
-          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_API_TOKEN }}
-        run: |
-          cargo publish --manifest-path manifest/Cargo.toml

.github/workflows/release-rust.yaml

@@ -1,4 +1,6 @@
 on:
+  release:
+    types: [published, edited]
   workflow_dispatch:
 
 name: Release Runtime/Rust SDK
@@ -10,6 +12,21 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+        with:
+          ref: '${{ github.ref }}'
+
+      - name: Set version
+        shell: bash
+        run: |
+          version="${{ github.ref }}"
+          if [[ "$version" = "refs/heads/main" ]]; then
+            version="0.0.0-dev"
+          else
+            version="${version/refs\/tags\/v/}"
+          fi
+          sed -i -e "s/0.0.0+replaced-by-ci/${version}/g" Cargo.toml
+          pyproject="$(cat extism-maturin/pyproject.toml)"
+          <<<"$pyproject" >extism-maturin/pyproject.toml sed -e 's/^version = "0.0.0.replaced-by-ci"/version = "'"$version"'"/g'
 
       - name: Setup Rust env
         uses: actions-rs/toolchain@v1
@@ -19,8 +36,50 @@ jobs:
           override: true
           target: ${{ matrix.target }}
 
+      - name: Release Rust convert-macros Crate
+        env:
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_API_TOKEN }}
+        run: |
+          version=$(cargo metadata --format-version=1 | jq -r '.packages[] | select(.name == "extism") | .version')
+
+          if ! &>/dev/null curl -sLIf https://crates.io/api/v1/crates/extism-convert-macros/${version}/download; then
+            cargo publish --manifest-path convert-macros/Cargo.toml --allow-dirty
+          else
+            echo "already published ${version}"
+          fi
+
+      - name: Release Rust Convert Crate
+        env:
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_API_TOKEN }}
+        run: |
+          version=$(cargo metadata --format-version=1 | jq -r '.packages[] | select(.name == "extism") | .version')
+
+          if ! &>/dev/null curl -sLIf https://crates.io/api/v1/crates/extism-convert/${version}/download; then
+            cargo publish --manifest-path convert/Cargo.toml --allow-dirty
+          else
+            echo "already published ${version}"
+          fi
+
+      - name: Release Rust Manifest Crate
+        env:
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_API_TOKEN }}
+        run: |
+          version=$(cargo metadata --format-version=1 | jq -r '.packages[] | select(.name == "extism") | .version')
+
+          if ! &>/dev/null curl -sLIf https://crates.io/api/v1/crates/extism-manifest/${version}/download; then
+            cargo publish --manifest-path manifest/Cargo.toml --allow-dirty
+          else
+            echo "already published ${version}"
+          fi
+
       - name: Release Runtime
         env:
           CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_API_TOKEN }}
         run: |
-          cargo publish --manifest-path runtime/Cargo.toml
+          version=$(cargo metadata --format-version=1 | jq -r '.packages[] | select(.name == "extism") | .version')
+
+          if ! &>/dev/null curl -sLIf https://crates.io/api/v1/crates/extism/${version}/download; then
+            cargo publish --manifest-path runtime/Cargo.toml --allow-dirty
+          else
+            echo "already published ${version}"
+          fi

.github/workflows/release.yml

@@ -7,11 +7,6 @@ on:
 
 name: Release
 
-env:
-  RUNTIME_MANIFEST: runtime/Cargo.toml
-  RUNTIME_CRATE: libextism
-  RUSTFLAGS: -C target-feature=-crt-static
-  ARTIFACT_DIR: release-artifacts
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -21,6 +16,11 @@ jobs:
   release:
     name: ${{ matrix.os }} ${{ matrix.target }}
     runs-on: ${{ matrix.os }}-latest
+    env:
+      RUNTIME_MANIFEST: runtime/Cargo.toml
+      RUNTIME_CRATE: libextism
+      RUSTFLAGS: -C target-feature=-crt-static
+      ARTIFACT_DIR: release-artifacts-${{ matrix.os }}-${{ matrix.target }}
     strategy:
       matrix:
         include:
@@ -28,48 +28,56 @@ jobs:
             target: 'x86_64-apple-darwin'
             artifact: 'libextism.dylib'
             static-artifact: 'libextism.a'
+            static-dll-artifact: ''
             pc-in: 'extism.pc.in'
             static-pc-in: 'extism-static.pc.in'
           - os: 'macos'
             target: 'aarch64-apple-darwin'
             artifact: 'libextism.dylib'
             static-artifact: 'libextism.a'
+            static-dll-artifact: ''
             pc-in: 'extism.pc.in'
             static-pc-in: 'extism-static.pc.in'
           - os: 'ubuntu'
             target: 'aarch64-unknown-linux-gnu'
             artifact: 'libextism.so'
             static-artifact: 'libextism.a'
+            static-dll-artifact: ''
             pc-in: 'extism.pc.in'
             static-pc-in: 'extism-static.pc.in'
           - os: 'ubuntu'
             target: 'aarch64-unknown-linux-musl'
             artifact: 'libextism.so'
             static-artifact: 'libextism.a'
+            static-dll-artifact: ''
             pc-in: 'extism.pc.in'
             static-pc-in: 'extism-static.pc.in'
           - os: 'ubuntu'
             target: 'x86_64-unknown-linux-gnu'
             artifact: 'libextism.so'
             static-artifact: 'libextism.a'
+            static-dll-artifact: ''
             pc-in: 'extism.pc.in'
             static-pc-in: 'extism-static.pc.in'
           - os: 'ubuntu'
             target: 'x86_64-unknown-linux-musl'
-            artifact: ''
+            artifact: 'libextism.so'
             static-artifact: 'libextism.a'
-            pc-in: ''
+            static-dll-artifact: ''
+            pc-in: 'extism.pc.in'
             static-pc-in: 'extism-static.pc.in'
           - os: 'windows'
             target: 'x86_64-pc-windows-gnu'
             artifact: 'extism.dll'
             static-artifact: 'libextism.a'
+            static-dll-artifact: 'libextism.dll.a'
             pc-in: 'extism.pc.in'
             static-pc-in: 'extism-static.pc.in'
           - os: 'windows'
             target: 'x86_64-pc-windows-msvc'
             artifact: 'extism.dll'
             static-artifact: 'extism.lib'
+            static-dll-artifact: 'extism.dll.lib'
             pc-in: ''
             static-pc-in: ''
 
@@ -90,13 +98,11 @@ jobs:
           pyproject="$(cat extism-maturin/pyproject.toml)"
           <<<"$pyproject" >extism-maturin/pyproject.toml sed -e 's/^version = "0.0.0.replaced-by-ci"/version = "'"$version"'"/g'
 
-      - name: Install Rust
-        uses: actions-rs/toolchain@v1
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
-          toolchain: stable
-          profile: minimal
-          override: true
           target: ${{ matrix.target }}
+          override: true
+          toolchain: stable
 
       - uses: Swatinem/rust-cache@v2
         with:
@@ -105,11 +111,15 @@ jobs:
           cache-on-failure: "true"
 
       - name: Build Target (${{ matrix.os }} ${{ matrix.target }})
-        uses: actions-rs/cargo@v1
-        with:
-          use-cross: ${{ matrix.os != 'windows' }}
-          command: build
-          args: --release --target ${{ matrix.target }} -p ${{ env.RUNTIME_CRATE }}
+        if: ${{ matrix.os != 'windows' }}
+        run: |
+          cargo install cross
+          cross build --release --target ${{ matrix.target }} -p ${{ env.RUNTIME_CRATE }}
+
+      - name: Build Target (${{ matrix.os }} ${{ matrix.target }})
+        if: ${{ matrix.os == 'windows' }}
+        run: |
+          cargo build --release --target ${{ matrix.target }} -p ${{ env.RUNTIME_CRATE }}
 
       - uses: actions/setup-python@v4
         with:
@@ -158,7 +168,8 @@ jobs:
           cp LICENSE ${SRC_DIR}
           tar -C ${SRC_DIR} -czvf ${ARCHIVE} extism.h \
             ${{ matrix.artifact }} ${{ matrix.static-artifact }}  \
-            ${{ matrix.pc-in }} ${{ matrix.static-pc-in }}
+            ${{ matrix.pc-in }} ${{ matrix.static-pc-in }} \
+            ${{ matrix.static-dll-artifact }}
           ls -ll ${ARCHIVE}
 
           if &>/dev/null which shasum; then
@@ -181,7 +192,7 @@ jobs:
           ls -ll ${DEST_DIR}
 
       - name: Upload Artifact to Summary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ env.ARTIFACT_DIR }}
           path: ${{ env.ARTIFACT_DIR }}
@@ -200,9 +211,10 @@ jobs:
     needs: [release]
     if: github.ref == 'refs/heads/main'
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
-          name: ${{ env.ARTIFACT_DIR }}
+          pattern: release-artifacts-*
+          merge-multiple: true
 
       - uses: "marvinpinto/action-automatic-releases@latest"
         with:

.gitignore

@@ -46,4 +46,6 @@ java/.DS_Store
 extism-maturin/src/extism.h
 runtime/*.log
 libextism/example
-libextism/extism*.pc
+libextism/extism*.pc
+*.cwasm
+test-cache

Cargo.toml

@@ -1,6 +1,6 @@
 [workspace]
 resolver = "2"
-members = ["extism-maturin", "manifest", "runtime", "libextism", "convert"]
+members = ["extism-maturin", "manifest", "runtime", "libextism", "convert", "convert-macros"]
 exclude = ["kernel"]
 
 [workspace.package]
@@ -14,4 +14,5 @@ version = "0.0.0+replaced-by-ci"
 [workspace.dependencies]
 extism = { path = "./runtime", version = "0.0.0+replaced-by-ci" }
 extism-convert = { path = "./convert", version = "0.0.0+replaced-by-ci" }
+extism-convert-macros = { path = "./convert-macros", version = "0.0.0+replaced-by-ci" }
 extism-manifest = { path = "./manifest", version = "0.0.0+replaced-by-ci" }

Makefile

@@ -4,10 +4,12 @@ AEXT=a
 FEATURES?=default
 DEFAULT_FEATURES?=yes
 RUST_TARGET?=
+EXTRA_LIBS=
 
 UNAME := $(shell uname -s)
 ifeq ($(UNAME),Darwin)
 	SOEXT=dylib
+	EXTRA_LIBS=-framework Security
 endif
 
 ifeq ($(DEFAULT_FEATURES),no)
@@ -29,7 +31,8 @@ endif
 build:
 	cargo build --release $(FEATURE_FLAGS) --manifest-path libextism/Cargo.toml $(TARGET_FLAGS)
 	sed -e "s%@CMAKE_INSTALL_PREFIX@%$(DEST)%" libextism/extism.pc.in > libextism/extism.pc
-	sed -e "s%@CMAKE_INSTALL_PREFIX@%$(DEST)%" libextism/extism-static.pc.in > libextism/extism-static.pc
+	sed -e "s%@CMAKE_INSTALL_PREFIX@%$(DEST)%" \
+	    -e "s%Libs: %Libs: $(EXTRA_LIBS) %" libextism/extism-static.pc.in > libextism/extism-static.pc
 
 bench:
 	@(cargo criterion $(TARGET_FLAGS) || echo 'For nicer output use cargo-criterion: `cargo install cargo-criterion` - using `cargo bench`') && cargo bench $(TARGET_FLAGS)

README.md

@@ -1,62 +1,188 @@
-### _Welcome!_
+<div align="center">
+    <a href="https://extism.org">
+    <picture>
+        <source media="(prefers-color-scheme: dark)" srcset=".github/assets/logo-horizontal-darkmode.png">
+        <img alt="Extism - the WebAssembly framework" width="75%" style="max-width: 600px" src=".github/assets/logo-horizontal.png">
+    </picture>
+    </a>
 
-**Please note:** This project still under active development and APIs are changing as we hit 1.0. Currently, the main branch has many breaking changes. Our current release is 0.5.x. Until we release 1.0, we are cutting these releases from the [stable](https://github.com/extism/extism/tree/stable) branch.
+[![Discord](https://img.shields.io/discord/1011124058408112148?color=%23404eed&label=Community%20Chat&logo=Discord&logoColor=%23404eed)](https://extism.org/discord)
+![GitHub Org's stars](https://img.shields.io/github/stars/extism)
+![Downloads](https://img.shields.io/crates/d/extism-manifest)
+![GitHub License](https://img.shields.io/github/license/extism/extism)
+![GitHub release (with filter)](https://img.shields.io/github/v/release/extism/extism)
 
-If you're interested in working on or building with Extism, please join our [Discord](https://discord.gg/cx3usBCWnc) and let us know - we are happy to help get you started.
+</div>
 
-[![Discord](https://img.shields.io/discord/1011124058408112148?color=%23404eed&label=Community%20Chat&logo=Discord&logoColor=%23404eed)](https://discord.gg/cx3usBCWnc)
+# Overview
 
-# [Extism](https://extism.org)
+Extism is a lightweight framework for building with WebAssembly (Wasm). It
+supports running Wasm code on servers, the edge, CLIs, IoT, browsers and
+everything in between. Extism is designed to be "universal" in that it supports
+a common interface, no matter where it runs.
 
-The universal plug-in system. Run WebAssembly extensions inside your app. Use idiomatic Host SDKs for [Go](https://github.com/extism/go-sdk#readme),
-[Ruby](https://github.com/extism/ruby-sdk#readme),
-[Python](https://github.com/extism/python-sdk#readme),
-[JavaScript](https://github.com/extism/js-sdk#readme),
-[Rust](/runtime/#readme),
-[C](libextism/#readme),
-[C++](https://github.com/extism/cpp-sdk/#readme),
-[OCaml](https://github.com/extism/ocaml-sdk#readme),
-[Haskell](https://github.com/extism/haskell-sdk#readme),
-[PHP](https://github.com/extism/php-sdk#readme),
-[Elixir](https://github.com/extism/elixir-sdk#readme),
-[.NET](https://github.com/extism/dotnet-sdk#readme),
-[Java](https://github.com/extism/java-sdk#readme),
-[Zig](https://github.com/extism/zig-sdk#readme),
-[D](https://github.com/extism/d-sdk#readme),
-&amp; more (others coming soon).
+> **Note:** One of the primary use cases for Extism is **building extensible
+> software & plugins**. You want to be able to execute arbitrary, untrusted code
+> from your users? Extism makes this safe and practical to do.
 
-Plug-in development kits (PDK) for plug-in authors supported in [Rust](https://github.com/extism/rust-pdk#readme), [AssemblyScript](https://github.com/extism/assemblyscript-pdk#readme), [Go](https://github.com/extism/go-pdk#readme), [C/C++](https://github.com/extism/c-pdk#readme), [Haskell](https://github.com/extism/haskell-pdk#readme), [JavaScript](https://github.com/extism/js-pdk#readme), [C#](https://github.com/extism/dotnet-pdk#readme), [F#](https://github.com/extism/dotnet-pdk#readme) and [Zig](https://github.com/extism/zig-pdk#readme).
+Additionally, Extism adds some extra utilities on top of standard Wasm runtimes.
+For example, we support persistent memory/module-scope variables, secure &
+host-controlled HTTP without WASI, runtime limiters & timers, simpler host
+function linking, and more. Extism users build:
 
-<p align="center">
-  <img style="width: 70%;" src="https://user-images.githubusercontent.com/7517515/210286900-39b144fd-1b26-4dd0-b7a9-2b5755bc174d.png" alt="Extism embedded SDK language support"/>
-</p>
+- plug-in systems
+- FaaS platforms
+- code generators
+- web applications
+- & much more...
 
+# Supported Targets
 
-Add a flexible, secure, and _bLaZiNg FaSt_ plug-in system to your project. Server, desktop, mobile, web, database -- you name it. Enable users to write and execute safe extensions to your software in **3 easy steps:**
+We currently provide releases for the following targets:
 
-### 1. Import
+- aarch64-apple-darwin
+- aarch64-unknown-linux-gnu
+- aarch64-unknown-linux-musl
+- x86_64-apple-darwin
+- x86_64-pc-windows-gnu
+- x86_64-pc-windows-msvc
+- x86_64-unknown-linux-gnu
+- x86_64-unknown-linux-musl
 
-Import an Extism Host SDK into your code as a library dependency.
+For Android we suggest taking a look at the [Chicory SDK](https://github.com/extism/chicory-sdk) for a pure Java
+Extism runtime.
 
-### 2. Integrate 
+# Run WebAssembly In Your App
 
-Identify the place(s) in your code where some arbitrary logic should run (the plug-in!), returning your code some results.
+Pick a SDK to import into your program, and refer to the documentation to get
+started:
 
-### 3. Execute
+| Type        | Language                                                                                       | Source Code                                                             | Package                                                                 |
+| ----------- | ---------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------- | ----------------------------------------------------------------------- |
+| Rust SDK    | <img alt="Rust SDK" src="https://extism.org/img/sdk-languages/rust.svg" width="50px"/>         | https://github.com/extism/extism/tree/main/runtime                      | [Crates.io](https://crates.io/crates/extism)                            |
+| JS SDK      | <img alt="JS SDK" src="https://extism.org/img/sdk-languages/js.svg" width="50px"/>             | https://github.com/extism/js-sdk <br/>(supports Web, Node, Deno & Bun!) | [NPM](https://www.npmjs.com/package/@extism/extism)                     |
+| Elixir SDK  | <img alt="Elixir SDK" src="https://extism.org/img/sdk-languages/elixir.svg" width="50px"/>     | https://github.com/extism/elixir-sdk                                    | [Hex](https://hex.pm/packages/extism)                                   |
+| Go SDK      | <img alt="Go SDK" src="https://extism.org/img/sdk-languages/go.svg" width="50px"/>             | https://github.com/extism/go-sdk                                        | [Go mod](https://pkg.go.dev/github.com/extism/go-sdk)                   |
+| Haskell SDK | <img alt="Haskell SDK" src="https://extism.org/img/sdk-languages/haskell.svg" width="50px"/>   | https://github.com/extism/haskell-sdk                                   | [Hackage](https://hackage.haskell.org/package/extism)                   |
+| Java SDK    | <img alt="Java SDK" src="https://extism.org/img/sdk-languages/java-android.svg" width="50px"/> | https://github.com/extism/java-sdk                                      | [Sonatype](https://central.sonatype.com/artifact/org.extism.sdk/extism) |
+| .NET SDK    | <img alt=".NET SDK" src="https://extism.org/img/sdk-languages/dotnet.svg" width="50px"/>       | https://github.com/extism/dotnet-sdk <br/>(supports C# & F#!)           | [Nuget](https://www.nuget.org/packages/Extism.Sdk)                      |
+| OCaml SDK   | <img alt="OCaml SDK" src="https://extism.org/img/sdk-languages/ocaml.svg" width="50px"/>       | https://github.com/extism/ocaml-sdk                                     | [opam](https://opam.ocaml.org/packages/extism/)                         |
+| Perl SDK    | <img alt="Perl SDK" src="https://extism.org/img/sdk-languages/perl.svg" width="50px"/>         | https://github.com/extism/perl-sdk                                      | [CPAN](https://metacpan.org/pod/Extism)                                 |
+| PHP SDK     | <img alt="PHP SDK" src="https://extism.org/img/sdk-languages/php.svg" width="50px"/>           | https://github.com/extism/php-sdk                                       | [Packagist](https://packagist.org/packages/extism/extism)               |
+| Python SDK  | <img alt="Python SDK" src="https://extism.org/img/sdk-languages/python.svg" width="50px"/>     | https://github.com/extism/python-sdk                                    | [PyPi](https://pypi.org/project/extism/)                                |
+| Ruby SDK    | <img alt="Ruby SDK" src="https://extism.org/img/sdk-languages/ruby.svg" width="50px"/>         | https://github.com/extism/ruby-sdk                                      | [RubyGems](https://rubygems.org/gems/extism)                            |
+| Zig SDK     | <img alt="Zig SDK" src="https://extism.org/img/sdk-languages/zig.svg" width="50px"/>           | https://github.com/extism/zig-sdk                                       | N/A                                                                     |
+| C SDK       | <img alt="C SDK" src="https://extism.org/img/sdk-languages/c.svg" width="50px"/>               | https://github.com/extism/extism/tree/main/libextism                    | N/A                                                                     |
+| C++ SDK     | <img alt="C++ SDK" src="https://extism.org/img/sdk-languages/cpp.svg" width="50px"/>           | https://github.com/extism/cpp-sdk                                       | N/A                                                                     |
 
-Load WebAssembly modules at any time in your app's lifetime and Extism will execute them in a secure sandbox, fully isolated from your program's memory.
+# Compile WebAssembly to run in Extism Hosts
 
---- 
+Extism Hosts (running the SDK) must execute WebAssembly code that has a
+[PDK, or Plug-in Development Kit](https://extism.org/docs/concepts/pdk), library
+compiled in to the `.wasm` binary. PDKs make it easy for plug-in / extension
+code authors to read input from the host and return data back, read provided
+configuration, set/get variables, make outbound HTTP calls if allowed, and more.
+
+Pick a PDK to import into your Wasm program, and refer to the documentation to
+get started:
+
+| Type               | Language                                                                                                   | Source Code                                                   | Package                                                   |
+| ------------------ | ---------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------- | --------------------------------------------------------- |
+| Rust PDK           | <img alt="Rust PDK" src="https://extism.org/img/sdk-languages/rust.svg" width="50px"/>                     | https://github.com/extism/rust-pdk                            | [Crates.io](https://crates.io/crates/extism-pdk)          |
+| JS PDK             | <img alt="JS PDK" src="https://extism.org/img/sdk-languages/js.svg" width="50px"/>                         | https://github.com/extism/js-pdk                              | N/A                                                       |
+| Python PDK         | <img alt="Python PDK" src="https://extism.org/img/sdk-languages/python.svg" width="50px"/>                 | https://github.com/extism/python-pdk                          | N/A                                                       |
+| Go PDK             | <img alt="Go PDK" src="https://extism.org/img/sdk-languages/go.svg" width="50px"/>                         | https://github.com/extism/go-pdk                              | [Go mod](https://pkg.go.dev/github.com/extism/go-pdk)     |
+| Haskell PDK        | <img alt="Haskell PDK" src="https://extism.org/img/sdk-languages/haskell.svg" width="50px"/>               | https://github.com/extism/haskell-pdk                         | [Hackage](https://hackage.haskell.org/package/extism-pdk) |
+| AssemblyScript PDK | <img alt="AssemblyScript PDK" src="https://extism.org/img/sdk-languages/assemblyscript.svg" width="50px"/> | https://github.com/extism/assemblyscript-pdk                  | [NPM](https://www.npmjs.com/package/@extism/as-pdk)       |
+| .NET PDK           | <img alt=".NET PDK" src="https://extism.org/img/sdk-languages/dotnet.svg" width="50px"/>                   | https://github.com/extism/dotnet-pdk <br/>(supports C# & F#!) | [Nuget](https://www.nuget.org/packages/Extism.Pdk)        |
+| C PDK              | <img alt="C PDK" src="https://extism.org/img/sdk-languages/c.svg" width="50px"/>                           | https://github.com/extism/c-pdk                               | N/A                                                       |
+| C++ PDK            | <img alt="C++ PDK" src="https://extism.org/img/sdk-languages/cpp.svg" width="50px"/>                       | https://github.com/extism/cpp-pdk                             | N/A                                                       |
+| Zig PDK            | <img alt="Zig PDK" src="https://extism.org/img/sdk-languages/zig.svg" width="50px"/>                       | https://github.com/extism/zig-pdk                             | N/A                                                       |
+
+# Generating Bindings
+
+It's often very useful to define a schema to describe the function signatures
+and types you want to use between Extism SDK and PDK languages.
+
+[XTP Bindgen](https://github.com/dylibso/xtp-bindgen) is an open source
+framework to generate PDK bindings for Extism plug-ins. It's used by the
+[XTP Platform](https://www.getxtp.com/), but can be used outside of the platform
+to define any Extism compatible plug-in system.
+
+## 1. Install the `xtp` CLI.
+
+See installation instructions
+[here](https://docs.xtp.dylibso.com/docs/cli#installation).
+
+## 2. Create a schema using our OpenAPI-inspired IDL:
+
+```yaml
+version: v1-draft
+exports: 
+  CountVowels:
+      input: 
+          type: string
+          contentType: text/plain; charset=utf-8
+      output:
+          $ref: "#/components/schemas/VowelReport"
+          contentType: application/json
+# components.schemas defined in example-schema.yaml...
+```
+
+> See an example in [example-schema.yaml](./example-schema.yaml), or a full
+> "kitchen sink" example on
+> [the docs page](https://docs.xtp.dylibso.com/docs/concepts/xtp-schema/).
+
+## 3. Generate bindings to use from your plugins:
+
+```
+xtp plugin init --schema-file ./example-schema.yaml
+  > 1. TypeScript                      
+    2. Go                              
+    3. Rust                            
+    4. Python                          
+    5. C#                              
+    6. Zig                             
+    7. C++                             
+    8. GitHub Template                 
+    9. Local Template
+```
+
+This will create an entire boilerplate plugin project for you to get started
+with. Implement the empty function(s), and run `xtp plugin build` to compile
+your plugin.
+
+> For more information about XTP Bindgen, see the
+> [dylibso/xtp-bindgen](https://github.com/dylibso/xtp-bindgen) repository and
+> the official
+> [XTP Schema documentation](https://docs.xtp.dylibso.com/docs/concepts/xtp-schema).
+
+# Support
+
+## Discord
+
+If you experience any problems or have any questions, please join our
+[Discord](https://extism.org/discord) and let us know. Our community is very
+responsive and happy to help get you started.
 
 ## Usage
 
-Head to the [project website](https://extism.org) for more information and docs. Also, consider reading an [overview](https://extism.org/docs/overview) of Extism and its goals & approach.
+Head to the [project website](https://extism.org) for more information and docs.
+Also, consider reading an [overview](https://extism.org/docs/overview) of Extism
+and its goals & approach.
 
 ## Contribution
 
-Thank you for considering a contribution to Extism, we are happy to help you make a PR or find something to work on! 
+Thank you for considering a contribution to Extism, we are happy to help you
+make a PR or find something to work on!
 
-The easiest way to start would be to join the [Discord](https://discord.gg/cx3usBCWnc) or open an issue on the [`extism/proposals`](https://github.com/extism/proposals) issue tracker, which can eventually become an Extism Improvement Proposal (EIP).
+The easiest way to start would be to join the
+[Discord](https://extism.org/discord) or open an issue on the
+[`extism/proposals`](https://github.com/extism/proposals) issue tracker, which
+can eventually become an Extism Improvement Proposal (EIP).
+
+For more information, please read the
+[Contributing](https://extism.org/docs/concepts/contributing) guide.
 
 ---
 
@@ -65,8 +191,8 @@ The easiest way to start would be to join the [Discord](https://discord.gg/cx3us
 Extism is an open-source product from the team at:
 
 <p align="left">
-  <a href="https://dylib.so" _target="blanks"><img width="200px" src="https://user-images.githubusercontent.com/7517515/198204119-5afdebb9-a5d8-4322-bd2a-46179c8d7b24.svg"/></a>
+  <a href="https://dylibso.com" _target="blanks"><img width="200px" src="https://user-images.githubusercontent.com/7517515/198204119-5afdebb9-a5d8-4322-bd2a-46179c8d7b24.svg"/></a>
 </p>
 
-
-_Reach out and tell us what you're building! We'd love to help._
+_Reach out and tell us what you're building! We'd love to help:_
+<a href="mailto:hello@dylibso.com">hello@dylibso.com</a>

convert-macros/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "extism-convert-macros"
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+homepage.workspace = true
+repository.workspace = true
+version.workspace = true
+description = "Macros to remove boilerplate with Extism"
+
+[lib]
+proc-macro = true
+
+[features]
+extism-path = []
+extism-pdk-path = []
+
+[dependencies]
+manyhow.version = "0.11.0"
+proc-macro-crate = "3.1.0"
+proc-macro2 = "1.0.78"
+quote = "1.0.35"
+syn = { version = "2.0.48", features = ["derive"] }
+
+[dev-dependencies]
+trybuild = "1.0.89"

convert-macros/src/lib.rs

@@ -0,0 +1,108 @@
+use std::iter;
+
+use manyhow::{ensure, error_message, manyhow, Result};
+use proc_macro_crate::{crate_name, FoundCrate};
+use quote::{format_ident, quote, ToTokens};
+use syn::{parse_quote, Attribute, DeriveInput, Path};
+
+/// Tries to resolve the path to `extism_convert` dynamically, falling back to feature flags when unsuccessful.
+fn convert_path() -> Path {
+    match (
+        crate_name("extism"),
+        crate_name("extism-convert"),
+        crate_name("extism-pdk"),
+    ) {
+        (Ok(FoundCrate::Name(name)), ..) => {
+            let ident = format_ident!("{name}");
+            parse_quote!(::#ident::convert)
+        }
+        (_, Ok(FoundCrate::Name(name)), ..) | (.., Ok(FoundCrate::Name(name))) => {
+            let ident = format_ident!("{name}");
+            parse_quote!(::#ident)
+        }
+        (Ok(FoundCrate::Itself), ..) => parse_quote!(::extism::convert),
+        (_, Ok(FoundCrate::Itself), ..) => parse_quote!(::extism_convert),
+        (.., Ok(FoundCrate::Itself)) => parse_quote!(::extism_pdk),
+        _ if cfg!(feature = "extism-path") => parse_quote!(::extism::convert),
+        _ if cfg!(feature = "extism-pdk-path") => parse_quote!(::extism_pdk),
+        _ => parse_quote!(::extism_convert),
+    }
+}
+
+fn extract_encoding(attrs: &[Attribute]) -> Result<Path> {
+    let encodings: Vec<_> = attrs
+        .iter()
+        .filter(|attr| attr.path().is_ident("encoding"))
+        .collect();
+    ensure!(!encodings.is_empty(), "encoding needs to be specified"; try = "`#[encoding(Json)]`");
+    ensure!(encodings.len() < 2, encodings[1], "only one encoding can be specified"; try = "remove `{}`", encodings[1].to_token_stream());
+
+    Ok(encodings[0].parse_args().map_err(
+        |e| error_message!(e.span(), "{e}"; note= "expects a path"; try = "`#[encoding(Json)]`"),
+    )?)
+}
+
+#[manyhow]
+#[proc_macro_derive(ToBytes, attributes(encoding))]
+pub fn to_bytes(
+    DeriveInput {
+        attrs,
+        ident,
+        generics,
+        ..
+    }: DeriveInput,
+) -> Result {
+    let encoding = extract_encoding(&attrs)?;
+    let convert = convert_path();
+
+    let (_, type_generics, _) = generics.split_for_impl();
+
+    let mut generics = generics.clone();
+    generics.make_where_clause().predicates.push(
+        parse_quote!(for<'__to_bytes_b> #encoding<&'__to_bytes_b Self>: #convert::ToBytes<'__to_bytes_b>)
+    );
+    generics.params = iter::once(parse_quote!('__to_bytes_a))
+        .chain(generics.params)
+        .collect();
+    let (impl_generics, _, where_clause) = generics.split_for_impl();
+
+    Ok(quote! {
+        impl #impl_generics #convert::ToBytes<'__to_bytes_a> for #ident #type_generics #where_clause
+        {
+            type Bytes = ::std::vec::Vec<u8>;
+
+            fn to_bytes(&self) -> Result<Self::Bytes, #convert::Error> {
+                #convert::ToBytes::to_bytes(&#encoding(self)).map(|__bytes| __bytes.as_ref().to_vec())
+            }
+        }
+
+    })
+}
+
+#[manyhow]
+#[proc_macro_derive(FromBytes, attributes(encoding))]
+pub fn from_bytes(
+    DeriveInput {
+        attrs,
+        ident,
+        mut generics,
+        ..
+    }: DeriveInput,
+) -> Result {
+    let encoding = extract_encoding(&attrs)?;
+    let convert = convert_path();
+    generics
+        .make_where_clause()
+        .predicates
+        .push(parse_quote!(#encoding<Self>: #convert::FromBytesOwned));
+    let (impl_generics, type_generics, where_clause) = generics.split_for_impl();
+    Ok(quote! {
+        impl #impl_generics #convert::FromBytesOwned for #ident #type_generics #where_clause
+        {
+            fn from_bytes_owned(__data: &[u8]) -> Result<Self, #convert::Error> {
+                <#encoding<Self> as #convert::FromBytesOwned>::from_bytes_owned(__data).map(|__encoding| __encoding.0)
+            }
+        }
+
+    })
+}

convert-macros/tests/ui.rs

@@ -0,0 +1,5 @@
+#[test]
+fn ui() {
+    let t = trybuild::TestCases::new();
+    t.compile_fail("tests/ui/*.rs");
+}

convert-macros/tests/ui/invalid-encoding.rs

@@ -0,0 +1,23 @@
+use extism_convert_macros::ToBytes;
+
+#[derive(ToBytes)]
+struct MissingEncoding;
+
+#[derive(ToBytes)]
+#[encoding]
+struct EmptyAttr;
+
+#[derive(ToBytes)]
+#[encoding = "string"]
+struct EqNoParen;
+
+#[derive(ToBytes)]
+#[encoding(something, else)]
+struct NotAPath;
+
+#[derive(ToBytes)]
+#[encoding(Multiple)]
+#[encoding(Encodings)]
+struct MultipleEncodings;
+
+fn main() {}

convert-macros/tests/ui/invalid-encoding.stderr

@@ -0,0 +1,44 @@
+error: encoding needs to be specified
+
+         = try: `#[encoding(Json)]`
+ --> tests/ui/invalid-encoding.rs:3:10
+  |
+3 | #[derive(ToBytes)]
+  |          ^^^^^^^
+  |
+  = note: this error originates in the derive macro `ToBytes` (in Nightly builds, run with -Z macro-backtrace for more info)
+
+error: expected attribute arguments in parentheses: #[encoding(...)]
+
+         = note: expects a path
+         = try: `#[encoding(Json)]`
+ --> tests/ui/invalid-encoding.rs:7:3
+  |
+7 | #[encoding]
+  |   ^^^^^^^^
+
+error: expected parentheses: #[encoding(...)]
+
+         = note: expects a path
+         = try: `#[encoding(Json)]`
+  --> tests/ui/invalid-encoding.rs:11:12
+   |
+11 | #[encoding = "string"]
+   |            ^
+
+error: unexpected token
+
+         = note: expects a path
+         = try: `#[encoding(Json)]`
+  --> tests/ui/invalid-encoding.rs:15:21
+   |
+15 | #[encoding(something, else)]
+   |                     ^
+
+error: only one encoding can be specified
+
+         = try: remove `#[encoding(Encodings)]`
+  --> tests/ui/invalid-encoding.rs:20:1
+   |
+20 | #[encoding(Encodings)]
+   | ^^^^^^^^^^^^^^^^^^^^^^

convert/Cargo.toml

@@ -11,16 +11,21 @@ description = "Traits to make Rust types usable with Extism"
 
 [dependencies]
 anyhow = "1.0.75"
-base64 = "~0.21"
-prost = { version = "0.12.0", optional = true }
+base64 = "~0.22"
+bytemuck = {version = "1.14.0", optional = true }
+prost = { version = "0.14.1", optional = true }
+protobuf = { version = "3.2.0", optional = true }
 rmp-serde = { version = "1.1.2", optional = true }
 serde = "1.0.186"
 serde_json = "1.0.105"
+extism-convert-macros.workspace = true
 
 [dev-dependencies]
 serde = { version = "1.0.186", features = ["derive"] }
 
 [features]
-default = ["msgpack", "protobuf"]
+default = ["msgpack", "prost", "raw"]
 msgpack = ["rmp-serde"]
-protobuf = ["prost"]
+raw = ["bytemuck"]
+extism-path = ["extism-convert-macros/extism-path"]
+extism-pdk-path = ["extism-convert-macros/extism-pdk-path"]

convert/src/encoding.rs

@@ -11,14 +11,14 @@ use base64::Engine;
 /// extism_convert::encoding!(MyJson, serde_json::to_vec, serde_json::from_slice);
 /// ```
 ///
-/// This will create a struct `struct MyJson<T>(pub T)` and implement `ToBytes` using `serde_json::to_vec`
-/// and `FromBytesOwned` using `serde_json::from_vec`
+/// This will create a struct `struct MyJson<T>(pub T)` and implement [`ToBytes`] using [`serde_json::to_vec`]
+/// and [`FromBytesOwned`] using [`serde_json::from_slice`]
 #[macro_export]
 macro_rules! encoding {
-    ($name:ident, $to_vec:expr, $from_slice:expr) => {
+    ($pub:vis $name:ident, $to_vec:expr, $from_slice:expr) => {
         #[doc = concat!(stringify!($name), " encoding")]
         #[derive(Debug)]
-        pub struct $name<T>(pub T);
+        $pub struct $name<T>(pub T);
 
         impl<T> $name<T> {
             pub fn into_inner(self) -> T {
@@ -50,12 +50,12 @@ macro_rules! encoding {
     };
 }
 
-encoding!(Json, serde_json::to_vec, serde_json::from_slice);
+encoding!(pub Json, serde_json::to_vec, serde_json::from_slice);
 
 #[cfg(feature = "msgpack")]
-encoding!(Msgpack, rmp_serde::to_vec, rmp_serde::from_slice);
+encoding!(pub Msgpack, rmp_serde::to_vec, rmp_serde::from_slice);
 
-impl<'a> ToBytes<'a> for serde_json::Value {
+impl ToBytes<'_> for serde_json::Value {
     type Bytes = Vec<u8>;
 
     fn to_bytes(&self) -> Result<Self::Bytes, Error> {
@@ -85,7 +85,7 @@ impl<T: AsRef<[u8]>> From<T> for Base64<T> {
     }
 }
 
-impl<'a, T: AsRef<[u8]>> ToBytes<'a> for Base64<T> {
+impl<T: AsRef<[u8]>> ToBytes<'_> for Base64<T> {
     type Bytes = String;
 
     fn to_bytes(&self) -> Result<Self::Bytes, Error> {
@@ -112,19 +112,19 @@ impl FromBytesOwned for Base64<String> {
 /// Protobuf encoding
 ///
 /// Allows for `prost` Protobuf messages to be used as arguments to Extism plugin calls
-#[cfg(feature = "protobuf")]
+#[cfg(feature = "prost")]
 #[derive(Debug)]
-pub struct Protobuf<T: prost::Message>(pub T);
+pub struct Prost<T: prost::Message>(pub T);
 
-#[cfg(feature = "protobuf")]
-impl<T: prost::Message> From<T> for Protobuf<T> {
+#[cfg(feature = "prost")]
+impl<T: prost::Message> From<T> for Prost<T> {
     fn from(data: T) -> Self {
         Self(data)
     }
 }
 
-#[cfg(feature = "protobuf")]
-impl<'a, T: prost::Message> ToBytes<'a> for Protobuf<T> {
+#[cfg(feature = "prost")]
+impl<T: prost::Message> ToBytes<'_> for Prost<T> {
     type Bytes = Vec<u8>;
 
     fn to_bytes(&self) -> Result<Self::Bytes, Error> {
@@ -132,9 +132,56 @@ impl<'a, T: prost::Message> ToBytes<'a> for Protobuf<T> {
     }
 }
 
-#[cfg(feature = "protobuf")]
-impl<T: Default + prost::Message> FromBytesOwned for Protobuf<T> {
+#[cfg(feature = "prost")]
+impl<T: Default + prost::Message> FromBytesOwned for Prost<T> {
     fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
-        Ok(Protobuf(T::decode(data)?))
+        Ok(Prost(T::decode(data)?))
     }
 }
+
+/// Protobuf encoding
+///
+/// Allows for `rust-protobuf` Protobuf messages to be used as arguments to Extism plugin calls
+#[cfg(feature = "protobuf")]
+pub struct Protobuf<T: protobuf::Message>(pub T);
+
+#[cfg(feature = "protobuf")]
+impl<T: protobuf::Message> ToBytes<'_> for Protobuf<T> {
+    type Bytes = Vec<u8>;
+
+    fn to_bytes(&self) -> Result<Self::Bytes, Error> {
+        Ok(self.0.write_to_bytes()?)
+    }
+}
+
+#[cfg(feature = "protobuf")]
+impl<T: Default + protobuf::Message> FromBytesOwned for Protobuf<T> {
+    fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
+        Ok(Protobuf(T::parse_from_bytes(data)?))
+    }
+}
+
+/// Raw does no conversion, it just copies the memory directly.
+/// Note: This will only work for types that implement [bytemuck::Pod](https://docs.rs/bytemuck/latest/bytemuck/trait.Pod.html)
+#[cfg(all(feature = "raw", target_endian = "little"))]
+pub struct Raw<'a, T: bytemuck::Pod>(pub &'a T);
+
+#[cfg(all(feature = "raw", target_endian = "little"))]
+impl<'a, T: bytemuck::Pod> ToBytes<'a> for Raw<'a, T> {
+    type Bytes = &'a [u8];
+
+    fn to_bytes(&self) -> Result<Self::Bytes, Error> {
+        Ok(bytemuck::bytes_of(self.0))
+    }
+}
+
+#[cfg(all(feature = "raw", target_endian = "little"))]
+impl<'a, T: bytemuck::Pod> FromBytes<'a> for Raw<'a, T> {
+    fn from_bytes(data: &'a [u8]) -> Result<Self, Error> {
+        let x = bytemuck::try_from_bytes(data).map_err(|x| Error::msg(x.to_string()))?;
+        Ok(Raw(x))
+    }
+}
+
+#[cfg(all(feature = "raw", target_endian = "big"))]
+compile_error!("The raw feature is only supported on little endian targets");

convert/src/from_bytes.rs

@@ -1,14 +1,68 @@
 use crate::*;
 
+pub use extism_convert_macros::FromBytes;
+
 /// `FromBytes` is used to define how a type should be decoded when working with
 /// Extism memory. It is used for plugin output and host function input.
+///
+/// `FromBytes` can be derived by delegating encoding to generic type implementing
+/// `FromBytes`, e.g., [`Json`], [`Msgpack`].
+///
+/// ```
+/// use extism_convert::{Json, FromBytes};
+/// use serde::Deserialize;
+///
+/// #[derive(FromBytes, Deserialize, PartialEq, Debug)]
+/// #[encoding(Json)]
+/// struct Struct {
+///     hello: String,
+/// }
+///
+/// assert_eq!(Struct::from_bytes(br#"{"hello":"hi"}"#)?, Struct { hello: "hi".into() });
+/// # Ok::<(), extism_convert::Error>(())
+/// ```
+///
+/// Custom encodings can also be used, through new-types with a single generic
+/// argument, i.e., `Type<T>(T)`, that implement `FromBytesOwned` for the struct.
+///
+/// ```
+/// use std::str::{self, FromStr};
+/// use std::convert::Infallible;
+/// use extism_convert::{Error, FromBytes, FromBytesOwned};
+///
+/// // Custom serialization using `FromStr`
+/// struct StringEnc<T>(T);
+/// impl<T: FromStr> FromBytesOwned for StringEnc<T> where Error: From<<T as FromStr>::Err> {
+///     fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
+///         Ok(Self(str::from_utf8(data)?.parse()?))
+///     }
+/// }
+///
+/// #[derive(FromBytes, PartialEq, Debug)]
+/// #[encoding(StringEnc)]
+/// struct Struct {
+///     hello: String,
+/// }
+///
+/// impl FromStr for Struct {
+///     type Err = Infallible;
+///     fn from_str(s: &str) -> Result<Self, Infallible> {
+///         Ok(Self { hello: s.to_owned() })
+///     }
+/// }
+///
+/// assert_eq!(Struct::from_bytes(b"hi")?, Struct { hello: "hi".into() });
+/// # Ok::<(), extism_convert::Error>(())
+/// ```
 pub trait FromBytes<'a>: Sized {
     /// Decode a value from a slice of bytes
     fn from_bytes(data: &'a [u8]) -> Result<Self, Error>;
 }
 
-/// `FromBytesOwned` is similar to `FromBytes` but it doesn't borrow from the input slice.
-/// `FromBytes` is automatically implemented for all types that implement `FromBytesOwned`
+/// `FromBytesOwned` is similar to [`FromBytes`] but it doesn't borrow from the input slice.
+/// [`FromBytes`] is automatically implemented for all types that implement `FromBytesOwned`.
+///
+/// `FromBytesOwned` can be derived through [`#[derive(FromBytes)]`](FromBytes).
 pub trait FromBytesOwned: Sized {
     /// Decode a value from a slice of bytes, the resulting value should not borrow the input
     /// data.
@@ -87,6 +141,16 @@ impl FromBytesOwned for u32 {
     }
 }
 
+impl FromBytesOwned for bool {
+    fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
+        if let Some(x) = data.first() {
+            Ok(*x != 0)
+        } else {
+            Err(Error::msg("Expected one byte to read boolean value"))
+        }
+    }
+}
+
 impl FromBytesOwned for () {
     fn from_bytes_owned(_: &[u8]) -> Result<Self, Error> {
         Ok(())
@@ -98,3 +162,13 @@ impl<'a, T: FromBytes<'a>> FromBytes<'a> for std::io::Cursor<T> {
         Ok(std::io::Cursor::new(T::from_bytes(data)?))
     }
 }
+
+impl<'a, T: FromBytes<'a>> FromBytes<'a> for Option<T> {
+    fn from_bytes(data: &'a [u8]) -> Result<Self, Error> {
+        if data.is_empty() {
+            return Ok(None);
+        }
+
+        T::from_bytes(data).map(Some)
+    }
+}

convert/src/lib.rs

@@ -5,6 +5,9 @@
 //! similar to [axum extractors](https://docs.rs/axum/latest/axum/extract/index.html#intro) - they are
 //! implemented as a tuple struct with a single field that is meant to be extracted using pattern matching.
 
+// Makes proc-macros able to resolve `::extism_convert` correctly
+extern crate self as extism_convert;
+
 pub use anyhow::Error;
 
 mod encoding;
@@ -18,9 +21,15 @@ pub use encoding::{Base64, Json};
 #[cfg(feature = "msgpack")]
 pub use encoding::Msgpack;
 
+#[cfg(feature = "prost")]
+pub use encoding::Prost;
+
 #[cfg(feature = "protobuf")]
 pub use encoding::Protobuf;
 
+#[cfg(all(feature = "raw", target_endian = "little"))]
+pub use encoding::Raw;
+
 pub use from_bytes::{FromBytes, FromBytesOwned};
 pub use memory_handle::MemoryHandle;
 pub use to_bytes::ToBytes;

convert/src/tests.rs

@@ -20,6 +20,7 @@ fn roundtrip_json() {
 }
 
 #[test]
+#[cfg(feature = "msgpack")]
 fn roundtrip_msgpack() {
     let x = Testing {
         a: "foobar".to_string(),
@@ -37,3 +38,64 @@ fn roundtrip_base64() {
     let Base64(s): Base64<String> = FromBytes::from_bytes(bytes.as_bytes()).unwrap();
     assert_eq!(s, "this is a test");
 }
+
+#[test]
+fn rountrip_option() {
+    // `None` case
+    let e0: Option<Json<Testing>> = FromBytes::from_bytes(&[]).unwrap();
+    let b = e0.to_bytes().unwrap();
+    let e1: Option<Json<Testing>> = FromBytes::from_bytes(&b).unwrap();
+    assert!(e0.is_none());
+    assert_eq!(e0.is_none(), e1.is_none());
+
+    // `Some` case
+    let x = Testing {
+        a: "foobar".to_string(),
+        b: 123,
+        c: 456.7,
+    };
+    let bytes = Json(&x).to_bytes().unwrap();
+    let y: Option<Json<Testing>> = FromBytes::from_bytes(&bytes).unwrap();
+    let b = ToBytes::to_bytes(&y).unwrap();
+    let z: Option<Json<Testing>> = FromBytes::from_bytes(&b).unwrap();
+    assert_eq!(y.unwrap().0, z.unwrap().0);
+}
+
+#[test]
+fn check_bool() {
+    // `None` case
+    let a = true.to_bytes().unwrap();
+    let b = false.to_bytes().unwrap();
+    assert_ne!(a, b);
+
+    assert_eq!(a, [1]);
+    assert_eq!(b, [0]);
+}
+
+#[cfg(all(feature = "raw", target_endian = "little"))]
+mod raw_tests {
+    use crate::*;
+
+    #[test]
+    fn test_raw() {
+        #[derive(Debug, Clone, Copy, PartialEq)]
+        struct TestRaw {
+            a: i32,
+            b: f64,
+            c: bool,
+        }
+        unsafe impl bytemuck::Pod for TestRaw {}
+        unsafe impl bytemuck::Zeroable for TestRaw {}
+        let x = TestRaw {
+            a: 123,
+            b: 45678.91011,
+            c: true,
+        };
+        let raw = Raw(&x).to_bytes().unwrap();
+        let y = Raw::from_bytes(raw).unwrap();
+        assert_eq!(&x, y.0);
+
+        let y: Result<Raw<[u8; std::mem::size_of::<TestRaw>()]>, Error> = Raw::from_bytes(raw);
+        assert!(y.is_ok());
+    }
+}

convert/src/to_bytes.rs

@@ -1,7 +1,58 @@
 use crate::*;
 
+pub use extism_convert_macros::ToBytes;
+
 /// `ToBytes` is used to define how a type should be encoded when working with
 /// Extism memory. It is used for plugin input and host function output.
+///
+/// `ToBytes` can be derived by delegating encoding to generic type implementing
+/// `ToBytes`, e.g., [`Json`], [`Msgpack`].
+///
+/// ```
+/// use extism_convert::{Json, ToBytes};
+/// use serde::Serialize;
+///
+/// #[derive(ToBytes, Serialize)]
+/// #[encoding(Json)]
+/// struct Struct {
+///     hello: String,
+/// }
+///
+/// assert_eq!(Struct { hello: "hi".into() }.to_bytes()?, br#"{"hello":"hi"}"#);
+/// # Ok::<(), extism_convert::Error>(())
+/// ```
+///
+/// But custom types can also be used, as long as they are new-types with a single
+/// generic argument, i.e., `Type<T>(T)`, that implement `ToBytes` for the struct.
+///
+/// ```
+/// use extism_convert::{Error, ToBytes};
+///
+/// // Custom serialization using `ToString`
+/// struct StringEnc<T>(T);
+/// impl<T: ToString> ToBytes<'_> for StringEnc<&T> {
+///     type Bytes = String;
+///     
+///     fn to_bytes(&self) -> Result<String, Error> {
+///         Ok(self.0.to_string())
+///     }
+/// }
+///
+/// #[derive(ToBytes)]
+/// #[encoding(StringEnc)]
+/// struct Struct {
+///     hello: String,
+/// }
+///
+/// impl ToString for Struct {
+///    fn to_string(&self) -> String {
+///        self.hello.clone()     
+///    }
+/// }
+///
+/// assert_eq!(Struct { hello: "hi".into() }.to_bytes()?, b"hi");
+/// # Ok::<(), Error>(())
+/// ```
 pub trait ToBytes<'a> {
     /// A configurable byte slice representation, allows any type that implements `AsRef<[u8]>`
     type Bytes: AsRef<[u8]>;
@@ -10,21 +61,21 @@ pub trait ToBytes<'a> {
     fn to_bytes(&self) -> Result<Self::Bytes, Error>;
 }
 
-impl<'a> ToBytes<'a> for () {
+impl ToBytes<'_> for () {
     type Bytes = [u8; 0];
     fn to_bytes(&self) -> Result<Self::Bytes, Error> {
         Ok([])
     }
 }
 
-impl<'a> ToBytes<'a> for Vec<u8> {
+impl ToBytes<'_> for Vec<u8> {
     type Bytes = Vec<u8>;
     fn to_bytes(&self) -> Result<Self::Bytes, Error> {
         Ok(self.clone())
     }
 }
 
-impl<'a> ToBytes<'a> for String {
+impl ToBytes<'_> for String {
     type Bytes = String;
     fn to_bytes(&self) -> Result<Self::Bytes, Error> {
         Ok(self.clone())
@@ -45,7 +96,7 @@ impl<'a> ToBytes<'a> for &'a str {
     }
 }
 
-impl<'a> ToBytes<'a> for f64 {
+impl ToBytes<'_> for f64 {
     type Bytes = [u8; 8];
 
     fn to_bytes(&self) -> Result<Self::Bytes, Error> {
@@ -53,7 +104,7 @@ impl<'a> ToBytes<'a> for f64 {
     }
 }
 
-impl<'a> ToBytes<'a> for f32 {
+impl ToBytes<'_> for f32 {
     type Bytes = [u8; 4];
 
     fn to_bytes(&self) -> Result<Self::Bytes, Error> {
@@ -61,7 +112,7 @@ impl<'a> ToBytes<'a> for f32 {
     }
 }
 
-impl<'a> ToBytes<'a> for i64 {
+impl ToBytes<'_> for i64 {
     type Bytes = [u8; 8];
 
     fn to_bytes(&self) -> Result<Self::Bytes, Error> {
@@ -69,7 +120,7 @@ impl<'a> ToBytes<'a> for i64 {
     }
 }
 
-impl<'a> ToBytes<'a> for i32 {
+impl ToBytes<'_> for i32 {
     type Bytes = [u8; 4];
 
     fn to_bytes(&self) -> Result<Self::Bytes, Error> {
@@ -77,7 +128,7 @@ impl<'a> ToBytes<'a> for i32 {
     }
 }
 
-impl<'a> ToBytes<'a> for u64 {
+impl ToBytes<'_> for u64 {
     type Bytes = [u8; 8];
 
     fn to_bytes(&self) -> Result<Self::Bytes, Error> {
@@ -85,7 +136,7 @@ impl<'a> ToBytes<'a> for u64 {
     }
 }
 
-impl<'a> ToBytes<'a> for u32 {
+impl ToBytes<'_> for u32 {
     type Bytes = [u8; 4];
 
     fn to_bytes(&self) -> Result<Self::Bytes, Error> {
@@ -93,6 +144,14 @@ impl<'a> ToBytes<'a> for u32 {
     }
 }
 
+impl ToBytes<'_> for bool {
+    type Bytes = [u8; 1];
+
+    fn to_bytes(&self) -> Result<Self::Bytes, Error> {
+        Ok([*self as u8])
+    }
+}
+
 impl<'a, T: ToBytes<'a>> ToBytes<'a> for &'a T {
     type Bytes = T::Bytes;
 
@@ -100,3 +159,26 @@ impl<'a, T: ToBytes<'a>> ToBytes<'a> for &'a T {
         <T as ToBytes>::to_bytes(self)
     }
 }
+
+impl<'a, T: ToBytes<'a>> ToBytes<'a> for Option<T> {
+    type Bytes = Vec<u8>;
+
+    fn to_bytes(&self) -> Result<Self::Bytes, Error> {
+        match self {
+            Some(x) => x.to_bytes().map(|x| x.as_ref().to_vec()),
+            None => Ok(vec![]),
+        }
+    }
+}
+
+#[test]
+fn test() {
+    use extism_convert::{Json, ToBytes};
+    use serde::Serialize;
+
+    #[derive(ToBytes, Serialize)]
+    #[encoding(Json)]
+    struct Struct {
+        hello: String,
+    }
+}

example-schema.yaml

@@ -0,0 +1,28 @@
+# yaml-language-server: $schema=https://xtp.dylibso.com/assets/wasm/schema.json
+# Learn more at https://docs.xtp.dylibso.com/docs/concepts/xtp-schema
+version: v1-draft
+exports: 
+  CountVowels:
+      input: 
+          type: string
+          contentType: text/plain; charset=utf-8
+      output:
+          $ref: "#/components/schemas/VowelReport"
+          contentType: application/json
+components:
+  schemas:
+    VowelReport:
+      description: The result of counting vowels on the Vowels input.
+      properties:
+        count:
+          type: integer
+          format: int32
+          description: The count of vowels for input string.
+        total:
+          type: integer
+          format: int32
+          description: The cumulative amount of vowels counted, if this keeps state across multiple function calls.
+          nullable: true
+        vowels:
+          type: string
+          description: The set of vowels used to get the count, e.g. "aAeEiIoOuU"

extism-maturin/Cargo.toml

@@ -1,11 +1,13 @@
 [package]
 name = "extism-sys"
 edition.workspace = true
-authors.workspace = true
 license.workspace = true
 homepage.workspace = true
 repository.workspace = true
 version.workspace = true
+# Explicitly omit authors from this package since our Cargo "authors" are
+# incompatible with PyPI's requirements.
+# authors.workspace = true
 
 [lib]
 name = "extism_sys"

kernel/Cargo.toml

@@ -5,6 +5,9 @@ edition = "2021"
 
 [dependencies]
 
+[dev-dependencies]
+wasm-bindgen-test = "0.3.39"
+
 [features]
 default = ["bounds-checking"]
 bounds-checking = []

kernel/build.sh

@@ -17,6 +17,9 @@ done
 
 cargo build --package extism-runtime-kernel --bin extism-runtime --release --target wasm32-unknown-unknown $CARGO_FLAGS
 cp target/wasm32-unknown-unknown/release/extism-runtime.wasm .
-wasm-strip extism-runtime.wasm
-mv extism-runtime.wasm ../runtime/src/extism-runtime.wasm
 
+wasm-tools parse extism-context.wat -o extism-context.wasm
+wasm-merge --enable-reference-types ./extism-runtime.wasm runtime extism-context.wasm context -o ../runtime/src/extism-runtime.wasm
+rm extism-context.wasm
+rm extism-runtime.wasm
+wasm-strip ../runtime/src/extism-runtime.wasm

kernel/extism-context.wat

@@ -0,0 +1,3 @@
+(module
+  (global (export "extism_context") (mut externref) (ref.null extern))
+)

kernel/src/bin/extism-runtime.rs

@@ -3,7 +3,7 @@
 
 pub use extism_runtime_kernel::*;
 
-#[cfg(target_arch = "wasm32")]
+#[cfg(all(target_arch = "wasm32", not(test)))]
 #[panic_handler]
 fn panic(_info: &core::panic::PanicInfo) -> ! {
     core::arch::wasm32::unreachable()

kernel/src/lib.rs

@@ -39,7 +39,7 @@
 use core::sync::atomic::*;
 
 pub type Pointer = u64;
-pub type Length = u64;
+pub type Handle = u64;
 
 /// WebAssembly page size
 const PAGE_SIZE: usize = 65536;
@@ -81,13 +81,13 @@ pub struct MemoryRoot {
     /// Offset of error block
     pub error: AtomicU64,
     /// Input position in memory
-    pub input_offset: Pointer,
+    pub input_offset: Handle,
     /// Input length
-    pub input_length: Length,
+    pub input_length: u64,
     /// Output position in memory
     pub output_offset: Pointer,
     /// Output length
-    pub output_length: Length,
+    pub output_length: u64,
     /// A pointer to the start of the first block
     pub blocks: [MemoryBlock; 0],
 }
@@ -139,7 +139,9 @@ impl MemoryRoot {
         }
 
         // Ensure that at least one page is allocated to store the `MemoryRoot` data
-        if core::arch::wasm32::memory_size(0) == 0 && core::arch::wasm32::memory_grow(0, 1) == usize::MAX {
+        if core::arch::wasm32::memory_size(0) == 0
+            && core::arch::wasm32::memory_grow(0, 1) == usize::MAX
+        {
             core::arch::wasm32::unreachable()
         }
 
@@ -168,12 +170,15 @@ impl MemoryRoot {
 
     /// Resets the position of the allocator and zeroes out all allocations
     pub unsafe fn reset(&mut self) {
+        // Clear allocated data
+        let self_position = self.position.fetch_and(0, Ordering::SeqCst);
         core::ptr::write_bytes(
             self.blocks.as_mut_ptr() as *mut u8,
-            0,
-            self.length.load(Ordering::Acquire) as usize,
+            MemoryStatus::Unused as u8,
+            self_position as usize,
         );
-        self.position.store(0, Ordering::Release);
+
+        // Clear extism runtime metadata
         self.error.store(0, Ordering::Release);
         self.input_offset = 0;
         self.input_length = 0;
@@ -193,15 +198,15 @@ impl MemoryRoot {
     fn pointer_in_bounds_fast(p: Pointer) -> bool {
         // Similar to `pointer_in_bounds` but less accurate on the upper bound. This uses the total memory size,
         // instead of checking `MemoryRoot::length`
-        let end = core::arch::wasm32::memory_size(0) << 16;
-        p >= core::mem::size_of::<Self>() as Pointer && p <= end as Pointer
+        let end = (core::arch::wasm32::memory_size(0) as u64) << 16;
+        p >= core::mem::size_of::<Self>() as Pointer && p <= end as u64
     }
 
     // Find a block that is free to use, this can be a new block or an existing freed block. The `self_position` argument
     // is used to avoid loading the allocators position more than once when performing an allocation.
     unsafe fn find_free_block(
         &mut self,
-        length: Length,
+        length: u64,
         self_position: u64,
     ) -> Option<&'static mut MemoryBlock> {
         // Get the first block
@@ -223,7 +228,7 @@ impl MemoryRoot {
             if status == MemoryStatus::Free as u8 && b.size >= length as usize {
                 // Split block if there is too much excess
                 if b.size - length as usize >= 128 {
-                    b.size -= length as usize;
+                    b.size -= length as usize + core::mem::size_of::<MemoryBlock>();
                     b.used = 0;
 
                     let block1 = b.data.as_mut_ptr().add(b.size) as *mut MemoryBlock;
@@ -247,30 +252,37 @@ impl MemoryRoot {
 
     /// Create a new `MemoryBlock`, when `Some(block)` is returned, `block` will contain at least enough room for `length` bytes
     /// but may be as large as `length` + `BLOCK_SPLIT_SIZE` bytes. When `None` is returned the allocation has failed.
-    pub unsafe fn alloc(&mut self, length: Length) -> Option<&'static mut MemoryBlock> {
+    pub unsafe fn alloc(&mut self, length: u64) -> Option<&'static mut MemoryBlock> {
         let self_position = self.position.load(Ordering::Acquire);
         let self_length = self.length.load(Ordering::Acquire);
-        let b = self.find_free_block(length, self_position);
-
-        // If there's a free block then re-use it
-        if let Some(b) = b {
-            b.used = length as usize;
-            b.status
-                .store(MemoryStatus::Active as u8, Ordering::Release);
-            return Some(b);
-        }
 
         // Get the current index for a new block
         let curr = self.blocks.as_ptr() as u64 + self_position;
 
         // Get the number of bytes available
         let mem_left = self_length - self_position - core::mem::size_of::<MemoryRoot>() as u64;
+        let length_with_block = length + core::mem::size_of::<MemoryBlock>() as u64;
 
         // When the allocation is larger than the number of bytes available
         // we will need to try to grow the memory
-        if length >= mem_left {
+        if length_with_block >= mem_left {
+            // If the current position is large enough to hold the length of the block being
+            // allocated then check for existing free blocks that can be re-used before
+            // growing memory
+            if length_with_block <= self_position {
+                let b = self.find_free_block(length, self_position);
+
+                // If there's a free block then re-use it
+                if let Some(b) = b {
+                    b.used = length as usize;
+                    b.status
+                        .store(MemoryStatus::Active as u8, Ordering::Release);
+                    return Some(b);
+                }
+            }
+
             // Calculate the number of pages needed to cover the remaining bytes
-            let npages = num_pages(length - mem_left);
+            let npages = num_pages(length_with_block - mem_left);
             let x = core::arch::wasm32::memory_grow(0, npages);
             if x == usize::MAX {
                 return None;
@@ -301,9 +313,26 @@ impl MemoryRoot {
         if !Self::pointer_in_bounds_fast(offs) {
             return None;
         }
-        let ptr = offs - core::mem::size_of::<MemoryBlock>() as u64;
-        let ptr = ptr as *mut MemoryBlock;
-        Some(&mut *ptr)
+
+        // Get the first block
+        let mut block = self.blocks.as_mut_ptr();
+
+        // Only loop while the block pointer is less then the current position
+        while (block as u64) < self.blocks.as_ptr() as u64 + offs {
+            let b = &mut *block;
+
+            // Get the block status, this lets us know if we are able to re-use it
+            let status = b.status.load(Ordering::Acquire);
+
+            if status == MemoryStatus::Active as u8 && b.data.as_ptr() as Pointer == offs {
+                return Some(b);
+            }
+
+            // Get the next block
+            block = b.next_ptr();
+        }
+
+        None
     }
 }
 
@@ -328,21 +357,21 @@ impl MemoryBlock {
 
 /// Allocate a block of memory and return the offset
 #[no_mangle]
-pub unsafe fn alloc(n: Length) -> Pointer {
+pub unsafe fn alloc(n: u64) -> Handle {
     if n == 0 {
         return 0;
     }
     let region = MemoryRoot::new();
     let block = region.alloc(n);
     match block {
-        Some(block) => block.data.as_mut_ptr() as Pointer,
+        Some(block) => block.data.as_mut_ptr() as Handle,
         None => 0,
     }
 }
 
 /// Free allocated memory
 #[no_mangle]
-pub unsafe fn free(p: Pointer) {
+pub unsafe fn free(p: Handle) {
     if p == 0 {
         return;
     }
@@ -360,13 +389,42 @@ pub unsafe fn free(p: Pointer) {
 }
 
 /// Get the length of an allocated memory block
+///
+/// Note: this should only be called on memory handles returned
+/// by a call to `alloc` - it will return garbage on invalid offsets
 #[no_mangle]
-pub unsafe fn length(p: Pointer) -> Length {
+pub unsafe fn length_unsafe(p: Handle) -> u64 {
     if p == 0 {
         return 0;
     }
+
+    if !MemoryRoot::pointer_in_bounds_fast(p) {
+        return 0;
+    }
+
+    let ptr = p - core::mem::size_of::<MemoryBlock>() as u64;
+    let block = &mut *(ptr as *mut MemoryBlock);
+
+    // Simplest sanity check to verify the pointer is a block
+    if block.status.load(Ordering::Acquire) != MemoryStatus::Active as u8 {
+        return 0;
+    }
+
+    block.used as u64
+}
+
+/// Get the length but returns 0 if the offset is not a valid handle.
+///
+/// Note: this function walks each node in the allocations list, which ensures correctness, but is also
+/// slow
+#[no_mangle]
+pub unsafe fn length(p: Pointer) -> u64 {
+    if p == 0 {
+        return 0;
+    }
+
     if let Some(block) = MemoryRoot::new().find_block(p) {
-        block.used as Length
+        block.used as u64
     } else {
         0
     }
@@ -394,24 +452,24 @@ pub unsafe fn load_u64(p: Pointer) -> u64 {
 
 /// Load a byte from the input data
 #[no_mangle]
-pub unsafe fn input_load_u8(p: Pointer) -> u8 {
+pub unsafe fn input_load_u8(offset: u64) -> u8 {
     let root = MemoryRoot::new();
     #[cfg(feature = "bounds-checking")]
-    if p >= root.input_length {
+    if offset >= root.input_length {
         return 0;
     }
-    *((root.input_offset + p) as *mut u8)
+    *((root.input_offset + offset) as *mut u8)
 }
 
 /// Load a u64 from the input data
 #[no_mangle]
-pub unsafe fn input_load_u64(p: Pointer) -> u64 {
+pub unsafe fn input_load_u64(offset: u64) -> u64 {
     let root = MemoryRoot::new();
     #[cfg(feature = "bounds-checking")]
-    if p + core::mem::size_of::<u64>() as Pointer > root.input_length {
+    if offset + core::mem::size_of::<u64>() as u64 > root.input_length {
         return 0;
     }
-    *((root.input_offset + p) as *mut u64)
+    *((root.input_offset + offset) as *mut u64)
 }
 
 /// Write a byte in Extism-managed memory
@@ -435,22 +493,28 @@ pub unsafe fn store_u64(p: Pointer, x: u64) {
 }
 
 /// Set the range of the input data in memory
+/// h must always be a handle so that length works on it
+/// len must match length(handle)
+/// **Note**: this function takes ownership of the handle passed in
+/// the caller should not `free` this value
 #[no_mangle]
-pub unsafe fn input_set(p: Pointer, len: Length) {
+pub unsafe fn input_set(h: Handle, len: u64) {
     let root = MemoryRoot::new();
     #[cfg(feature = "bounds-checking")]
     {
-        if !root.pointer_in_bounds(p) || !root.pointer_in_bounds(p + len - 1) {
+        if !root.pointer_in_bounds(h) || !root.pointer_in_bounds(h + len - 1) {
             return;
         }
     }
-    root.input_offset = p;
+    root.input_offset = h;
     root.input_length = len;
 }
 
 /// Set the range of the output data in memory
+/// **Note**: this function takes ownership of the handle passed in
+/// the caller should not `free` this value
 #[no_mangle]
-pub unsafe fn output_set(p: Pointer, len: Length) {
+pub unsafe fn output_set(p: Pointer, len: u64) {
     let root = MemoryRoot::new();
     #[cfg(feature = "bounds-checking")]
     {
@@ -464,25 +528,25 @@ pub unsafe fn output_set(p: Pointer, len: Length) {
 
 /// Get the input length
 #[no_mangle]
-pub fn input_length() -> Length {
+pub fn input_length() -> u64 {
     unsafe { MemoryRoot::new().input_length }
 }
 
 /// Get the input offset in Exitsm-managed memory
 #[no_mangle]
-pub fn input_offset() -> Length {
+pub fn input_offset() -> Handle {
     unsafe { MemoryRoot::new().input_offset }
 }
 
 /// Get the output length
 #[no_mangle]
-pub fn output_length() -> Length {
+pub fn output_length() -> u64 {
     unsafe { MemoryRoot::new().output_length }
 }
 
 /// Get the output offset in Extism-managed memory
 #[no_mangle]
-pub unsafe fn output_offset() -> Length {
+pub unsafe fn output_offset() -> Pointer {
     MemoryRoot::new().output_offset
 }
 
@@ -492,32 +556,60 @@ pub unsafe fn reset() {
     MemoryRoot::new().reset()
 }
 
-/// Set the error message offset
+/// Set the error message offset, the handle passed to this
+/// function should not be freed after this call
+/// **Note**: this function takes ownership of the handle passed in
+/// the caller should not `free` this value
 #[no_mangle]
-pub unsafe fn error_set(ptr: Pointer) {
+pub unsafe fn error_set(h: Handle) {
     let root = MemoryRoot::new();
 
     // Allow ERROR to be set to 0
-    if ptr == 0 {
-        root.error.store(ptr, Ordering::SeqCst);
+    if h == 0 {
+        root.error.store(h, Ordering::SeqCst);
         return;
     }
 
     #[cfg(feature = "bounds-checking")]
-    if !root.pointer_in_bounds(ptr) {
+    if !root.pointer_in_bounds(h) {
         return;
     }
-    root.error.store(ptr, Ordering::SeqCst);
+    root.error.store(h, Ordering::SeqCst);
 }
 
 /// Get the error message offset, if it's `0` then no error has been set
 #[no_mangle]
-pub unsafe fn error_get() -> Pointer {
+pub unsafe fn error_get() -> Handle {
     MemoryRoot::new().error.load(Ordering::SeqCst)
 }
 
 /// Get the position of the allocator, this can be used as an indication of how many bytes are currently in-use
 #[no_mangle]
-pub unsafe fn memory_bytes() -> Length {
+pub unsafe fn memory_bytes() -> u64 {
     MemoryRoot::new().length.load(Ordering::Acquire)
 }
+
+#[cfg(test)]
+mod test {
+    use crate::*;
+    use wasm_bindgen_test::*;
+
+    #[wasm_bindgen_test]
+    fn test_oom() {
+        let size = 1024 * 1024 * 5;
+
+        let mut last = 0;
+        for _ in 0..1024 {
+            unsafe {
+                let ptr = alloc(size);
+                last = ptr;
+                if ptr == 0 {
+                    break;
+                }
+                assert_eq!(length(ptr), size);
+            }
+        }
+
+        assert_eq!(last, 0);
+    }
+}

kernel/test.sh

@@ -0,0 +1,5 @@
+# install wasm-bindgen-cli to get wasm-bindgen-runner if it is not installed yet
+which wasm-bindgen-test-runner 1>/dev/null || cargo install -f wasm-bindgen-cli
+
+# run tests with the wasm-bindgen-runner
+CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER=wasm-bindgen-test-runner cargo test --release --target=wasm32-unknown-unknown

libextism/.clang-format

@@ -0,0 +1,2 @@
+BasedOnStyle: LLVM
+IndentWidth: 2

libextism/API.md

@@ -14,7 +14,9 @@ Create a new plugin.
 - `functions`: is an array of `ExtismFunction*`
 - `n_functions`: is the number of functions
 - `with_wasi`: enables/disables WASI
-- `errmsg`: error message during plugin creation
+- `errmsg`: error message during plugin creation, this should be freed with
+  `extism_plugin_new_error_free`
+
 
 ```c
 ExtismPlugin extism_plugin_new(const uint8_t *wasm,
@@ -24,6 +26,17 @@ ExtismPlugin extism_plugin_new(const uint8_t *wasm,
                                bool with_wasi,
                                char **errmsg);
 ```
+---
+
+### `extism_plugin_new_error_free`
+
+Frees the error message returned when creating a plugin
+
+```c
+void extism_plugin_new_error_free(char *err);
+```
+
+---
 
 ### `extism_plugin_free`
 
@@ -33,6 +46,8 @@ Remove a plugin from the registry and free associated memory.
 void extism_plugin_free(ExtismPlugin *plugin);
 ```
 
+---
+
 ### `extism_plugin_config`
 
 Update plugin config values, this will merge with the existing values.
@@ -43,6 +58,8 @@ bool extism_plugin_config(ExtismPlugin *plugin,
                           ExtismSize json_size);
 ```
 
+---
+
 ### `extism_plugin_function_exists`
 
 Returns true if `func_name` exists.
@@ -52,6 +69,8 @@ bool extism_plugin_function_exists(ExtismPlugin *plugin,
                                    const char *func_name);
 ```
 
+---
+
 ### `extism_plugin_call`
 
 Call a function.
@@ -59,6 +78,8 @@ Call a function.
 - `data`: is the input data
 - `data_len`: is the length of `data`
 
+Returns `0` when the call is successful.
+
 ```c
 int32_t extism_plugin_call(ExtismPlugin *plugin,
                            const char *func_name,
@@ -66,6 +87,28 @@ int32_t extism_plugin_call(ExtismPlugin *plugin,
                            ExtismSize data_len);
 ```
 
+---
+
+### `extism_plugin_call_with_host_context`
+
+Call a function with additional host context that can be accessed from inside host functions.
+- `func_name`: is the function to call
+- `data`: is the input data
+- `data_len`: is the length of `data`
+- `host_ctx`: an opaque pointer that can be accessed in host functions
+
+Returns `0` when the call is successful.
+
+```c
+int32_t extism_plugin_call_with_host_context(ExtismPlugin *plugin,
+                           const char *func_name,
+                           const uint8_t *data,
+                           ExtismSize data_len,
+                           void *host_ctx);
+```
+
+---
+
 ### `extism_plugin_error`
 
 Get the error associated with a `Plugin`
@@ -74,6 +117,8 @@ Get the error associated with a `Plugin`
 const char *extism_plugin_error(ExtismPlugin *plugin);
 ```
 
+---
+
 ### `extism_plugin_output_length`
 
 Get the length of a plugin's output data.
@@ -82,6 +127,8 @@ Get the length of a plugin's output data.
 ExtismSize extism_plugin_output_length(ExtismPlugin *plugin);
 ```
 
+---
+
 ### `extism_plugin_output_data`
 
 Get the plugin's output data.
@@ -90,6 +137,18 @@ Get the plugin's output data.
 const uint8_t *extism_plugin_output_data(ExtismPlugin *plugin);
 ```
 
+---
+
+### `extism_plugin_reset`
+
+Reset the Extism runtime, this will invalidate all allocated memory.
+
+```c
+bool extism_plugin_reset(ExtismPlugin *plugin);
+```
+
+---
+
 ### `extism_log_file`
 
 Set log file and level.
@@ -98,6 +157,8 @@ Set log file and level.
 bool extism_log_file(const char *filename, const char *log_level);
 ```
 
+---
+
 ### `extism_log_custom`
 
 Enable a custom log handler, this will buffer logs until `extism_log_drain`
@@ -107,6 +168,8 @@ is called Log level should be one of: info, error, trace, debug, warn
 bool extism_log_custom(const char *log_level);
 ```
 
+---
+
 ### `extism_log_drain`
 
 Calls the provided callback function for each buffered log line.
@@ -116,6 +179,8 @@ This is only needed when `extism_log_custom` is used.
 void extism_log_drain(void (*handler)(const char *, uintptr_t));
 ```
 
+---
+
 ### `extism_version`
 
 Get the Extism version string.
@@ -124,6 +189,8 @@ Get the Extism version string.
 const char *extism_version(void);
 ```
 
+---
+
 ### `extism_current_plugin_memory`
 
 Returns a pointer to the memory of the currently running plugin
@@ -132,6 +199,19 @@ Returns a pointer to the memory of the currently running plugin
 uint8_t *extism_current_plugin_memory(ExtismCurrentPlugin *plugin);
 ```
 
+---
+
+### `extism_current_plugin_host_context`
+
+Get access to the host context, passed in using `extism_plugin_call_with_host_context`
+
+```c
+void *extism_current_plugin_host_context(ExtismCurrentPlugin *plugin);
+```
+
+---
+
+
 ### `extism_current_plugin_memory_alloc`
 
 Allocate a memory block in the currently running plugin
@@ -140,6 +220,8 @@ Allocate a memory block in the currently running plugin
 uint64_t extism_current_plugin_memory_alloc(ExtismCurrentPlugin *plugin, ExtismSize n);
 ```
 
+---
+
 ### `extism_current_plugin_memory_length`
 
 Get the length of an allocated block
@@ -148,6 +230,8 @@ Get the length of an allocated block
 ExtismSize extism_current_plugin_memory_length(ExtismCurrentPlugin *plugin, ExtismSize n);
 ```
 
+---
+
 ### `extism_current_plugin_memory_free`
 
 Free an allocated memory block
@@ -156,6 +240,8 @@ Free an allocated memory block
 void extism_current_plugin_memory_free(ExtismCurrentPlugin *plugin, uint64_t ptr);
 ```
 
+---
+
 ### `extism_function_new`
 Create a new host function
 - `name`: function name, this should be valid UTF-8
@@ -182,6 +268,8 @@ ExtismFunction *extism_function_new(const char *name,
                                     void (*free_user_data)(void *_));
 ```
 
+---
+
 ### `extism_function_set_namespace`
 
 Set the namespace of an `ExtismFunction`
@@ -190,6 +278,8 @@ Set the namespace of an `ExtismFunction`
 void extism_function_set_namespace(ExtismFunction *ptr, const char *namespace_);
 ```
 
+---
+
 ### `extism_function_free`
 
 Free an `ExtismFunction`
@@ -198,6 +288,8 @@ Free an `ExtismFunction`
 void extism_function_free(ExtismFunction *ptr);
 ```
 
+---
+
 ### `extism_plugin_cancel_handle`
 
 Get handle for plugin cancellation
@@ -206,6 +298,8 @@ Get handle for plugin cancellation
 const ExtismCancelHandle *extism_plugin_cancel_handle(const ExtismPlugin *plugin);
 ```
 
+---
+
 ### `extism_plugin_cancel`
 
 Cancel a running plugin from another thread
@@ -214,12 +308,14 @@ Cancel a running plugin from another thread
 bool extism_plugin_cancel(const ExtismCancelHandle *handle);
 ```
 
+---
+
 ## Type definitions:
 
 ### `ExtismPlugin`
 
 ```c
-typedef int32_t ExtismPlugin;
+typedef struct ExtismPlugin ExtismPlugin;
 ```
 
 ### `ExtismSize`
@@ -251,3 +347,5 @@ typedef struct ExtismCurrentPlugin ExtismCurrentPlugin;
 ```c
 typedef struct ExtismCancelHandle ExtismCancelHandle;
 ```
+
+

libextism/Makefile

@@ -4,7 +4,7 @@ build:
 
 .PHONY: static
 static:
-	$(CC) -g -o example example.c -l:libextism.a -lm
+	$(CC) -g -o example example.c -l:libextism.a -lm -lpthread
 
 # if needed, set PKG_CONFIG_PATH= to the directory with extism*.pc installed
 LDFLAGS=`pkg-config --libs extism`

libextism/README.md

@@ -56,7 +56,6 @@ Since you may not have an Extism plug-in on hand to test, let's load a demo plug
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <unistd.h>
 
 void print_plugin_output(ExtismPlugin *plugin, int32_t rc){
   if (rc != EXTISM_SUCCESS) {
@@ -64,9 +63,9 @@ void print_plugin_output(ExtismPlugin *plugin, int32_t rc){
     return;
   }
 
-  size_t outlen = extism_plugin_output_length(plugin);
+  ExtismSize outlen = extism_plugin_output_length(plugin);
   const uint8_t *out = extism_plugin_output_data(plugin);
-  write(STDOUT_FILENO, out, outlen);
+  fwrite(out, 1, outlen, stdout);
 }
 
 int main(void) {
@@ -106,9 +105,9 @@ if (rc != EXTISM_SUCCESS) {
   exit(2);
 }
 
-size_t outlen = extism_plugin_output_length(plugin);
+ExtismSize outlen = extism_plugin_output_length(plugin);
 const uint8_t *out = extism_plugin_output_data(plugin);
-write(STDOUT_FILENO, out, outlen);
+fwrite(out, 1, outlen, stdout);
 ```
 
 Will print
@@ -171,7 +170,6 @@ We want to expose two functions to our plugin, `kv_write(key: String, value: Byt
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <unistd.h>
 
 // A stubbed out KV store
 typedef struct KVStore KVStore;
@@ -184,14 +182,14 @@ extern const uint32_t fake_kv_store_get(KVStore *kv, const char *key,
 
 // Our host functions to access the fake KV store
 void kv_get(ExtismCurrentPlugin *plugin, const ExtismVal *inputs,
-            size_t ninputs, ExtismVal *outputs, size_t noutputs,
+            ExtismSize ninputs, ExtismVal *outputs, ExtismSize noutputs,
             void *userdata) {
   // Cast the userdata pointer
   KVStore *kv = (KVStore *)userdata;
 
   // Get the offset to the key in the plugin memory
   uint64_t offs = inputs[0].v.i64;
-  size_t keylen = extism_current_plugin_memory_length(plugin, offs);
+  ExtismSize keylen = extism_current_plugin_memory_length(plugin, offs);
 
   // Allocate a new block to return
   uint64_t outoffs =
@@ -205,23 +203,23 @@ void kv_get(ExtismCurrentPlugin *plugin, const ExtismVal *inputs,
   *(uint64_t *)(extism_current_plugin_memory(plugin) + outoffs) = value;
 
   // Return the offset to our allocated block
-  outputs[0].t = PTR;
+  outputs[0].t = EXTISM_PTR;
   outputs[0].v.i64 = outoffs;
 }
 
 void kv_set(ExtismCurrentPlugin *plugin, const ExtismVal *inputs,
-            size_t ninputs, ExtismVal *outputs, size_t noutputs,
+            ExtismSize ninputs, ExtismVal *outputs, ExtismSize noutputs,
             void *userdata) {
   // Cast the userdata pointer
   KVStore *kv = (KVStore *)userdata;
 
   // Get the offset to the key in the plugin memory
   uint64_t keyoffs = inputs[0].v.i64;
-  size_t keylen = extism_current_plugin_memory_length(plugin, keyoffs);
+  ExtismSize keylen = extism_current_plugin_memory_length(plugin, keyoffs);
 
   // Get the offset to the value in the plugin memory
   uint64_t valueoffs = inputs[1].v.i64;
-  size_t valuelen = extism_current_plugin_memory_length(plugin, valueoffs);
+  ExtismSize valuelen = extism_current_plugin_memory_length(plugin, valueoffs);
 
   // Set key => value
   fake_kv_store_set(
@@ -234,13 +232,13 @@ int main(void) {
   const char *manifest = "{\"wasm\": [{\"url\": "
                          "\"https://github.com/extism/plugins/releases/latest/"
                          "download/count_vowels_kvstore.wasm\"}]}";
-  const ExtismValType kv_get_inputs[] = {PTR};
-  const ExtismValType kv_get_outputs[] = {PTR};
+  const ExtismValType kv_get_inputs[] = {EXTISM_PTR};
+  const ExtismValType kv_get_outputs[] = {EXTISM_PTR};
   ExtismFunction *kv_get_fn = extism_function_new(
       "kv_get", kv_get_inputs, 1, kv_get_outputs, 1, kv_get, kv, NULL);
 
-  const ExtismValType kv_set_inputs[] = {PTR};
-  const ExtismValType kv_set_outputs[] = {PTR};
+  const ExtismValType kv_set_inputs[] = {EXTISM_PTR};
+  const ExtismValType kv_set_outputs[] = {EXTISM_PTR};
   ExtismFunction *kv_set_fn = extism_function_new(
       "kv_set", kv_set_inputs, 1, kv_set_outputs, 1, kv_set, kv, NULL);
   const ExtismFunction *functions[] = {kv_get_fn};
@@ -288,5 +286,3 @@ print_plugin_output(plugin, extism_plugin_call(plugin, "count_vowels",
 # => Writing value=6 from key=count-vowels"
 # => {"count": 3, "total": 6, "vowels": "aeiouAEIOU"}
 ```
-
-

libextism/example.c

@@ -9,7 +9,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
-void log_handler(const char *line, uintptr_t length) {
+void log_handler(const char *line, ExtismSize length) {
   fwrite(line, length, 1, stderr);
 }
 
@@ -63,8 +63,8 @@ int main(int argc, char *argv[]) {
 
   size_t len = 0;
   uint8_t *data = read_file("../wasm/code-functions.wasm", &len);
-  ExtismValType inputs[] = {PTR};
-  ExtismValType outputs[] = {PTR};
+  ExtismValType inputs[] = {EXTISM_PTR};
+  ExtismValType outputs[] = {EXTISM_PTR};
   ExtismFunction *f =
       extism_function_new("hello_world", inputs, 1, outputs, 1, hello_world,
                           "Hello, again!", free_data);

libextism/extism-static.pc.in

@@ -5,6 +5,6 @@ includedir=${prefix}/include
 Version: 1.0.0
 Name: Extism
 Description: The framework for building with WebAssembly (wasm).
-Libs: -L${libdir} -l:libextism.a
-Libs.private: -lm
+Libs: ${libdir}/libextism.a
+Libs.private: -lm -lpthread
 Cflags: -I${includedir}

libextism/extism.pc.in

@@ -6,5 +6,5 @@ Version: 1.0.0
 Name: Extism
 Description: The framework for building with WebAssembly (wasm).
 Libs: -L${libdir} -lextism
-Libs.private: -lm
+Libs.private: -lm -lpthread
 Cflags: -I${includedir}

manifest/Cargo.toml

@@ -10,7 +10,7 @@ version.workspace = true
 
 [dependencies]
 serde = { version = "1", features = ["derive"] }
-base64 = "~0.21"
+base64 = "~0.22"
 schemars = { version = "0.8", optional = true }
 serde_json = "1"
 

manifest/schema.json

@@ -1,9 +1,11 @@
 {
   "$schema": "http://json-schema.org/draft-07/schema#",
   "title": "Manifest",
+  "description": "The `Manifest` type is used to configure the runtime and specify how to load modules.",
   "type": "object",
   "properties": {
     "allowed_hosts": {
+      "description": "Specifies which hosts may be accessed via HTTP, if this is empty then no hosts may be accessed. Wildcards may be used.",
       "default": null,
       "type": [
         "array",
@@ -14,6 +16,7 @@
       }
     },
     "allowed_paths": {
+      "description": "Specifies which paths should be made available on disk when using WASI. This is a mapping from the path on disk to the path it should be available inside the plugin. For example, `\".\": \"/tmp\"` would mount the current directory as `/tmp` inside the module",
       "default": null,
       "type": [
         "object",
@@ -24,6 +27,7 @@
       }
     },
     "config": {
+      "description": "Config values are made accessible using the PDK `extism_config_get` function",
       "default": {},
       "type": "object",
       "additionalProperties": {
@@ -31,8 +35,11 @@
       }
     },
     "memory": {
+      "description": "Memory options",
       "default": {
-        "max_pages": null
+        "max_http_response_bytes": null,
+        "max_pages": null,
+        "max_var_bytes": null
       },
       "allOf": [
         {
@@ -41,6 +48,8 @@
       ]
     },
     "timeout_ms": {
+      "description": "The plugin timeout in milliseconds",
+      "default": null,
       "type": [
         "integer",
         "null"
@@ -49,6 +58,7 @@
       "minimum": 0.0
     },
     "wasm": {
+      "description": "WebAssembly modules, the `main` module should be named `main` or listed last",
       "default": [],
       "type": "array",
       "items": {
@@ -56,35 +66,63 @@
       }
     }
   },
+  "additionalProperties": false,
   "definitions": {
     "MemoryOptions": {
+      "description": "Configure memory settings",
       "type": "object",
       "properties": {
+        "max_http_response_bytes": {
+          "description": "The maximum number of bytes allowed in an HTTP response",
+          "default": null,
+          "type": [
+            "integer",
+            "null"
+          ],
+          "format": "uint64",
+          "minimum": 0.0
+        },
         "max_pages": {
+          "description": "The max number of WebAssembly pages that should be allocated",
           "type": [
             "integer",
             "null"
           ],
           "format": "uint32",
           "minimum": 0.0
+        },
+        "max_var_bytes": {
+          "description": "The maximum number of bytes allowed to be used by plugin vars. Setting this to 0 will disable Extism vars. The default value is 1mb.",
+          "default": 1048576,
+          "type": [
+            "integer",
+            "null"
+          ],
+          "format": "uint64",
+          "minimum": 0.0
         }
-      }
+      },
+      "additionalProperties": false
     },
     "Wasm": {
+      "description": "The `Wasm` type specifies how to access a WebAssembly module",
       "anyOf": [
         {
+          "description": "From disk",
           "type": "object",
           "required": [
             "path"
           ],
           "properties": {
             "hash": {
+              "description": "Module hash, if the data loaded from disk or via HTTP doesn't match an error will be raised",
               "type": [
                 "string",
                 "null"
               ]
             },
             "name": {
+              "description": "Module name, this is used by Extism to determine which is the `main` module",
               "type": [
                 "string",
                 "null"
@@ -93,45 +131,72 @@
             "path": {
               "type": "string"
             }
-          }
+          },
+          "additionalProperties": false
         },
         {
+          "description": "From memory",
           "type": "object",
           "required": [
             "data"
           ],
           "properties": {
             "data": {
-              "type": "string",
-              "format": "string"
+              "type": [
+                "string",
+                "object"
+              ],
+              "required": [
+                "len",
+                "ptr"
+              ],
+              "properties": {
+                "len": {
+                  "type": "integer",
+                  "format": "uint64",
+                  "minimum": 0.0
+                },
+                "ptr": {
+                  "type": "integer",
+                  "format": "uint64",
+                  "minimum": 0.0
+                }
+              },
+              "additionalProperties": false
             },
             "hash": {
+              "description": "Module hash, if the data loaded from disk or via HTTP doesn't match an error will be raised",
               "type": [
                 "string",
                 "null"
               ]
             },
             "name": {
+              "description": "Module name, this is used by Extism to determine which is the `main` module",
               "type": [
                 "string",
                 "null"
               ]
             }
-          }
+          },
+          "additionalProperties": false
         },
         {
+          "description": "Via HTTP",
           "type": "object",
           "required": [
             "url"
           ],
           "properties": {
             "hash": {
+              "description": "Module hash, if the data loaded from disk or via HTTP doesn't match an error will be raised",
               "type": [
                 "string",
                 "null"
               ]
             },
             "headers": {
+              "description": "Request headers",
               "default": {},
               "type": "object",
               "additionalProperties": {
@@ -139,21 +204,25 @@
               }
             },
             "method": {
+              "description": "Request method",
               "type": [
                 "string",
                 "null"
               ]
             },
             "name": {
+              "description": "Module name, this is used by Extism to determine which is the `main` module",
               "type": [
                 "string",
                 "null"
               ]
             },
             "url": {
+              "description": "The request URL",
               "type": "string"
             }
-          }
+          },
+          "additionalProperties": false
         }
       ]
     }

manifest/src/lib.rs

@@ -5,17 +5,54 @@ use std::path::{Path, PathBuf};
 pub type ManifestMemory = MemoryOptions;
 
 /// Configure memory settings
-#[derive(Default, Clone, serde::Serialize, serde::Deserialize)]
+#[derive(Default, Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
 #[cfg_attr(feature = "json_schema", derive(schemars::JsonSchema))]
 #[serde(deny_unknown_fields)]
 pub struct MemoryOptions {
     /// The max number of WebAssembly pages that should be allocated
-    #[serde(alias = "max")]
     pub max_pages: Option<u32>,
+
+    /// The maximum number of bytes allowed in an HTTP response
+    #[serde(default)]
+    pub max_http_response_bytes: Option<u64>,
+
+    /// The maximum number of bytes allowed to be used by plugin vars. Setting this to 0
+    /// will disable Extism vars. The default value is 1mb.
+    #[serde(default = "default_var_bytes")]
+    pub max_var_bytes: Option<u64>,
+}
+
+impl MemoryOptions {
+    /// Create an empty `MemoryOptions` value
+    pub fn new() -> Self {
+        Default::default()
+    }
+
+    /// Set max pages
+    pub fn with_max_pages(mut self, pages: u32) -> Self {
+        self.max_pages = Some(pages);
+        self
+    }
+
+    /// Set max HTTP response size
+    pub fn with_max_http_response_bytes(mut self, bytes: u64) -> Self {
+        self.max_http_response_bytes = Some(bytes);
+        self
+    }
+
+    /// Set max size of Extism vars
+    pub fn with_max_var_bytes(mut self, bytes: u64) -> Self {
+        self.max_var_bytes = Some(bytes);
+        self
+    }
+}
+
+fn default_var_bytes() -> Option<u64> {
+    Some(1024 * 1024)
 }
 
 /// Generic HTTP request structure
-#[derive(Clone, serde::Serialize, serde::Deserialize)]
+#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
 #[cfg_attr(feature = "json_schema", derive(schemars::JsonSchema))]
 #[serde(deny_unknown_fields)]
 pub struct HttpRequest {
@@ -24,7 +61,6 @@ pub struct HttpRequest {
 
     /// Request headers
     #[serde(default)]
-    #[serde(alias = "header")]
     pub headers: std::collections::BTreeMap<String, String>,
 
     /// Request method
@@ -55,7 +91,7 @@ impl HttpRequest {
 }
 
 /// Provides additional metadata about a Webassembly module
-#[derive(Default, Clone, serde::Serialize, serde::Deserialize)]
+#[derive(Default, Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
 #[cfg_attr(feature = "json_schema", derive(schemars::JsonSchema))]
 #[serde(deny_unknown_fields)]
 pub struct WasmMetadata {
@@ -97,7 +133,7 @@ impl From<Vec<u8>> for Wasm {
 pub type ManifestWasm = Wasm;
 
 /// The `Wasm` type specifies how to access a WebAssembly module
-#[derive(Clone, serde::Serialize, serde::Deserialize)]
+#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
 #[cfg_attr(feature = "json_schema", derive(schemars::JsonSchema))]
 #[serde(untagged)]
 #[serde(deny_unknown_fields)]
@@ -111,8 +147,8 @@ pub enum Wasm {
 
     /// From memory
     Data {
-        #[serde(with = "base64")]
-        #[cfg_attr(feature = "json_schema", schemars(schema_with = "base64_schema"))]
+        #[serde(with = "wasmdata")]
+        #[cfg_attr(feature = "json_schema", schemars(schema_with = "wasmdata_schema"))]
         data: Vec<u8>,
         #[serde(flatten)]
         meta: WasmMetadata,
@@ -181,24 +217,51 @@ impl Wasm {
             Wasm::Url { req: _, meta } => meta,
         }
     }
+
+    /// Update Wasm module name
+    pub fn with_name(mut self, name: impl Into<String>) -> Self {
+        self.meta_mut().name = Some(name.into());
+        self
+    }
+
+    /// Update Wasm module hash
+    pub fn with_hash(mut self, hash: impl Into<String>) -> Self {
+        self.meta_mut().hash = Some(hash.into());
+        self
+    }
+}
+
+#[derive(Default, Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
+#[cfg_attr(feature = "json_schema", derive(schemars::JsonSchema))]
+#[serde(deny_unknown_fields)]
+struct DataPtrLength {
+    ptr: u64,
+    len: u64,
 }
 
 #[cfg(feature = "json_schema")]
-fn base64_schema(gen: &mut schemars::gen::SchemaGenerator) -> schemars::schema::Schema {
+fn wasmdata_schema(gen: &mut schemars::gen::SchemaGenerator) -> schemars::schema::Schema {
     use schemars::{schema::SchemaObject, JsonSchema};
     let mut schema: SchemaObject = <String>::json_schema(gen).into();
-    schema.format = Some("string".to_owned());
+    let objschema: SchemaObject = <DataPtrLength>::json_schema(gen).into();
+    let types = schemars::schema::SingleOrVec::<schemars::schema::InstanceType>::Vec(vec![
+        schemars::schema::InstanceType::String,
+        schemars::schema::InstanceType::Object,
+    ]);
+    schema.instance_type = Some(types);
+    schema.object = objschema.object;
     schema.into()
 }
 
 /// The `Manifest` type is used to configure the runtime and specify how to load modules.
-#[derive(Default, Clone, serde::Serialize, serde::Deserialize)]
+#[derive(Default, Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
 #[cfg_attr(feature = "json_schema", derive(schemars::JsonSchema))]
 #[serde(deny_unknown_fields)]
 pub struct Manifest {
     /// WebAssembly modules, the `main` module should be named `main` or listed last
     #[serde(default)]
     pub wasm: Vec<Wasm>,
+
     /// Memory options
     #[serde(default)]
     pub memory: MemoryOptions,
@@ -206,8 +269,8 @@ pub struct Manifest {
     /// Config values are made accessible using the PDK `extism_config_get` function
     #[serde(default)]
     pub config: BTreeMap<String, String>,
-    #[serde(default)]
 
+    #[serde(default)]
     /// Specifies which hosts may be accessed via HTTP, if this is empty then
     /// no hosts may be accessed. Wildcards may be used.
     pub allowed_hosts: Option<Vec<String>>,
@@ -216,27 +279,27 @@ pub struct Manifest {
     /// the path on disk to the path it should be available inside the plugin.
     /// For example, `".": "/tmp"` would mount the current directory as `/tmp` inside the module
     #[serde(default)]
-    pub allowed_paths: Option<BTreeMap<PathBuf, PathBuf>>,
+    pub allowed_paths: Option<BTreeMap<String, PathBuf>>,
 
-    /// The plugin timeout, by default this is set to 30s
-    #[serde(default = "default_timeout")]
+    /// The plugin timeout in milliseconds
+    #[serde(default)]
     pub timeout_ms: Option<u64>,
 }
 
-fn default_timeout() -> Option<u64> {
-    Some(30000)
-}
-
 impl Manifest {
     /// Create a new manifest
     pub fn new(wasm: impl IntoIterator<Item = impl Into<Wasm>>) -> Manifest {
         Manifest {
             wasm: wasm.into_iter().map(|x| x.into()).collect(),
-            timeout_ms: default_timeout(),
             ..Default::default()
         }
     }
 
+    pub fn with_wasm(mut self, wasm: impl Into<Wasm>) -> Self {
+        self.wasm.push(wasm.into());
+        self
+    }
+
     /// Disallow HTTP requests to all hosts
     pub fn disallow_all_hosts(mut self) -> Self {
         self.allowed_hosts = Some(vec![]);
@@ -274,8 +337,7 @@ impl Manifest {
     }
 
     /// Add a path to `allowed_paths`
-    pub fn with_allowed_path(mut self, src: impl AsRef<Path>, dest: impl AsRef<Path>) -> Self {
-        let src = src.as_ref().to_path_buf();
+    pub fn with_allowed_path(mut self, src: String, dest: impl AsRef<Path>) -> Self {
         let dest = dest.as_ref().to_path_buf();
         match &mut self.allowed_paths {
             Some(p) => {
@@ -292,7 +354,7 @@ impl Manifest {
     }
 
     /// Set `allowed_paths`
-    pub fn with_allowed_paths(mut self, paths: impl Iterator<Item = (PathBuf, PathBuf)>) -> Self {
+    pub fn with_allowed_paths(mut self, paths: impl Iterator<Item = (String, PathBuf)>) -> Self {
         self.allowed_paths = Some(paths.collect());
         self
     }
@@ -323,32 +385,46 @@ impl Manifest {
     }
 }
 
-mod base64 {
+mod wasmdata {
+    use crate::DataPtrLength;
     use base64::{engine::general_purpose, Engine as _};
     use serde::{Deserialize, Serialize};
     use serde::{Deserializer, Serializer};
+    use std::slice;
 
     pub fn serialize<S: Serializer>(v: &Vec<u8>, s: S) -> Result<S::Ok, S::Error> {
-        let base64 = general_purpose::STANDARD.encode(v);
+        let base64 = general_purpose::STANDARD.encode(v.as_slice());
         String::serialize(&base64, s)
     }
 
     pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Vec<u8>, D::Error> {
-        let base64 = String::deserialize(d)?;
-        general_purpose::STANDARD
-            .decode(base64.as_bytes())
-            .map_err(serde::de::Error::custom)
+        #[derive(Deserialize)]
+        #[serde(untagged)]
+        enum WasmDataTypes {
+            String(String),
+            DataPtrLength(DataPtrLength),
+        }
+        Ok(match WasmDataTypes::deserialize(d)? {
+            WasmDataTypes::String(string) => general_purpose::STANDARD
+                .decode(string.as_bytes())
+                .map_err(serde::de::Error::custom)?,
+            WasmDataTypes::DataPtrLength(ptrlen) => {
+                let slice =
+                    unsafe { slice::from_raw_parts(ptrlen.ptr as *const u8, ptrlen.len as usize) };
+                slice.to_vec()
+            }
+        })
     }
 }
 
-impl<'a> From<Manifest> for std::borrow::Cow<'a, [u8]> {
+impl From<Manifest> for std::borrow::Cow<'_, [u8]> {
     fn from(m: Manifest) -> Self {
         let s = serde_json::to_vec(&m).unwrap();
         std::borrow::Cow::Owned(s)
     }
 }
 
-impl<'a> From<&Manifest> for std::borrow::Cow<'a, [u8]> {
+impl From<&Manifest> for std::borrow::Cow<'_, [u8]> {
     fn from(m: &Manifest) -> Self {
         let s = serde_json::to_vec(&m).unwrap();
         std::borrow::Cow::Owned(s)

nuget/.gitignore

@@ -0,0 +1,484 @@
+## Ignore Visual Studio temporary files, build results, and
+## files generated by popular Visual Studio add-ons.
+##
+## Get latest from `dotnet new gitignore`
+
+# dotenv files
+.env
+
+# User-specific files
+*.rsuser
+*.suo
+*.user
+*.userosscache
+*.sln.docstates
+
+# User-specific files (MonoDevelop/Xamarin Studio)
+*.userprefs
+
+# Mono auto generated files
+mono_crash.*
+
+# Build results
+[Dd]ebug/
+[Dd]ebugPublic/
+[Rr]elease/
+[Rr]eleases/
+x64/
+x86/
+[Ww][Ii][Nn]32/
+[Aa][Rr][Mm]/
+[Aa][Rr][Mm]64/
+bld/
+[Bb]in/
+[Oo]bj/
+[Ll]og/
+[Ll]ogs/
+
+# Visual Studio 2015/2017 cache/options directory
+.vs/
+# Uncomment if you have tasks that create the project's static files in wwwroot
+#wwwroot/
+
+# Visual Studio 2017 auto generated files
+Generated\ Files/
+
+# MSTest test Results
+[Tt]est[Rr]esult*/
+[Bb]uild[Ll]og.*
+
+# NUnit
+*.VisualState.xml
+TestResult.xml
+nunit-*.xml
+
+# Build Results of an ATL Project
+[Dd]ebugPS/
+[Rr]eleasePS/
+dlldata.c
+
+# Benchmark Results
+BenchmarkDotNet.Artifacts/
+
+# .NET
+project.lock.json
+project.fragment.lock.json
+artifacts/
+
+# Tye
+.tye/
+
+# ASP.NET Scaffolding
+ScaffoldingReadMe.txt
+
+# StyleCop
+StyleCopReport.xml
+
+# Files built by Visual Studio
+*_i.c
+*_p.c
+*_h.h
+*.ilk
+*.meta
+*.obj
+*.iobj
+*.pch
+*.pdb
+*.ipdb
+*.pgc
+*.pgd
+*.rsp
+*.sbr
+*.tlb
+*.tli
+*.tlh
+*.tmp
+*.tmp_proj
+*_wpftmp.csproj
+*.log
+*.tlog
+*.vspscc
+*.vssscc
+.builds
+*.pidb
+*.svclog
+*.scc
+
+# Chutzpah Test files
+_Chutzpah*
+
+# Visual C++ cache files
+ipch/
+*.aps
+*.ncb
+*.opendb
+*.opensdf
+*.sdf
+*.cachefile
+*.VC.db
+*.VC.VC.opendb
+
+# Visual Studio profiler
+*.psess
+*.vsp
+*.vspx
+*.sap
+
+# Visual Studio Trace Files
+*.e2e
+
+# TFS 2012 Local Workspace
+$tf/
+
+# Guidance Automation Toolkit
+*.gpState
+
+# ReSharper is a .NET coding add-in
+_ReSharper*/
+*.[Rr]e[Ss]harper
+*.DotSettings.user
+
+# TeamCity is a build add-in
+_TeamCity*
+
+# DotCover is a Code Coverage Tool
+*.dotCover
+
+# AxoCover is a Code Coverage Tool
+.axoCover/*
+!.axoCover/settings.json
+
+# Coverlet is a free, cross platform Code Coverage Tool
+coverage*.json
+coverage*.xml
+coverage*.info
+
+# Visual Studio code coverage results
+*.coverage
+*.coveragexml
+
+# NCrunch
+_NCrunch_*
+.*crunch*.local.xml
+nCrunchTemp_*
+
+# MightyMoose
+*.mm.*
+AutoTest.Net/
+
+# Web workbench (sass)
+.sass-cache/
+
+# Installshield output folder
+[Ee]xpress/
+
+# DocProject is a documentation generator add-in
+DocProject/buildhelp/
+DocProject/Help/*.HxT
+DocProject/Help/*.HxC
+DocProject/Help/*.hhc
+DocProject/Help/*.hhk
+DocProject/Help/*.hhp
+DocProject/Help/Html2
+DocProject/Help/html
+
+# Click-Once directory
+publish/
+
+# Publish Web Output
+*.[Pp]ublish.xml
+*.azurePubxml
+# Note: Comment the next line if you want to checkin your web deploy settings,
+# but database connection strings (with potential passwords) will be unencrypted
+*.pubxml
+*.publishproj
+
+# Microsoft Azure Web App publish settings. Comment the next line if you want to
+# checkin your Azure Web App publish settings, but sensitive information contained
+# in these scripts will be unencrypted
+PublishScripts/
+
+# NuGet Packages
+*.nupkg
+# NuGet Symbol Packages
+*.snupkg
+# The packages folder can be ignored because of Package Restore
+**/[Pp]ackages/*
+# except build/, which is used as an MSBuild target.
+!**/[Pp]ackages/build/
+# Uncomment if necessary however generally it will be regenerated when needed
+#!**/[Pp]ackages/repositories.config
+# NuGet v3's project.json files produces more ignorable files
+*.nuget.props
+*.nuget.targets
+
+# Microsoft Azure Build Output
+csx/
+*.build.csdef
+
+# Microsoft Azure Emulator
+ecf/
+rcf/
+
+# Windows Store app package directories and files
+AppPackages/
+BundleArtifacts/
+Package.StoreAssociation.xml
+_pkginfo.txt
+*.appx
+*.appxbundle
+*.appxupload
+
+# Visual Studio cache files
+# files ending in .cache can be ignored
+*.[Cc]ache
+# but keep track of directories ending in .cache
+!?*.[Cc]ache/
+
+# Others
+ClientBin/
+~$*
+*~
+*.dbmdl
+*.dbproj.schemaview
+*.jfm
+*.pfx
+*.publishsettings
+orleans.codegen.cs
+
+# Including strong name files can present a security risk
+# (https://github.com/github/gitignore/pull/2483#issue-259490424)
+#*.snk
+
+# Since there are multiple workflows, uncomment next line to ignore bower_components
+# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
+#bower_components/
+
+# RIA/Silverlight projects
+Generated_Code/
+
+# Backup & report files from converting an old project file
+# to a newer Visual Studio version. Backup files are not needed,
+# because we have git ;-)
+_UpgradeReport_Files/
+Backup*/
+UpgradeLog*.XML
+UpgradeLog*.htm
+ServiceFabricBackup/
+*.rptproj.bak
+
+# SQL Server files
+*.mdf
+*.ldf
+*.ndf
+
+# Business Intelligence projects
+*.rdl.data
+*.bim.layout
+*.bim_*.settings
+*.rptproj.rsuser
+*- [Bb]ackup.rdl
+*- [Bb]ackup ([0-9]).rdl
+*- [Bb]ackup ([0-9][0-9]).rdl
+
+# Microsoft Fakes
+FakesAssemblies/
+
+# GhostDoc plugin setting file
+*.GhostDoc.xml
+
+# Node.js Tools for Visual Studio
+.ntvs_analysis.dat
+node_modules/
+
+# Visual Studio 6 build log
+*.plg
+
+# Visual Studio 6 workspace options file
+*.opt
+
+# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
+*.vbw
+
+# Visual Studio 6 auto-generated project file (contains which files were open etc.)
+*.vbp
+
+# Visual Studio 6 workspace and project file (working project files containing files to include in project)
+*.dsw
+*.dsp
+
+# Visual Studio 6 technical files
+*.ncb
+*.aps
+
+# Visual Studio LightSwitch build output
+**/*.HTMLClient/GeneratedArtifacts
+**/*.DesktopClient/GeneratedArtifacts
+**/*.DesktopClient/ModelManifest.xml
+**/*.Server/GeneratedArtifacts
+**/*.Server/ModelManifest.xml
+_Pvt_Extensions
+
+# Paket dependency manager
+.paket/paket.exe
+paket-files/
+
+# FAKE - F# Make
+.fake/
+
+# CodeRush personal settings
+.cr/personal
+
+# Python Tools for Visual Studio (PTVS)
+__pycache__/
+*.pyc
+
+# Cake - Uncomment if you are using it
+# tools/**
+# !tools/packages.config
+
+# Tabs Studio
+*.tss
+
+# Telerik's JustMock configuration file
+*.jmconfig
+
+# BizTalk build output
+*.btp.cs
+*.btm.cs
+*.odx.cs
+*.xsd.cs
+
+# OpenCover UI analysis results
+OpenCover/
+
+# Azure Stream Analytics local run output
+ASALocalRun/
+
+# MSBuild Binary and Structured Log
+*.binlog
+
+# NVidia Nsight GPU debugger configuration file
+*.nvuser
+
+# MFractors (Xamarin productivity tool) working folder
+.mfractor/
+
+# Local History for Visual Studio
+.localhistory/
+
+# Visual Studio History (VSHistory) files
+.vshistory/
+
+# BeatPulse healthcheck temp database
+healthchecksdb
+
+# Backup folder for Package Reference Convert tool in Visual Studio 2017
+MigrationBackup/
+
+# Ionide (cross platform F# VS Code tools) working folder
+.ionide/
+
+# Fody - auto-generated XML schema
+FodyWeavers.xsd
+
+# VS Code files for those working on multiple tools
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+*.code-workspace
+
+# Local History for Visual Studio Code
+.history/
+
+# Windows Installer files from build outputs
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# JetBrains Rider
+*.sln.iml
+.idea
+
+##
+## Visual studio for Mac
+##
+
+
+# globs
+Makefile.in
+*.userprefs
+*.usertasks
+config.make
+config.status
+aclocal.m4
+install-sh
+autom4te.cache/
+*.tar.gz
+tarballs/
+test-results/
+
+# Mac bundle stuff
+*.dmg
+*.app
+
+# content below from: https://github.com/github/gitignore/blob/master/Global/macOS.gitignore
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+# content below from: https://github.com/github/gitignore/blob/master/Global/Windows.gitignore
+# Windows thumbnail cache files
+Thumbs.db
+ehthumbs.db
+ehthumbs_vista.db
+
+# Dump file
+*.stackdump
+
+# Folder config file
+[Dd]esktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk
+
+# Vim temporary swap files
+*.swp

nuget/Directory.Build.props

@@ -0,0 +1,32 @@
+<Project>
+
+	<PropertyGroup>
+		<Authors>Extism Contributors</Authors>
+		<PackageTags>extism, wasm, plugin</PackageTags>
+		<PackageLicenseExpression>BSD-3-Clause</PackageLicenseExpression>
+		<PackageReadmeFile>README.md</PackageReadmeFile>
+		<PackageProjectUrl>https://extism.org</PackageProjectUrl>
+		<RepositoryUrl>https://github.com/your/repository.git</RepositoryUrl>
+
+		<TargetFramework>netstandard2.1</TargetFramework>
+		<Deterministic>true</Deterministic>
+		<NoBuild>true</NoBuild>
+		<IncludeBuildOutput>false</IncludeBuildOutput>
+
+		<MinVerIgnoreHeight>true</MinVerIgnoreHeight>
+		<MinVerTagPrefix>v</MinVerTagPrefix>
+		<MinVerVerbosity>normal</MinVerVerbosity>
+	</PropertyGroup>
+
+	<ItemGroup>
+		<None Include="README.md" Pack="true" PackagePath="\" />
+	</ItemGroup>
+
+	<ItemGroup>
+		<PackageReference Include="MinVer" Version="4.3.0">
+			<PrivateAssets>all</PrivateAssets>
+			<IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
+		</PackageReference>
+	</ItemGroup>
+
+</Project>

nuget/Extism.runtime.all.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <NuspecFile>Extism.runtime.all.nuspec</NuspecFile>
+  </PropertyGroup>
+  
+  <Target Name="Replace" BeforeTargets="GenerateNuspec">
+    <WriteLinesToFile
+      File="$(NuspecFile)"
+      Lines="$([System.IO.File]::ReadAllText($(NuspecFile)).Replace('$Version','$(MinVerVersion)'))"
+      Overwrite="true"/>
+  </Target>
+
+</Project>

nuget/Extism.runtime.all.nuspec

@@ -0,0 +1,27 @@
+<?xml version="1.0"?>
+<package>
+    <metadata>
+        <id>Extism.runtime.all</id>
+        <version>$Version</version>
+        <authors>Extism Contributors</authors>
+        <owners>Extism Contributors</owners>
+        <description>Internal implementation package for Extism</description>
+        <tags>extism wasm plugin</tags>
+        <license type="expression">BSD-3-Clause</license>
+        <projectUrl>https://github.com/extism/extism</projectUrl>
+        <requireLicenseAcceptance>false</requireLicenseAcceptance>
+        <copyright>Copyright © 2023 Extism Contributors</copyright>
+        <readme>README.md</readme>
+        <dependencies>
+            <dependency id="Extism.runtime.linux-arm64" version="$Version" />
+            <dependency id="Extism.runtime.linux-musl-arm64" version="$Version" />
+            <dependency id="Extism.runtime.linux-x64" version="$Version" />
+            <dependency id="Extism.runtime.osx-arm64" version="$Version" />
+            <dependency id="Extism.runtime.osx-x64" version="$Version" />
+            <dependency id="Extism.runtime.win-x64" version="$Version" />
+        </dependencies>
+    </metadata>
+    <files>
+        <file src="README.md" target="" />
+    </files>
+</package>

nuget/Extism.runtime.linux-arm64.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <PackageId>Extism.runtime.linux-arm64</PackageId>
+    <Description>Internal implementation package for Extism to work on Linux ARM64</Description>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Content Include="runtimes\linux-arm64\native\libextism.so"
+      CopyToOutputDirectory="Always"
+      Pack="true"
+      PackagePath="runtimes\linux-arm64\native\libextism.so" />
+  </ItemGroup>
+
+</Project>

nuget/Extism.runtime.linux-musl-arm64.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <PackageId>Extism.runtime.linux-musl-arm64</PackageId>
+    <Description>Internal implementation package for Extism to work on Linux Musl ARM64</Description>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Content Include="runtimes\linux-musl-arm64\native\libextism.so"
+      CopyToOutputDirectory="Always"
+      Pack="true"
+      PackagePath="runtimes\linux-musl-arm64\native\libextism.so" />
+  </ItemGroup>
+
+</Project>

nuget/Extism.runtime.linux-x64.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <PackageId>Extism.runtime.linux-x64</PackageId>
+    <Description>Internal implementation package for Extism to work on Linux x64</Description>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Content Include="runtimes\linux-x64\native\libextism.so"
+      CopyToOutputDirectory="Always"
+      Pack="true"
+      PackagePath="runtimes\linux-x64\native\libextism.so" />
+  </ItemGroup>
+
+</Project>

nuget/Extism.runtime.osx-arm64.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <PackageId>Extism.runtime.osx-arm64</PackageId>
+    <Description>Internal implementation package for Extism to work on macOS ARM64</Description>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Content Include="runtimes\osx-arm64\native\libextism.dylib"
+      CopyToOutputDirectory="Always"
+      Pack="true"
+      PackagePath="runtimes\osx-arm64\native\libextism.dylib" />
+  </ItemGroup>
+
+</Project>

nuget/Extism.runtime.osx-x64.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <PackageId>Extism.runtime.osx-x64</PackageId>
+    <Description>Internal implementation package for Extism to work on macOS x64</Description>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Content Include="runtimes\osx-x64\native\libextism.dylib"
+      CopyToOutputDirectory="Always"
+      Pack="true"
+      PackagePath="runtimes\osx-x64\native\libextism.dylib" />
+  </ItemGroup>
+
+</Project>

nuget/Extism.runtime.win-x64.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <PackageId>Extism.runtime.win-x64</PackageId>
+    <Description>Internal implementation package for Extism to work on Windows x64</Description>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Content Include="runtimes\win-x64\native\extism.dll"
+      CopyToOutputDirectory="Always"
+      Pack="true"
+      PackagePath="runtimes\win-x64\native\extism.dll" />
+  </ItemGroup>
+
+</Project>

nuget/README.md

@@ -0,0 +1,2 @@
+# Extism
+The cross-language framework for building with WebAssembly (wasm). For more information see https://extism.org

runtime/Cargo.toml

@@ -9,34 +9,56 @@ repository.workspace = true
 version.workspace = true
 
 [dependencies]
-wasmtime = ">= 14.0.0, < 16.0.0"
-wasmtime-wasi = ">= 14.0.0, < 16.0.0"
+wasmtime = { version="37", default-features = false, features = [
+  'cache',
+  'gc',
+  'gc-drc',
+  'cranelift',
+  'coredump',
+  'wat',
+  'parallel-compilation',
+  'pooling-allocator',
+  'demangle',
+] }
+wasi-common = "37"
+wiggle = "37"
 anyhow = "1"
-serde = {version = "1", features = ["derive"]}
+serde = { version = "1", features = ["derive"] }
 serde_json = "1"
-toml = "0.8"
+toml = "0.9"
 sha2 = "0.10"
 tracing = "0.1"
-tracing-subscriber = {version = "0.3", features = ["std", "env-filter", "fmt"]}
+tracing-subscriber = { version = "0.3.18", features = [
+  "std",
+  "env-filter",
+  "fmt",
+] }
 url = "2"
 glob = "0.3"
-ureq = {version = "2.5", optional=true}
+ureq = { version = "3.0", optional = true }
 extism-manifest = { workspace = true }
-extism-convert = { workspace = true }
+extism-convert = { workspace = true, features = ["extism-path"] }
 uuid = { version = "1", features = ["v4"] }
 libc = "0.2"
 
 [features]
-default = ["http", "register-http", "register-filesystem"]
-register-http = ["ureq"] # enables wasm to be downloaded using http
-register-filesystem = [] # enables wasm to be loaded from disk
-http = ["ureq"]          # enables extism_http_request
+default = ["http", "register-http", "register-filesystem", "wasmtime-default-features"]
+register-http = ["ureq"]           # enables wasm to be downloaded using http
+register-filesystem = []           # enables wasm to be loaded from disk
+http = ["ureq"]                    # enables extism_http_request
+wasmtime-exceptions = []           # enables exception-handling proposal in wasmtime (requires wasmtime gc feature)
+wasmtime-default-features = [
+  'wasmtime/default',
+]
+
 
 [build-dependencies]
-cbindgen = "0.26"
+cbindgen = { version = "0.29", default-features = false }
 
 [dev-dependencies]
-criterion = "0.5.1"
+criterion = "0.7.0"
+quickcheck = "1"
+rand = "0.9.0"
 
 [[bench]]
 name = "bench"

runtime/README.md

@@ -12,7 +12,7 @@ To use the `extism` crate, you can add it to your Cargo file:
 
 ```toml
 [dependencies]
-extism = "^1.0.0-rc3"
+extism = "1.4.1"
 ```
 
 ## Environment variables
@@ -24,6 +24,7 @@ There are a few environment variables that can be used for debugging purposes:
 - `EXTISM_COREDUMP=extism.core`: write [coredump](https://github.com/WebAssembly/tool-conventions/blob/main/Coredump.md) to a file when a WebAssembly function traps
 - `EXTISM_DEBUG=1`: generate debug information
 - `EXTISM_PROFILE=perf|jitdump|vtune`: enable Wasmtime profiling
+- `EXTISM_CACHE_CONFIG=path/to/config.toml`: enable Wasmtime cache, details [here](#wasmtime-caching)
 
 > *Note*: The debug and coredump info will only be written if the plug-in has an error.
 
@@ -51,7 +52,7 @@ fn main() {
 }
 ```
 
-> **Note**: See [the Manifest docs](https://docs.rs/extism-manifest/extism_manifest/) as it has a rich schema and a lot of options.
+> **Note**: See [the Manifest docs](https://docs.rs/extism-manifest/latest/extism_manifest/) as it has a rich schema and a lot of options.
 
 ### Calling A Plug-in's Exports
 
@@ -87,7 +88,7 @@ println!("{:?}", res);
 
 ### Plug-in State
 
-Plug-ins may be stateful or stateless. Plug-ins can maintain state b/w calls by the use of variables. Our count vowels plug-in remembers the total number of vowels it's ever counted in the "total" key in the result. You can see this by making subsequent calls to the export:
+Plug-ins may be stateful or stateless. Plug-ins can maintain state between calls by the use of variables. Our count vowels plug-in remembers the total number of vowels it's ever counted in the "total" key in the result. You can see this by making subsequent calls to the export:
 
 ```rust
 let res = plugin.call::<&str, &str>("count_vowels", "Hello, world!").unwrap();
@@ -111,7 +112,8 @@ let mut plugin = Plugin::new(&manifest, [], true);
 let res = plugin.call::<&str, &str>("count_vowels", "Yellow, world!").unwrap();
 println!("{}", res);
 # => {"count": 3, "total": 3, "vowels": "aeiouAEIOU"}
-let mut plugin = Plugin::new(&manifest, [], true).with_config_key("vowels", "aeiouyAEIOUY");
+let manifest = Manifest::new([url]).with_config_key("vowels", "aeiouyAEIOUY");
+let mut plugin = Plugin::new(&manifest, [], true).unwrap();
 let res = plugin.call::<&str, &str>("count_vowels", "Yellow, world!").unwrap();
 println!("{}", res);
 # => {"count": 4, "total": 4, "vowels": "aeiouyAEIOUY"}
@@ -200,7 +202,7 @@ fn main() {
 }
 ```
 
-> *Note*: In order to write host functions you should get familiar with the methods on the [Extism::CurrentPlugin](https://docs.rs/extism/latest/extism/struct.CurrentPlugin.html) and [Extism::CurrentPlugin](https://docs.rs/extism/latest/extism/struct.UserData.html) types.
+> *Note*: In order to write host functions you should get familiar with the methods on the [CurrentPlugin](https://docs.rs/extism/latest/extism/struct.CurrentPlugin.html) and [UserData](https://docs.rs/extism/latest/extism/enum.UserData.html) types.
 
 Now we can invoke the event:
 
@@ -218,5 +220,52 @@ println!("{}", res);
 # => {"count": 3, "total": 6, "vowels": "aeiouAEIOU"}
 ```
 
+### Logging
 
+Plug-ins can't directly print anything to the console. They can however use Extism's built-in logging functionality, for example the [`log!` macro in the rust-pdk](https://docs.rs/extism-pdk/1.4.0/extism_pdk/macro.log.html) or the [`logInfo` function in the haskell-pdk](https://hackage.haskell.org/package/extism-pdk-1.2.0.0/docs/Extism-PDK.html#v:logInfo).
 
+Inside your host application, the rust-sdk emits these as [tracing](https://github.com/tokio-rs/tracing) events. The simplest way to make the logged messages visible is by adding the `tracing_subscriber` dependency to your crate and then initializing a tracing subscriber at the top of your main function:
+
+```rust
+tracing_subscriber::fmt::init();
+```
+
+### Wasmtime Caching
+
+To enable or disable caching for plugin compilation, you need to provide a configuration file that will be used by the [wasmtime crate](https://github.com/bytecodealliance/wasmtime).
+
+For more information and values that can be used for configuring caching, take a look at [the docs](https://docs.wasmtime.dev/cli-cache.html).
+
+> *Note*: As of now extism uses wasmtime [`version = ">= 27.0.0, < 31.0.0"`](https://github.com/extism/extism/blob/v1.11.1/runtime/Cargo.toml#L12), but the `enabled` key requirement [was removed](https://github.com/bytecodealliance/wasmtime/pull/10859) from `wasmtime` and its documentation, this could explain the `failed to parse config file` error you might encounter without it.
+
+An example configuration for caching would be:
+
+```toml
+[cache]
+enabled = true # This value is required
+directory = "/some/path"
+```
+
+You can :
+- [Create a global `wasmtime` configuration file](#using-a-configuration-file) in `$HOME/.config/wasmtime/config.toml`.
+- [Set the `EXTISM_CACHE_CONFIG` environment variable](#using-an-environment-variable)
+- [Set the configuration file path using `PluginBuilder`](#using-pluginbuilder)
+
+#### Using a configuration file
+
+The [wasmtime](https://github.com/bytecodealliance/wasmtime) crate, by default, will look for a configuration file in your systems' default configuration directory (for example on UNIX systems: `$HOME/.config/wasmtime/config.toml`),
+for more [information on this behaviour](`https://docs.rs/wasmtime/31.0.0/wasmtime/struct.Config.html#method.cache_config_load_default`).
+
+#### Using an environment variable
+
+You can set the `EXTISM_CACHE_CONFIG=path/to/config.toml` environment variable to set the path of the configuration file used by [wasmtime](https://github.com/bytecodealliance/wasmtime).
+Setting the variable to an empty string will disable caching (it won't load any configuration file).
+
+> *Note*: If the environment variable is not set, `wasmtime` will still try to read from a configuration file that may exist in your system's default configuration folder (e.g. `$HOME/.config/wasmtime/config.toml`).
+
+The environment variable does not override the path you might have set using `PluginBuilder`. will only be checked for if you did not specify a cache configuration path in `PluginBuilder`.
+
+#### Using PluginBuilder
+
+If you use a [PluginBuilder](https://docs.rs/extism/latest/extism/struct.PluginBuilder.html), you can set the `wasmtime` configuration path using the [with_cache_config](https://docs.rs/extism/latest/extism/struct.PluginBuilder.html#method.with_cache_config) method.
+This will override the `EXTISM_CACHE_CONFIG` environment variable if it's set, so you could have a "global" and per plugin configuration if needed.

runtime/benches/bench.rs

@@ -6,6 +6,7 @@ const COUNT_VOWELS: &[u8] = include_bytes!("../../wasm/code.wasm");
 const REFLECT: &[u8] = include_bytes!("../../wasm/reflect.wasm");
 const ECHO: &[u8] = include_bytes!("../../wasm/echo.wasm");
 const CONSUME: &[u8] = include_bytes!("../../wasm/consume.wasm");
+const ALLOCATIONS: &[u8] = include_bytes!("../../wasm/allocations.wasm");
 
 host_fn!(hello_world (a: String) -> String { Ok(a) });
 
@@ -35,6 +36,46 @@ pub fn create_plugin(c: &mut Criterion) {
     });
 }
 
+pub fn create_compiled(c: &mut Criterion) {
+    let mut g = c.benchmark_group("create");
+    g.noise_threshold(1.0);
+    g.significance_level(0.2);
+    g.bench_function("create_compiled", |b| {
+        b.iter(|| {
+            let plugin = PluginBuilder::new(COUNT_VOWELS).with_wasi(true);
+            let _compiled = CompiledPlugin::new(plugin).unwrap();
+        })
+    });
+}
+
+pub fn create_plugin_compiled(c: &mut Criterion) {
+    let mut g = c.benchmark_group("create");
+    g.noise_threshold(1.0);
+    g.significance_level(0.2);
+    let plugin = PluginBuilder::new(COUNT_VOWELS).with_wasi(true);
+    let compiled = CompiledPlugin::new(plugin).unwrap();
+    g.bench_function("create_plugin_compiled", |b| {
+        b.iter(|| {
+            let _plugin = Plugin::new_from_compiled(&compiled).unwrap();
+        })
+    });
+}
+
+pub fn create_plugin_no_cache(c: &mut Criterion) {
+    let mut g = c.benchmark_group("create");
+    g.noise_threshold(1.0);
+    g.significance_level(0.2);
+    g.bench_function("create_plugin_no_cache", |b| {
+        b.iter(|| {
+            let _plugin = PluginBuilder::new(COUNT_VOWELS)
+                .with_cache_disabled()
+                .with_wasi(true)
+                .build()
+                .unwrap();
+        })
+    });
+}
+
 #[derive(Debug, serde::Deserialize, PartialEq)]
 struct Count {
     count: u32,
@@ -65,17 +106,7 @@ pub fn consume(c: &mut Criterion) {
     g.sample_size(500);
     g.noise_threshold(1.0);
     g.significance_level(0.2);
-    let mut plugin = PluginBuilder::new(CONSUME)
-        .with_wasi(true)
-        .with_function(
-            "host_reflect",
-            [PTR],
-            [PTR],
-            UserData::default(),
-            hello_world,
-        )
-        .build()
-        .unwrap();
+    let mut plugin = PluginBuilder::new(CONSUME).build().unwrap();
 
     for (i, elements) in [
         b"a".repeat(65536),
@@ -103,17 +134,7 @@ pub fn echo(c: &mut Criterion) {
     g.sample_size(500);
     g.noise_threshold(1.0);
     g.significance_level(0.2);
-    let mut plugin = PluginBuilder::new(ECHO)
-        .with_wasi(true)
-        .with_function(
-            "host_reflect",
-            [PTR],
-            [PTR],
-            UserData::default(),
-            hello_world,
-        )
-        .build()
-        .unwrap();
+    let mut plugin = PluginBuilder::new(ECHO).build().unwrap();
 
     for (i, elements) in [
         b"a".repeat(65536),
@@ -138,9 +159,7 @@ pub fn echo(c: &mut Criterion) {
 
 pub fn reflect(c: &mut Criterion) {
     let mut g = c.benchmark_group("reflect");
-    g.sample_size(500);
-    g.noise_threshold(1.0);
-    g.significance_level(0.2);
+
     let mut plugin = PluginBuilder::new(REFLECT)
         .with_wasi(true)
         .with_function(
@@ -152,6 +171,108 @@ pub fn reflect(c: &mut Criterion) {
         )
         .build()
         .unwrap();
+    for (i, elements) in [
+        b"a".repeat(65536),
+        b"a".repeat(65536 * 10),
+        b"a".repeat(65536 * 100),
+        b"a".repeat(65536),
+    ]
+    .iter()
+    .enumerate()
+    {
+        g.throughput(criterion::Throughput::Bytes(elements.len() as u64));
+        g.bench_with_input(
+            format!("{i}: reflect {} bytes", elements.len()),
+            elements,
+            |b, elems| {
+                b.iter(|| {
+                    assert_eq!(elems, plugin.call::<_, &[u8]>("reflect", &elems).unwrap());
+                    // plugin.reset().unwrap();
+                });
+            },
+        );
+    }
+}
+
+pub fn allocations(c: &mut Criterion) {
+    let mut g = c.benchmark_group("allocations");
+
+    let mut plugin = PluginBuilder::new(ALLOCATIONS).build().unwrap();
+    g.bench_function("allocations", |b| {
+        b.iter(|| {
+            plugin.call::<_, ()>("allocations", "").unwrap();
+        })
+    });
+}
+
+// This is an apples-to-apples comparison of a linked wasm "reflect" function to our host "reflect"
+// function.
+pub fn reflect_linked(c: &mut Criterion) {
+    let mut g = c.benchmark_group("reflect");
+    g.sample_size(500);
+    g.noise_threshold(1.0);
+    g.significance_level(0.2);
+    let manifest = Manifest::new([
+        Wasm::Data {
+            data: br#"(module
+    (import "extism:host/env" "length" (func $length (param i64) (result i64)))
+    (import "extism:host/env" "load_u64" (func $load_u64 (param i64) (result i64)))
+    (import "extism:host/env" "load_u8" (func $load_u8 (param i64) (result i32)))
+    (import "extism:host/env" "store_u64" (func $store_u64 (param i64 i64)))
+    (import "extism:host/env" "store_u8" (func $store_u8 (param i64 i32)))
+
+    (func (export "host_reflect") (param $extism_offset i64) (result i64)
+        (local $len i64)
+        (local $offset i64)
+        (local $len64 i64)
+
+        (local.set $offset (i64.const 0))
+        (local.set $len (call $length (local.get $extism_offset)))
+
+        (local.set $len64 (i64.shl (i64.shr_u (local.get $len) (i64.const 16)) (i64.const 16)))
+
+        (loop $to_upper
+            (call $store_u64
+              (i64.add (local.get $extism_offset) (local.get $offset))
+              (call $load_u64 (i64.add (local.get $extism_offset) (local.get $offset)))
+            )
+
+            (local.set $offset (i64.add (local.get $offset) (i64.const 8)))
+            (br_if $to_upper (i64.lt_u (local.get $offset) (local.get $len64)))
+        )
+
+        (if (i64.ne (local.get $len64) (local.get $len)) (then
+          (loop $to_upper
+              (call $store_u8
+                (i64.add (local.get $extism_offset) (local.get $offset))
+                (call $load_u8 (i64.add (local.get $extism_offset) (local.get $offset)))
+              )
+
+              (local.set $offset (i64.add (i64.const 1) (local.get $offset)))
+              (br_if $to_upper (i64.lt_u (local.get $offset) (local.get $len)))
+          )
+        ))
+        local.get $extism_offset
+    )
+)"#
+            .to_vec(),
+            meta: WasmMetadata {
+                name: Some("extism:host/user".to_string()),
+                hash: None,
+            },
+        },
+        Wasm::Data {
+            data: REFLECT.to_vec(),
+            meta: WasmMetadata {
+                name: Some("main".to_string()),
+                hash: None,
+            },
+        },
+    ]);
+    let mut plugin = PluginBuilder::new(manifest)
+        .with_wasi(true)
+        .build()
+        .unwrap();
 
     for (i, elements) in [
         b"a".repeat(65536),
@@ -163,7 +284,7 @@ pub fn reflect(c: &mut Criterion) {
     {
         g.throughput(criterion::Throughput::Bytes(elements.len() as u64));
         g.bench_with_input(
-            format!("reflect {} bytes", 10u32.pow(i as u32) * 65536),
+            format!("reflect_linked {} bytes", 10u32.pow(i as u32) * 65536),
             elements,
             |b, elems| {
                 b.iter(|| {
@@ -176,11 +297,16 @@ pub fn reflect(c: &mut Criterion) {
 
 criterion_group!(
     benches,
+    allocations,
     consume,
     echo,
     reflect,
+    reflect_linked,
     basic,
     create_plugin,
+    create_plugin_compiled,
+    create_plugin_no_cache,
+    create_compiled,
     count_vowels
 );
 criterion_main!(benches);

runtime/build.rs

@@ -7,7 +7,7 @@ fn main() {
 #define EXTISM_SUCCESS 0
 
 /** An alias for I64 to signify an Extism pointer */
-#define PTR I64
+#define EXTISM_PTR ExtismValType_I64
 ";
     if let Ok(x) = cbindgen::Builder::new()
         .with_crate(".")
@@ -16,6 +16,7 @@ fn main() {
         .with_sys_include("stdint.h")
         .with_sys_include("stdbool.h")
         .with_pragma_once(true)
+        .with_cpp_compat(true)
         .with_after_include(fn_macro)
         .rename_item("Size", "ExtismSize")
         .rename_item("ValType", "ExtismValType")
@@ -23,6 +24,7 @@ fn main() {
         .rename_item("CurrentPlugin", "ExtismCurrentPlugin")
         .rename_item("CancelHandle", "ExtismCancelHandle")
         .rename_item("Plugin", "ExtismPlugin")
+        .rename_item("CompiledPlugin", "ExtismCompiledPlugin")
         .rename_item("Function", "ExtismFunction")
         .with_style(cbindgen::Style::Type)
         .generate()

runtime/examples/fs.rs

@@ -0,0 +1,36 @@
+use extism::*;
+fn main() {
+    let url = Wasm::file("../wasm/read_write.wasm");
+    let manifest = Manifest::new([url])
+        // This will fail because we're using a readonly path (specified with the `ro:` prefix)
+        // to overwrite the data file, remove `ro:` from the path on the following line
+        .with_allowed_path("ro:src/tests/data".to_string(), "/data")
+        .with_config_key("path", "/data/data.txt");
+
+    let mut plugin = PluginBuilder::new(manifest)
+        .with_wasi(true)
+        .build()
+        .unwrap();
+
+    println!("trying to read file: ");
+
+    let res = plugin.call::<&str, &str>("try_read", "").unwrap();
+
+    println!("{res:?}");
+
+    println!("-----------------------------------------------------");
+
+    // If the allowed path is readonly then writing back to the file should fail
+    println!("trying to write file: ");
+    let line = format!(
+        "Hello World at {:?}\n",
+        std::time::SystemTime::now()
+            .duration_since(std::time::SystemTime::UNIX_EPOCH)
+            .unwrap()
+    );
+    let res2 = plugin.call::<&str, &str>("try_write", &line).unwrap();
+
+    println!("{res2:?}");
+
+    println!("done!");
+}

runtime/examples/linking.rs

@@ -0,0 +1,35 @@
+use extism::*;
+
+fn main() {
+    let manifest = Manifest::new([
+        // upper.wat provides an export called `host_reflect` that takes a string
+        // and returns the same string uppercased
+        Wasm::File {
+            // See https://github.com/extism/plugins/blob/main/upper.wat
+            path: "../wasm/upper.wasm".into(),
+            meta: WasmMetadata {
+                name: Some("extism:host/user".to_string()),
+                hash: None,
+            },
+        },
+        // reflect expects host_reflect to be imported: https://github.com/extism/plugins/blob/e5578bbbdd87f9936a0a8d36df629768b2eff6bb/reflect/src/lib.rs#L5
+        // Extism will link the export from upper.wat to the import of reflect.rs at runtime so it
+        // can call it
+        Wasm::File {
+            // See https://github.com/extism/plugins/tree/main/reflect
+            path: "../wasm/reflect.wasm".into(),
+            meta: WasmMetadata {
+                name: Some("main".to_string()),
+                hash: None,
+            },
+        },
+    ]);
+    let mut plugin = PluginBuilder::new(manifest).build().unwrap();
+
+    for _ in 0..5 {
+        let res = plugin
+            .call::<&str, &str>("reflect", "Hello, world!")
+            .unwrap();
+        println!("{res}");
+    }
+}

runtime/examples/log_callback.rs

@@ -25,6 +25,6 @@ fn main() {
     println!("Dumping logs");
 
     for line in LOGS.lock().unwrap().iter() {
-        print!("{}", line);
+        print!("{line}");
     }
 }

runtime/examples/readme.rs

@@ -52,6 +52,6 @@ fn main() {
         let res = plugin
             .call::<&str, &str>("count_vowels", "Hello, world!")
             .unwrap();
-        println!("{}", res);
+        println!("{res}");
     }
 }

runtime/extism.h

@@ -10,7 +10,7 @@
 #define EXTISM_SUCCESS 0
 
 /** An alias for I64 to signify an Extism pointer */
-#define PTR I64
+#define EXTISM_PTR ExtismValType_I64
 
 
 /**
@@ -20,31 +20,31 @@ typedef enum {
   /**
    * Signed 32 bit integer.
    */
-  I32,
+  ExtismValType_I32,
   /**
    * Signed 64 bit integer.
    */
-  I64,
+  ExtismValType_I64,
   /**
    * Floating point 32 bit integer.
    */
-  F32,
+  ExtismValType_F32,
   /**
    * Floating point 64 bit integer.
    */
-  F64,
+  ExtismValType_F64,
   /**
    * A 128 bit number.
    */
-  V128,
+  ExtismValType_V128,
   /**
    * A reference to a Wasm function.
    */
-  FuncRef,
+  ExtismValType_FuncRef,
   /**
    * A reference to opaque data in the Wasm instance.
    */
-  ExternRef,
+  ExtismValType_ExternRef,
 } ExtismValType;
 
 /**
@@ -52,6 +52,8 @@ typedef enum {
  */
 typedef struct ExtismCancelHandle ExtismCancelHandle;
 
+typedef struct ExtismCompiledPlugin ExtismCompiledPlugin;
+
 /**
  * CurrentPlugin stores data that is available to the caller in PDK functions, this should
  * only be accessed from inside a host function
@@ -97,13 +99,28 @@ typedef void (*ExtismFunctionType)(ExtismCurrentPlugin *plugin,
                                    ExtismSize n_outputs,
                                    void *data);
 
+/**
+ * Log drain callback
+ */
+typedef void (*ExtismLogDrainFunctionType)(const char *data, ExtismSize size);
 
 
+
+#ifdef __cplusplus
+extern "C" {
+#endif // __cplusplus
+
 /**
  * Get a plugin's ID, the returned bytes are a 16 byte buffer that represent a UUIDv4
  */
 const uint8_t *extism_plugin_id(ExtismPlugin *plugin);
 
+/**
+ * Get the current plugin's associated host context data. Returns null if call was made without
+ * host context.
+ */
+void *extism_current_plugin_host_context(ExtismCurrentPlugin *plugin);
+
 /**
  * Returns a pointer to the memory of the currently running plugin
  * NOTE: this should only be called from host functions.
@@ -139,7 +156,7 @@ void extism_current_plugin_memory_free(ExtismCurrentPlugin *plugin, ExtismMemory
  * - `n_outputs`: number of return types
  * - `func`: the function to call
  * - `user_data`: a pointer that will be passed to the function when it's called
- *    this value should live as long as the function exists
+ *   this value should live as long as the function exists
  * - `free_user_data`: a callback to release the `user_data` value when the resulting
  *   `ExtismFunction` is freed.
  *
@@ -164,6 +181,32 @@ void extism_function_free(ExtismFunction *f);
  */
 void extism_function_set_namespace(ExtismFunction *ptr, const char *namespace_);
 
+/**
+ * Pre-compile an Extism plugin
+ */
+ExtismCompiledPlugin *extism_compiled_plugin_new(const uint8_t *wasm,
+                                                 ExtismSize wasm_size,
+                                                 const ExtismFunction **functions,
+                                                 ExtismSize n_functions,
+                                                 bool with_wasi,
+                                                 char **errmsg);
+
+/**
+ * Pre-compile an Extism plugin and set the number of instructions a plugin is allowed to execute
+ */
+ExtismCompiledPlugin *extism_compiled_plugin_new_with_fuel_limit(const uint8_t *wasm,
+                                                                 ExtismSize wasm_size,
+                                                                 const ExtismFunction **functions,
+                                                                 ExtismSize n_functions,
+                                                                 bool with_wasi,
+                                                                 uint64_t fuel_limit,
+                                                                 char **errmsg);
+
+/**
+ * Free `ExtismCompiledPlugin`
+ */
+void extism_compiled_plugin_free(ExtismCompiledPlugin *plugin);
+
 /**
  * Create a new plugin with host functions, the functions passed to this function no longer need to be manually freed using
  *
@@ -180,13 +223,34 @@ ExtismPlugin *extism_plugin_new(const uint8_t *wasm,
                                 bool with_wasi,
                                 char **errmsg);
 
+/**
+ * Create a new plugin from an `ExtismCompiledPlugin`
+ */
+ExtismPlugin *extism_plugin_new_from_compiled(const ExtismCompiledPlugin *compiled, char **errmsg);
+
+/**
+ * Create a new plugin and set the number of instructions a plugin is allowed to execute
+ */
+ExtismPlugin *extism_plugin_new_with_fuel_limit(const uint8_t *wasm,
+                                                ExtismSize wasm_size,
+                                                const ExtismFunction **functions,
+                                                ExtismSize n_functions,
+                                                bool with_wasi,
+                                                uint64_t fuel_limit,
+                                                char **errmsg);
+
+/**
+ * Enable HTTP response headers in plugins using `extism:host/env::http_request`
+ */
+void extism_plugin_allow_http_response_headers(ExtismPlugin *plugin);
+
 /**
  * Free the error returned by `extism_plugin_new`, errors returned from `extism_plugin_error` don't need to be freed
  */
 void extism_plugin_new_error_free(char *err);
 
 /**
- * Remove a plugin from the registry and free associated memory
+ * Free `ExtismPlugin`
  */
 void extism_plugin_free(ExtismPlugin *plugin);
 
@@ -222,6 +286,20 @@ int32_t extism_plugin_call(ExtismPlugin *plugin,
                            const uint8_t *data,
                            ExtismSize data_len);
 
+/**
+ * Call a function with host context.
+ *
+ * `func_name`: is the function to call
+ * `data`: is the input data
+ * `data_len`: is the length of `data`
+ * `host_context`: a pointer to context data that will be available in host functions
+ */
+int32_t extism_plugin_call_with_host_context(ExtismPlugin *plugin,
+                                             const char *func_name,
+                                             const uint8_t *data,
+                                             ExtismSize data_len,
+                                             void *host_context);
+
 /**
  * Get the error associated with a `Plugin`
  */
@@ -260,9 +338,18 @@ bool extism_log_custom(const char *log_level);
  * Calls the provided callback function for each buffered log line.
  * This is only needed when `extism_log_custom` is used.
  */
-void extism_log_drain(void (*handler)(const char*, uintptr_t));
+void extism_log_drain(ExtismLogDrainFunctionType handler);
+
+/**
+ * Reset the Extism runtime, this will invalidate all allocated memory
+ */
+bool extism_plugin_reset(ExtismPlugin *plugin);
 
 /**
  * Get the Extism version string
  */
 const char *extism_version(void);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif  // __cplusplus

runtime/src/current_plugin.rs

@@ -1,3 +1,5 @@
+use anyhow::Context;
+
 use crate::*;
 
 /// CurrentPlugin stores data that is available to the caller in PDK functions, this should
@@ -12,9 +14,11 @@ pub struct CurrentPlugin {
     pub(crate) linker: *mut wasmtime::Linker<CurrentPlugin>,
     pub(crate) wasi: Option<Wasi>,
     pub(crate) http_status: u16,
+    pub(crate) http_headers: Option<std::collections::BTreeMap<String, String>>,
     pub(crate) available_pages: Option<u32>,
     pub(crate) memory_limiter: Option<MemoryLimiter>,
     pub(crate) id: uuid::Uuid,
+    pub(crate) start_time: std::time::Instant,
 }
 
 unsafe impl Send for CurrentPlugin {}
@@ -52,7 +56,12 @@ impl wasmtime::ResourceLimiter for MemoryLimiter {
         Ok(true)
     }
 
-    fn table_growing(&mut self, _current: u32, desired: u32, maximum: Option<u32>) -> Result<bool> {
+    fn table_growing(
+        &mut self,
+        _current: usize,
+        desired: usize,
+        maximum: Option<usize>,
+    ) -> Result<bool> {
         if let Some(max) = maximum {
             return Ok(desired <= max);
         }
@@ -62,8 +71,16 @@ impl wasmtime::ResourceLimiter for MemoryLimiter {
 }
 
 impl CurrentPlugin {
+    /// Gets `Plugin`'s ID
+    pub fn id(&self) -> uuid::Uuid {
+        self.id
+    }
+
     /// Get a `MemoryHandle` from a memory offset
     pub fn memory_handle(&mut self, offs: u64) -> Option<MemoryHandle> {
+        if offs == 0 {
+            return Some(MemoryHandle::null());
+        }
         let len = self.memory_length(offs).unwrap_or_default();
         if len == 0 {
             trace!(
@@ -84,18 +101,27 @@ impl CurrentPlugin {
     }
 
     /// Access memory bytes as `str`
-    pub fn memory_str(&mut self, handle: MemoryHandle) -> Result<&mut str, Error> {
-        let bytes = self.memory_bytes(handle)?;
+    pub fn memory_str_mut(&mut self, handle: MemoryHandle) -> Result<&mut str, Error> {
+        let bytes = self.memory_bytes_mut(handle)?;
         let s = std::str::from_utf8_mut(bytes)?;
         Ok(s)
     }
 
+    pub fn memory_str(&mut self, handle: MemoryHandle) -> Result<&str, Error> {
+        let bytes = self.memory_bytes(handle)?;
+        let s = std::str::from_utf8(bytes)?;
+        Ok(s)
+    }
+
     /// Allocate a handle large enough for the encoded Rust type and copy it into Extism memory
     pub fn memory_new<'a, T: ToBytes<'a>>(&mut self, t: T) -> Result<MemoryHandle, Error> {
         let data = t.to_bytes()?;
         let data = data.as_ref();
+        if data.is_empty() {
+            return Ok(MemoryHandle::null());
+        }
         let handle = self.memory_alloc(data.len() as u64)?;
-        let bytes = self.memory_bytes(handle)?;
+        let bytes = self.memory_bytes_mut(handle)?;
         bytes.copy_from_slice(data.as_ref());
         Ok(handle)
     }
@@ -141,11 +167,11 @@ impl CurrentPlugin {
         Ok(())
     }
 
-    pub fn memory_bytes(&mut self, handle: MemoryHandle) -> Result<&mut [u8], Error> {
-        let (linker, mut store) = self.linker_and_store();
-        if let Some(mem) = linker.get(&mut store, EXTISM_ENV_MODULE, "memory") {
+    pub fn memory_bytes_mut(&mut self, handle: MemoryHandle) -> Result<&mut [u8], Error> {
+        let (linker, store) = self.linker_and_store();
+        if let Some(mem) = linker.get(&mut *store, EXTISM_ENV_MODULE, "memory") {
             let mem = mem.into_memory().unwrap();
-            let ptr = unsafe { mem.data_ptr(&store).add(handle.offset() as usize) };
+            let ptr = unsafe { mem.data_ptr(&*store).add(handle.offset() as usize) };
             if ptr.is_null() {
                 return Ok(&mut []);
             }
@@ -155,6 +181,44 @@ impl CurrentPlugin {
         anyhow::bail!("{} unable to locate extism memory", self.id)
     }
 
+    pub fn memory_bytes(&mut self, handle: MemoryHandle) -> Result<&[u8], Error> {
+        let (linker, store) = self.linker_and_store();
+        if let Some(mem) = linker.get(&mut *store, EXTISM_ENV_MODULE, "memory") {
+            let mem = mem.into_memory().unwrap();
+            let ptr = unsafe { mem.data_ptr(&*store).add(handle.offset() as usize) };
+            if ptr.is_null() {
+                return Ok(&[]);
+            }
+            return Ok(unsafe { std::slice::from_raw_parts(ptr, handle.len()) });
+        }
+
+        anyhow::bail!("{} unable to locate extism memory", self.id)
+    }
+
+    pub fn host_context<T: 'static>(&mut self) -> Result<&mut T, Error> {
+        let (linker, store) = self.linker_and_store();
+        let Some(Extern::Global(xs)) = linker.get(&mut *store, EXTISM_ENV_MODULE, "extism_context")
+        else {
+            anyhow::bail!("unable to locate an extism kernel global: extism_context",)
+        };
+
+        let Val::ExternRef(Some(xs)) = xs.get(&mut *store) else {
+            anyhow::bail!("expected extism_context to be an externref value",)
+        };
+
+        if let Some(d) = xs.data_mut(&mut *store)? {
+            match d.downcast_mut::<Box<dyn std::any::Any + Send + Sync>>() {
+                Some(xs) => match xs.downcast_mut::<T>() {
+                    Some(xs) => Ok(xs),
+                    None => anyhow::bail!("could not downcast extism_context inner value"),
+                },
+                None => anyhow::bail!("could not downcast extism_context"),
+            }
+        } else {
+            anyhow::bail!("extism_context not found")
+        }
+    }
+
     pub fn memory_alloc(&mut self, n: u64) -> Result<MemoryHandle, Error> {
         if n == 0 {
             return Ok(MemoryHandle {
@@ -165,9 +229,13 @@ impl CurrentPlugin {
         let (linker, mut store) = self.linker_and_store();
         let output = &mut [Val::I64(0)];
         if let Some(f) = linker.get(&mut store, EXTISM_ENV_MODULE, "alloc") {
-            f.into_func()
-                .unwrap()
-                .call(&mut store, &[Val::I64(n as i64)], output)?;
+            catch_out_of_fuel!(
+                &store,
+                f.into_func()
+                    .unwrap()
+                    .call(&mut *store, &[Val::I64(n as i64)], output)
+                    .context("failed to allocate extism memory")
+            )?;
         } else {
             anyhow::bail!("{} unable to allocate memory", self.id);
         }
@@ -189,11 +257,15 @@ impl CurrentPlugin {
 
     /// Free a block of Extism plugin memory
     pub fn memory_free(&mut self, handle: MemoryHandle) -> Result<(), Error> {
-        let (linker, mut store) = self.linker_and_store();
-        if let Some(f) = linker.get(&mut store, EXTISM_ENV_MODULE, "free") {
-            f.into_func()
-                .unwrap()
-                .call(&mut store, &[Val::I64(handle.offset as i64)], &mut [])?;
+        let (linker, store) = self.linker_and_store();
+        if let Some(f) = linker.get(&mut *store, EXTISM_ENV_MODULE, "free") {
+            catch_out_of_fuel!(
+                &store,
+                f.into_func()
+                    .unwrap()
+                    .call(&mut *store, &[Val::I64(handle.offset as i64)], &mut [])
+                    .context("failed to free extism memory")
+            )?;
         } else {
             anyhow::bail!("unable to locate an extism kernel function: free",)
         }
@@ -201,12 +273,16 @@ impl CurrentPlugin {
     }
 
     pub fn memory_length(&mut self, offs: u64) -> Result<u64, Error> {
-        let (linker, mut store) = self.linker_and_store();
+        let (linker, store) = self.linker_and_store();
         let output = &mut [Val::I64(0)];
-        if let Some(f) = linker.get(&mut store, EXTISM_ENV_MODULE, "length") {
-            f.into_func()
-                .unwrap()
-                .call(&mut store, &[Val::I64(offs as i64)], output)?;
+        if let Some(f) = linker.get(&mut *store, EXTISM_ENV_MODULE, "length") {
+            catch_out_of_fuel!(
+                &store,
+                f.into_func()
+                    .unwrap()
+                    .call(&mut *store, &[Val::I64(offs as i64)], output)
+                    .context("failed to get length of extism memory handle")
+            )?;
         } else {
             anyhow::bail!("unable to locate an extism kernel function: length",)
         }
@@ -220,6 +296,30 @@ impl CurrentPlugin {
         Ok(len)
     }
 
+    pub fn memory_length_unsafe(&mut self, offs: u64) -> Result<u64, Error> {
+        let (linker, store) = self.linker_and_store();
+        let output = &mut [Val::I64(0)];
+        if let Some(f) = linker.get(&mut *store, EXTISM_ENV_MODULE, "length_unsafe") {
+            catch_out_of_fuel!(
+                &store,
+                f.into_func()
+                    .unwrap()
+                    .call(&mut *store, &[Val::I64(offs as i64)], output)
+                    .context("failed to get length of extism memory using length_unsafe")
+            )?;
+        } else {
+            anyhow::bail!("unable to locate an extism kernel function: length_unsafe",)
+        }
+        let len = output[0].unwrap_i64() as u64;
+        trace!(
+            plugin = self.id.to_string(),
+            "memory_length_unsafe({}) = {}",
+            offs,
+            len
+        );
+        Ok(len)
+    }
+
     /// Access a plugin's variables
     pub fn vars(&self) -> &std::collections::BTreeMap<String, Vec<u8>> {
         &self.vars
@@ -239,35 +339,51 @@ impl CurrentPlugin {
         manifest: extism_manifest::Manifest,
         wasi: bool,
         available_pages: Option<u32>,
+        allow_http_response_headers: bool,
         id: uuid::Uuid,
     ) -> Result<Self, Error> {
         let wasi = if wasi {
-            let auth = wasmtime_wasi::ambient_authority();
-            let mut ctx = wasmtime_wasi::WasiCtxBuilder::new();
-            for (k, v) in manifest.config.iter() {
-                ctx.env(k, v)?;
-            }
+            let auth = wasi_common::sync::ambient_authority();
+            let random = wasi_common::sync::random_ctx();
+            let clocks = wasi_common::sync::clocks_ctx();
+            let sched = wasi_common::sync::sched_ctx();
+            let table = wasi_common::Table::new();
+            let ctx = wasi_common::WasiCtx::new(random, clocks, sched, table);
 
             if let Some(a) = &manifest.allowed_paths {
                 for (k, v) in a.iter() {
-                    let d = wasmtime_wasi::Dir::open_ambient_dir(k, auth)?;
-                    ctx.preopened_dir(d, v)?;
+                    let readonly = k.starts_with("ro:");
+
+                    let dir_path = if readonly { &k[3..] } else { k };
+
+                    let dir = wasi_common::sync::dir::Dir::from_cap_std(
+                        wasi_common::sync::Dir::open_ambient_dir(dir_path, auth)?,
+                    );
+
+                    let file: Box<dyn wasi_common::dir::WasiDir> = if readonly {
+                        Box::new(readonly_dir::ReadOnlyDir::new(dir))
+                    } else {
+                        Box::new(dir)
+                    };
+
+                    ctx.push_preopened_dir(file, v)?;
                 }
             }
 
             // Enable WASI output, typically used for debugging purposes
             if std::env::var("EXTISM_ENABLE_WASI_OUTPUT").is_ok() {
-                ctx.inherit_stdout().inherit_stderr();
+                ctx.set_stderr(Box::new(wasi_common::sync::stdio::stderr()));
+                ctx.set_stdout(Box::new(wasi_common::sync::stdio::stdout()));
             }
 
-            Some(Wasi { ctx: ctx.build() })
+            Some(Wasi { ctx })
         } else {
             None
         };
 
         let memory_limiter = if let Some(pgs) = available_pages {
             let n = pgs as usize * 65536;
-            Some(crate::current_plugin::MemoryLimiter {
+            Some(MemoryLimiter {
                 max_bytes: n,
                 bytes_left: n,
             })
@@ -285,6 +401,12 @@ impl CurrentPlugin {
             available_pages,
             memory_limiter,
             id,
+            start_time: std::time::Instant::now(),
+            http_headers: if allow_http_response_headers {
+                Some(BTreeMap::new())
+            } else {
+                None
+            },
         })
     }
 
@@ -329,12 +451,12 @@ impl CurrentPlugin {
     /// Clear the current plugin error
     pub fn clear_error(&mut self) {
         trace!(plugin = self.id.to_string(), "CurrentPlugin::clear_error");
-        let (linker, mut store) = self.linker_and_store();
-        if let Some(f) = linker.get(&mut store, EXTISM_ENV_MODULE, "error_set") {
+        let (linker, store) = self.linker_and_store();
+        if let Some(f) = linker.get(&mut *store, EXTISM_ENV_MODULE, "error_set") {
             let res = f
                 .into_func()
                 .unwrap()
-                .call(&mut store, &[Val::I64(0)], &mut []);
+                .call(&mut *store, &[Val::I64(0)], &mut []);
             if let Err(e) = res {
                 error!(
                     plugin = self.id.to_string(),
@@ -355,23 +477,22 @@ impl CurrentPlugin {
             offset: offs,
             length,
         });
-        match s {
-            Ok(s) => Some(s),
-            Err(_) => None,
-        }
+        s.ok()
     }
 
     #[doc(hidden)]
     pub fn set_error(&mut self, s: impl AsRef<str>) -> Result<(u64, u64), Error> {
         let s = s.as_ref();
         debug!(plugin = self.id.to_string(), "set error: {:?}", s);
-        let handle = self.current_plugin_mut().memory_new(s)?;
-        let (linker, mut store) = self.linker_and_store();
-        if let Some(f) = linker.get(&mut store, EXTISM_ENV_MODULE, "error_set") {
-            f.into_func().unwrap().call(
-                &mut store,
-                &[Val::I64(handle.offset() as i64)],
-                &mut [],
+        let handle = self.memory_new(s)?;
+        let (linker, store) = self.linker_and_store();
+        if let Some(f) = linker.get(&mut *store, EXTISM_ENV_MODULE, "error_set") {
+            catch_out_of_fuel!(
+                &store,
+                f.into_func()
+                    .unwrap()
+                    .call(&mut *store, &[Val::I64(handle.offset() as i64)], &mut [])
+                    .context("failed to set extism error")
             )?;
             Ok((handle.offset(), s.len() as u64))
         } else {
@@ -380,10 +501,13 @@ impl CurrentPlugin {
     }
 
     pub(crate) fn get_error_position(&mut self) -> (u64, u64) {
-        let (linker, mut store) = self.linker_and_store();
+        let (linker, store) = self.linker_and_store();
         let output = &mut [Val::I64(0)];
-        if let Some(f) = linker.get(&mut store, EXTISM_ENV_MODULE, "error_get") {
-            if let Err(e) = f.into_func().unwrap().call(&mut store, &[], output) {
+        if let Some(f) = linker.get(&mut *store, EXTISM_ENV_MODULE, "error_get") {
+            if let Err(e) = catch_out_of_fuel!(
+                &store,
+                f.into_func().unwrap().call(&mut *store, &[], output)
+            ) {
                 error!(
                     plugin = self.id.to_string(),
                     "unable to call extism:host/env::error_get: {:?}", e
@@ -395,6 +519,18 @@ impl CurrentPlugin {
         let length = self.memory_length(offs).unwrap_or_default();
         (offs, length)
     }
+
+    /// Returns the remaining time before a plugin will timeout, or
+    /// `None` if no timeout is configured in the manifest
+    pub fn time_remaining(&self) -> Option<std::time::Duration> {
+        if let Some(x) = &self.manifest.timeout_ms {
+            let elapsed = &self.start_time.elapsed().as_millis();
+            let ms_left = x.saturating_sub(*elapsed as u64);
+            return Some(std::time::Duration::from_millis(ms_left));
+        }
+
+        None
+    }
 }
 
 impl Internal for CurrentPlugin {
@@ -406,14 +542,6 @@ impl Internal for CurrentPlugin {
         unsafe { &mut *self.store }
     }
 
-    fn linker(&self) -> &Linker<CurrentPlugin> {
-        unsafe { &*self.linker }
-    }
-
-    fn linker_mut(&mut self) -> &mut Linker<CurrentPlugin> {
-        unsafe { &mut *self.linker }
-    }
-
     fn linker_and_store(&mut self) -> (&mut Linker<CurrentPlugin>, &mut Store<CurrentPlugin>) {
         unsafe { (&mut *self.linker, &mut *self.store) }
     }

runtime/src/function.rs

@@ -4,6 +4,7 @@ use wasmtime::Caller;
 use crate::{error, trace, CurrentPlugin, Error};
 
 /// An enumeration of all possible value types in WebAssembly.
+/// cbindgen:prefix-with-name
 #[derive(Debug, Clone, Hash, Eq, PartialEq)]
 #[repr(C)]
 pub enum ValType {
@@ -37,8 +38,13 @@ impl From<wasmtime::ValType> for ValType {
             F32 => ValType::F32,
             F64 => ValType::F64,
             V128 => ValType::V128,
-            FuncRef => ValType::FuncRef,
-            ExternRef => ValType::ExternRef,
+            Ref(t) => {
+                if t.heap_type().is_func() {
+                    ValType::FuncRef
+                } else {
+                    ValType::ExternRef
+                }
+            }
         }
     }
 }
@@ -52,8 +58,8 @@ impl From<ValType> for wasmtime::ValType {
             F32 => wasmtime::ValType::F32,
             F64 => wasmtime::ValType::F64,
             V128 => wasmtime::ValType::V128,
-            FuncRef => wasmtime::ValType::FuncRef,
-            ExternRef => wasmtime::ValType::ExternRef,
+            FuncRef => wasmtime::ValType::FUNCREF,
+            ExternRef => wasmtime::ValType::EXTERNREF,
         }
     }
 }
@@ -71,7 +77,9 @@ pub struct CPtr {
 /// UserDataHandle is an untyped version of `UserData` that is stored inside `Function` to keep a live reference.
 #[derive(Clone)]
 pub(crate) enum UserDataHandle {
+    #[allow(dead_code)]
     C(Arc<CPtr>),
+    #[allow(dead_code)]
     Rust(Arc<std::sync::Mutex<dyn std::any::Any>>),
 }
 
@@ -168,8 +176,8 @@ pub struct Function {
     /// Module name
     pub(crate) namespace: Option<String>,
 
-    /// Function type
-    pub(crate) ty: wasmtime::FuncType,
+    pub(crate) params: Vec<ValType>,
+    pub(crate) results: Vec<ValType>,
 
     /// Function handle
     pub(crate) f: Arc<FunctionInner>,
@@ -182,8 +190,8 @@ impl Function {
     /// Create a new host function
     pub fn new<T: 'static, F>(
         name: impl Into<String>,
-        args: impl IntoIterator<Item = ValType>,
-        returns: impl IntoIterator<Item = ValType>,
+        params: impl IntoIterator<Item = ValType>,
+        results: impl IntoIterator<Item = ValType>,
         user_data: UserData<T>,
         f: F,
     ) -> Function
@@ -195,13 +203,13 @@ impl Function {
     {
         let data = user_data.clone();
         let name = name.into();
-        let args = args.into_iter().map(wasmtime::ValType::from);
-        let returns = returns.into_iter().map(wasmtime::ValType::from);
-        let ty = wasmtime::FuncType::new(args, returns);
-        trace!("Creating function {name}: type={ty:?}");
+        let params = params.into_iter().collect();
+        let results = results.into_iter().collect();
+        trace!("Creating function {name}: params={params:?}, results={results:?}");
         Function {
             name,
-            ty,
+            params,
+            results,
             f: Arc::new(
                 move |mut caller: Caller<_>, inp: &[Val], outp: &mut [Val]| {
                     let x = data.clone();
@@ -216,6 +224,22 @@ impl Function {
         }
     }
 
+    pub(crate) fn ty(&self, engine: &wasmtime::Engine) -> wasmtime::FuncType {
+        wasmtime::FuncType::new(
+            engine,
+            self.params
+                .iter()
+                .cloned()
+                .map(wasmtime::ValType::from)
+                .collect::<Vec<_>>(),
+            self.results
+                .iter()
+                .cloned()
+                .map(wasmtime::ValType::from)
+                .collect::<Vec<_>>(),
+        )
+    }
+
     /// Host function name
     pub fn name(&self) -> &str {
         &self.name
@@ -239,9 +263,14 @@ impl Function {
         self
     }
 
-    /// Get function type
-    pub fn ty(&self) -> &wasmtime::FuncType {
-        &self.ty
+    /// Get param types
+    pub fn params(&self) -> &[ValType] {
+        &self.params
+    }
+
+    /// Get result types
+    pub fn results(&self) -> &[ValType] {
+        &self.results
     }
 }
 
@@ -250,36 +279,39 @@ impl Function {
 /// For example, the following defines a host function named `add_newline` that takes a
 /// string parameter and returns a string result:
 /// ```rust
-/// extism::host_fn!(add_newline(_user_data: (), a: String) -> String { Ok(a + "\n") });
+/// extism::host_fn!(add_newline(_user_data: (); a: String) -> String { Ok(a + "\n") });
 /// ```
 /// A few things worth noting:
 /// - The function always returns a `Result` that wraps the specified return type
 /// - If a first parameter and type are passed (`_user_data` above) followed by a semicolon it will be
-///    the name of the `UserData` parameter and can be used from inside the function
-//     definition.
+///   the name of the `UserData` parameter and can be used from inside the function
+//    definition.
 #[macro_export]
 macro_rules! host_fn {
-    ($name: ident  ($($arg:ident : $argty:ty),*) $(-> $ret:ty)? $b:block) => {
-        $crate::host_fn!($name (user_data: (); $($arg : $argty),*) $(-> $ret)? {$b});
+    ($pub:vis $name: ident  ($($arg:ident : $argty:ty),*) $(-> $ret:ty)? $b:block) => {
+       $crate::host_fn!($pub $name (user_data: (); $($arg : $argty),*) $(-> $ret)? {$b});
     };
-    ($name: ident  ($user_data:ident : $dataty:ty; $($arg:ident : $argty:ty),*) $(-> $ret:ty)? $b:block) => {
-        fn $name(
+    ($pub:vis $name: ident  ($user_data:ident : $dataty:ty; $($arg:ident : $argty:ty),*) $(-> $ret:ty)? $b:block) => {
+        $pub fn $name(
             plugin: &mut $crate::CurrentPlugin,
             inputs: &[$crate::Val],
             outputs: &mut [$crate::Val],
             #[allow(unused)]
             mut $user_data: $crate::UserData<$dataty>,
         ) -> Result<(), $crate::Error> {
-            let mut index = 0;
-            $(
-                let $arg: $argty = plugin.memory_get_val(&inputs[index])?;
-                #[allow(unused_assignments)]
-                {
-                    index += 1;
-                }
-            )*
-            let output = move || -> Result<_, $crate::Error> { $b };
-            let output: $crate::convert::MemoryHandle = plugin.memory_new(&output()?)?;
+            let output = {
+                let mut index = 0;
+                $(
+                    let $arg: $argty = plugin.memory_get_val(&inputs[index])?;
+                    #[allow(unused_assignments)]
+                    {
+                        index += 1;
+                    }
+                )*
+                move || -> Result<_, $crate::Error> { $b }
+            };
+            let output = output()?;
+            let output: $crate::convert::MemoryHandle = plugin.memory_new(&output)?;
             if !outputs.is_empty() {
                 outputs[0] = plugin.memory_to_val(output);
             }

runtime/src/internal.rs

@@ -3,7 +3,7 @@ use crate::*;
 /// WASI context
 pub struct Wasi {
     /// wasi
-    pub ctx: wasmtime_wasi::WasiCtx,
+    pub ctx: wasi_common::WasiCtx,
 }
 
 /// InternalExt provides a unified way of acessing `memory`, `store` and `internal` values
@@ -12,10 +12,6 @@ pub(crate) trait Internal {
 
     fn store_mut(&mut self) -> &mut Store<CurrentPlugin>;
 
-    fn linker(&self) -> &Linker<CurrentPlugin>;
-
-    fn linker_mut(&mut self) -> &mut Linker<CurrentPlugin>;
-
     fn linker_and_store(&mut self) -> (&mut Linker<CurrentPlugin>, &mut Store<CurrentPlugin>);
 
     fn current_plugin(&self) -> &CurrentPlugin {

runtime/src/lib.rs

@@ -1,8 +1,23 @@
+// Makes proc-macros able to resolve `::extism` correctly
+extern crate self as extism;
+
+macro_rules! catch_out_of_fuel {
+    ($store: expr, $x:expr) => {{
+        let y = $x;
+        if y.is_err() && $store.get_fuel().is_ok_and(|x| x == 0) {
+            Err(Error::msg("plugin ran out of fuel"))
+        } else {
+            y
+        }
+    }};
+}
+
 pub(crate) use extism_convert::*;
 pub(crate) use std::collections::BTreeMap;
 use std::str::FromStr;
 pub(crate) use wasmtime::*;
 
+#[doc(inline)]
 pub use extism_convert as convert;
 
 pub use anyhow::Error;
@@ -14,6 +29,8 @@ pub(crate) mod manifest;
 pub(crate) mod pdk;
 mod plugin;
 mod plugin_builder;
+mod pool;
+mod readonly_dir;
 mod timer;
 
 /// Extism C API
@@ -23,11 +40,13 @@ pub use current_plugin::CurrentPlugin;
 pub use extism_convert::{FromBytes, FromBytesOwned, ToBytes};
 pub use extism_manifest::{Manifest, Wasm, WasmMetadata};
 pub use function::{Function, UserData, Val, ValType, PTR};
-pub use plugin::{CancelHandle, Plugin, EXTISM_ENV_MODULE, EXTISM_USER_MODULE};
-pub use plugin_builder::PluginBuilder;
+pub use plugin::{
+    CancelHandle, CompiledPlugin, Plugin, WasmInput, EXTISM_ENV_MODULE, EXTISM_USER_MODULE,
+};
+pub use plugin_builder::{DebugOptions, PluginBuilder};
+pub use pool::{Pool, PoolBuilder, PoolPlugin};
 
 pub(crate) use internal::{Internal, Wasi};
-pub(crate) use plugin_builder::DebugOptions;
 pub(crate) use timer::{Timer, TimerAction};
 pub(crate) use tracing::{debug, error, trace, warn};
 
@@ -74,11 +93,16 @@ pub fn set_log_callback<F: 'static + Clone + Fn(&str)>(
     filter: impl AsRef<str>,
 ) -> Result<(), Error> {
     let filter = filter.as_ref();
-    let cfg = tracing_subscriber::FmtSubscriber::builder().with_env_filter(
-        tracing_subscriber::EnvFilter::builder()
-            .with_default_directive(tracing::Level::ERROR.into())
-            .parse_lossy(filter),
-    );
+    let is_level = tracing::Level::from_str(filter).is_ok();
+    let cfg = tracing_subscriber::FmtSubscriber::builder().with_env_filter({
+        let x = tracing_subscriber::EnvFilter::builder()
+            .with_default_directive(tracing::Level::ERROR.into());
+        if is_level {
+            x.parse_lossy(format!("extism={filter}"))
+        } else {
+            x.parse_lossy(filter)
+        }
+    });
     let w = LogFunction { func };
     cfg.with_ansi(false)
         .with_writer(move || w.clone())

runtime/src/manifest.rs

@@ -4,42 +4,20 @@ use std::io::Read;
 
 use sha2::Digest;
 
+use crate::plugin::{WasmInput, MAIN_KEY};
 use crate::*;
 
 fn hex(data: &[u8]) -> String {
     let mut s = String::new();
     for &byte in data {
-        write!(&mut s, "{:02x}", byte).unwrap();
+        write!(&mut s, "{byte:02x}").unwrap();
     }
     s
 }
 
-#[allow(unused)]
-fn cache_add_file(hash: &str, data: &[u8]) -> Result<(), Error> {
-    let cache_dir = std::env::temp_dir().join("extism-cache");
-    let _ = std::fs::create_dir(&cache_dir);
-    let file = cache_dir.join(hash);
-    if file.exists() {
-        return Ok(());
-    }
-    std::fs::write(file, data)?;
-    Ok(())
-}
-
-fn cache_get_file(hash: &str) -> Result<Option<Vec<u8>>, Error> {
-    let cache_dir = std::env::temp_dir().join("extism-cache");
-    let file = cache_dir.join(hash);
-    if file.exists() {
-        let r = std::fs::read(file)?;
-        return Ok(Some(r));
-    }
-
-    Ok(None)
-}
-
-fn check_hash(hash: &Option<String>, data: &[u8]) -> Result<(), Error> {
+fn check_hash(hash: &Option<String>, data: &[u8]) -> Result<Option<String>, Error> {
     match hash {
-        None => Ok(()),
+        None => Ok(None),
         Some(hash) => {
             let digest = sha2::Sha256::digest(data);
             let hex = hex(&digest);
@@ -50,7 +28,7 @@ fn check_hash(hash: &Option<String>, data: &[u8]) -> Result<(), Error> {
                     hash
                 ));
             }
-            Ok(())
+            Ok(Some(hex))
         }
     }
 }
@@ -65,28 +43,25 @@ fn to_module(engine: &Engine, wasm: &extism_manifest::Wasm) -> Result<(String, M
                 return Err(anyhow::format_err!("File-based registration is disabled"));
             }
 
-            // Figure out a good name for the file
-            let name = match &meta.name {
-                None => {
-                    let name = path.with_extension("");
-                    name.file_name().unwrap().to_string_lossy().to_string()
-                }
-                Some(n) => n.clone(),
-            };
+            // Use the configured name or `MAIN_KEY`
+            let name = meta.name.as_deref().unwrap_or(MAIN_KEY).to_string();
 
             // Load file
-            let mut buf = Vec::new();
-            let mut file = std::fs::File::open(path)?;
-            file.read_to_end(&mut buf)?;
+            let buf = std::fs::read(path).map_err(|err| {
+                Error::msg(format!(
+                    "Unable to load Wasm file \"{}\": {}",
+                    path.display(),
+                    err.kind()
+                ))
+            })?;
 
             check_hash(&meta.hash, &buf)?;
-
             Ok((name, Module::new(engine, buf)?))
         }
         extism_manifest::Wasm::Data { meta, data } => {
             check_hash(&meta.hash, data)?;
             Ok((
-                meta.name.as_deref().unwrap_or("main").to_string(),
+                meta.name.as_deref().unwrap_or(MAIN_KEY).to_string(),
                 Module::new(engine, data)?,
             ))
         }
@@ -100,59 +75,36 @@ fn to_module(engine: &Engine, wasm: &extism_manifest::Wasm) -> Result<(String, M
                 },
             meta,
         } => {
-            // Get the file name
-            let file_name = url.split('/').last().unwrap_or_default();
-            let name = match &meta.name {
-                Some(name) => name.as_str(),
-                None => {
-                    let mut name = "main";
-                    if let Some(n) = file_name.strip_suffix(".wasm") {
-                        name = n;
-                    }
-
-                    if let Some(n) = file_name.strip_suffix(".wat") {
-                        name = n;
-                    }
-                    name
-                }
-            };
-
-            if let Some(h) = &meta.hash {
-                if let Ok(Some(data)) = cache_get_file(h) {
-                    check_hash(&meta.hash, &data)?;
-                    let module = Module::new(engine, data)?;
-                    return Ok((name.to_string(), module));
-                }
-            }
+            // Use the configured name or `MAIN_KEY`
+            let name = meta.name.as_deref().unwrap_or(MAIN_KEY).to_string();
 
             #[cfg(not(feature = "register-http"))]
             {
-                return Err(anyhow::format_err!("HTTP registration is disabled"));
+                return anyhow::bail!("HTTP registration is disabled");
             }
 
             #[cfg(feature = "register-http")]
             {
                 // Setup request
-                let mut req = ureq::request(method.as_deref().unwrap_or("GET"), url);
+                let mut req = ureq::http::request::Builder::new()
+                    .method(method.as_deref().unwrap_or("GET").to_uppercase().as_str())
+                    .uri(url);
 
                 for (k, v) in headers.iter() {
-                    req = req.set(k, v);
+                    req = req.header(k, v);
                 }
 
                 // Fetch WASM code
-                let mut r = req.call()?.into_reader();
+                let mut r = ureq::run(req.body(())?)?.into_body().into_reader();
                 let mut data = Vec::new();
                 r.read_to_end(&mut data)?;
 
-                // Try to cache file
-                if let Some(hash) = &meta.hash {
-                    cache_add_file(hash, &data);
-                }
-
+                // Check hash against manifest
                 check_hash(&meta.hash, &data)?;
 
                 // Convert fetched data to module
                 let module = Module::new(engine, data)?;
+
                 Ok((name.to_string(), module))
             }
         }
@@ -163,59 +115,87 @@ const WASM_MAGIC: [u8; 4] = [0x00, 0x61, 0x73, 0x6d];
 
 pub(crate) fn load(
     engine: &Engine,
-    data: &[u8],
+    input: WasmInput<'_>,
 ) -> Result<(extism_manifest::Manifest, BTreeMap<String, Module>), Error> {
-    let extism_module = Module::new(engine, WASM)?;
-    let has_magic = data.len() >= 4 && data[0..4] == WASM_MAGIC;
-    let is_wast = data.starts_with(b"(module") || data.starts_with(b";;");
-    if !has_magic && !is_wast {
-        trace!("Loading manifest");
-        if let Ok(s) = std::str::from_utf8(data) {
-            if let Ok(t) = toml::from_str::<extism_manifest::Manifest>(s) {
-                trace!("Manifest is TOML");
-                let mut m = modules(&t, engine)?;
-                m.insert(EXTISM_ENV_MODULE.to_string(), extism_module);
-                return Ok((t, m));
+    let mut mods = BTreeMap::new();
+    mods.insert(EXTISM_ENV_MODULE.to_string(), Module::new(engine, WASM)?);
+
+    match input {
+        WasmInput::Data(data) => {
+            let has_magic = data.len() >= 4 && data[0..4] == WASM_MAGIC;
+            let s = std::str::from_utf8(&data);
+            let is_wat = s.is_ok_and(|s| {
+                let s = s.trim_start();
+                let starts_with_module = s.len() > 2
+                    && data[0] == b'('   // First character is `(`
+                    && s[1..].trim_start().starts_with("module"); // Then `module` (after any whitespace)
+                starts_with_module || s.starts_with(";;") || s.starts_with("(;")
+            });
+            if !has_magic && !is_wat {
+                trace!("Loading manifest");
+                if let Ok(s) = s {
+                    let t = if let Ok(t) = toml::from_str::<extism_manifest::Manifest>(s) {
+                        trace!("Manifest is TOML");
+                        modules(engine, &t, &mut mods)?;
+                        t
+                    } else if let Ok(t) = serde_json::from_str::<extism_manifest::Manifest>(s) {
+                        trace!("Manifest is JSON");
+                        modules(engine, &t, &mut mods)?;
+                        t
+                    } else {
+                        anyhow::bail!("Unknown manifest format");
+                    };
+                    return Ok((t, mods));
+                }
             }
+
+            let m = Module::new(engine, data)?;
+            mods.insert(MAIN_KEY.to_string(), m);
+            Ok((Default::default(), mods))
+        }
+        WasmInput::Manifest(m) => {
+            trace!("Loading from existing manifest");
+            modules(engine, &m, &mut mods)?;
+            Ok((m, mods))
+        }
+        WasmInput::ManifestRef(m) => {
+            trace!("Loading from existing manifest");
+            modules(engine, m, &mut mods)?;
+            Ok((m.clone(), mods))
         }
-
-        let t = serde_json::from_slice::<extism_manifest::Manifest>(data)?;
-        trace!("Manifest is JSON");
-        let mut m = modules(&t, engine)?;
-        m.insert(EXTISM_ENV_MODULE.to_string(), extism_module);
-        return Ok((t, m));
     }
-
-    trace!("Loading WASM module bytes");
-    let m = Module::new(engine, data)?;
-    let mut modules = BTreeMap::new();
-    modules.insert(EXTISM_ENV_MODULE.to_string(), extism_module);
-    modules.insert("main".to_string(), m);
-    Ok((Default::default(), modules))
 }
 
 pub(crate) fn modules(
-    manifest: &extism_manifest::Manifest,
     engine: &Engine,
-) -> Result<BTreeMap<String, Module>, Error> {
+    manifest: &extism_manifest::Manifest,
+    modules: &mut BTreeMap<String, Module>,
+) -> Result<(), Error> {
     if manifest.wasm.is_empty() {
-        return Err(anyhow::format_err!("No wasm files specified"));
+        return Err(anyhow::format_err!(
+            "No wasm files specified in Extism manifest"
+        ));
     }
 
-    let mut modules = BTreeMap::new();
-
     // If there's only one module, it should be called `main`
     if manifest.wasm.len() == 1 {
         let (_, m) = to_module(engine, &manifest.wasm[0])?;
-        modules.insert("main".to_string(), m);
-        return Ok(modules);
+        modules.insert(MAIN_KEY.to_string(), m);
+        return Ok(());
     }
 
-    for f in &manifest.wasm {
-        let (name, m) = to_module(engine, f)?;
+    for (i, f) in manifest.wasm.iter().enumerate() {
+        let (mut name, m) = to_module(engine, f)?;
+        // Rename the last module to `main` if no main is defined already
+        if i == manifest.wasm.len() - 1 && !modules.contains_key(MAIN_KEY) {
+            name = MAIN_KEY.to_string();
+        }
+        if modules.contains_key(&name) {
+            anyhow::bail!("Duplicate module name found in Extism manifest: {name}");
+        }
         trace!("Found module {}", name);
         modules.insert(name, m);
     }
 
-    Ok(modules)
+    Ok(())
 }

runtime/src/pdk.rs

@@ -20,6 +20,8 @@ macro_rules! args {
 /// Get a configuration value
 /// Params: i64 (offset)
 /// Returns: i64 (offset)
+/// **Note**: this function takes ownership of the handle passed in
+/// the caller should not `free` this value
 pub(crate) fn config_get(
     mut caller: Caller<CurrentPlugin>,
     input: &[Val],
@@ -30,7 +32,7 @@ pub(crate) fn config_get(
     let offset = args!(input, 0, i64) as u64;
     let handle = match data.memory_handle(offset) {
         Some(h) => h,
-        None => anyhow::bail!("invalid handle offset: {offset}"),
+        None => anyhow::bail!("invalid handle offset for config key: {offset}"),
     };
     let key = data.memory_str(handle)?;
     let key = unsafe {
@@ -38,6 +40,7 @@ pub(crate) fn config_get(
     };
     let val = data.manifest.config.get(key);
     let ptr = val.map(|x| (x.len(), x.as_ptr()));
+    data.memory_free(handle)?;
     let mem = match ptr {
         Some((len, ptr)) => {
             let bytes = unsafe { std::slice::from_raw_parts(ptr, len) };
@@ -55,6 +58,9 @@ pub(crate) fn config_get(
 /// Get a variable
 /// Params: i64 (offset)
 /// Returns: i64 (offset)
+/// **Note**: this function takes ownership of the handle passed in
+/// the caller should not `free` this value, but the return value
+/// will need to be freed
 pub(crate) fn var_get(
     mut caller: Caller<CurrentPlugin>,
     input: &[Val],
@@ -65,7 +71,7 @@ pub(crate) fn var_get(
     let offset = args!(input, 0, i64) as u64;
     let handle = match data.memory_handle(offset) {
         Some(h) => h,
-        None => anyhow::bail!("invalid handle offset: {offset}"),
+        None => anyhow::bail!("invalid handle offset for var key: {offset}"),
     };
     let key = data.memory_str(handle)?;
     let key = unsafe {
@@ -73,6 +79,8 @@ pub(crate) fn var_get(
     };
     let val = data.vars.get(key);
     let ptr = val.map(|x| (x.len(), x.as_ptr()));
+    data.memory_free(handle)?;
+
     let mem = match ptr {
         Some((len, ptr)) => {
             let bytes = unsafe { std::slice::from_raw_parts(ptr, len) };
@@ -90,6 +98,8 @@ pub(crate) fn var_get(
 /// Set a variable, if the value offset is 0 then the provided key will be removed
 /// Params: i64 (key offset), i64 (value offset)
 /// Returns: none
+/// **Note**: this function takes ownership of the handles passed in
+/// the caller should not `free` these values
 pub(crate) fn var_set(
     mut caller: Caller<CurrentPlugin>,
     input: &[Val],
@@ -97,25 +107,19 @@ pub(crate) fn var_set(
 ) -> Result<(), Error> {
     let data: &mut CurrentPlugin = caller.data_mut();
 
-    let mut size = 0;
-    for v in data.vars.values() {
-        size += v.len();
+    if data.manifest.memory.max_var_bytes.is_some_and(|x| x == 0) {
+        anyhow::bail!("Vars are disabled by this host")
     }
 
     let voffset = args!(input, 1, i64) as u64;
-
-    // If the store is larger than 100MB then stop adding things
-    if size > 1024 * 1024 * 100 && voffset != 0 {
-        return Err(Error::msg("Variable store is full"));
-    }
-
     let key_offs = args!(input, 0, i64) as u64;
+
+    let key_handle = match data.memory_handle(key_offs) {
+        Some(h) => h,
+        None => anyhow::bail!("invalid handle offset for var key: {key_offs}"),
+    };
     let key = {
-        let handle = match data.memory_handle(key_offs) {
-            Some(h) => h,
-            None => anyhow::bail!("invalid handle offset: {key_offs}"),
-        };
-        let key = data.memory_str(handle)?;
+        let key = data.memory_str(key_handle)?;
         let key_len = key.len();
         let key_ptr = key.as_ptr();
         unsafe { std::str::from_utf8_unchecked(std::slice::from_raw_parts(key_ptr, key_len)) }
@@ -124,16 +128,36 @@ pub(crate) fn var_set(
     // Remove if the value offset is 0
     if voffset == 0 {
         data.vars.remove(key);
+        data.memory_free(key_handle)?;
         return Ok(());
     }
 
     let handle = match data.memory_handle(voffset) {
         Some(h) => h,
-        None => anyhow::bail!("invalid handle offset: {key_offs}"),
+        None => anyhow::bail!("invalid handle offset for var value: {voffset}"),
     };
 
+    let mut size = std::mem::size_of::<String>()
+        + std::mem::size_of::<Vec<u8>>()
+        + key.len()
+        + handle.length as usize;
+
+    for (k, v) in data.vars.iter() {
+        size += k.len();
+        size += v.len();
+        size += std::mem::size_of::<String>() + std::mem::size_of::<Vec<u8>>();
+    }
+
+    // If the store is larger than the configured size, or 1mb by default, then stop adding things
+    if size > data.manifest.memory.max_var_bytes.unwrap_or(1024 * 1024) as usize && voffset != 0 {
+        return Err(Error::msg("Variable store is full"));
+    }
+
     let value = data.memory_bytes(handle)?.to_vec();
 
+    data.memory_free(handle)?;
+    data.memory_free(key_handle)?;
+
     // Insert the value from memory into the `vars` map
     data.vars.insert(key.to_string(), value);
 
@@ -143,6 +167,9 @@ pub(crate) fn var_set(
 /// Make an HTTP request
 /// Params: i64 (offset to JSON encoded HttpRequest), i64 (offset to body or 0)
 /// Returns: i64 (offset)
+/// **Note**: this function takes ownership of the handles passed in
+/// the caller should not `free` these values, the result will need to
+/// be freed.
 pub(crate) fn http_request(
     #[allow(unused_mut)] mut caller: Caller<CurrentPlugin>,
     input: &[Val],
@@ -156,6 +183,7 @@ pub(crate) fn http_request(
             Some(h) => h,
             None => anyhow::bail!("http_request input is invalid: {http_req_offset}"),
         };
+        data.memory_free(handle)?;
         let req: extism_manifest::HttpRequest = serde_json::from_slice(data.memory_bytes(handle)?)?;
         output[0] = Val::I64(0);
         anyhow::bail!(
@@ -166,12 +194,16 @@ pub(crate) fn http_request(
 
     #[cfg(feature = "http")]
     {
+        data.http_headers.iter_mut().for_each(|x| x.clear());
+        data.http_status = 0;
+
         use std::io::Read;
         let handle = match data.memory_handle(http_req_offset) {
             Some(h) => h,
-            None => anyhow::bail!("invalid handle offset: {http_req_offset}"),
+            None => anyhow::bail!("invalid handle offset for http request: {http_req_offset}"),
         };
         let req: extism_manifest::HttpRequest = serde_json::from_slice(data.memory_bytes(handle)?)?;
+        data.memory_free(handle)?;
 
         let body_offset = args!(input, 1, i64) as u64;
 
@@ -201,43 +233,91 @@ pub(crate) fn http_request(
             )));
         }
 
-        let mut r = ureq::request(req.method.as_deref().unwrap_or("GET"), &req.url);
+        let mut r = ureq::http::request::Builder::new()
+            .method(
+                req.method
+                    .as_deref()
+                    .unwrap_or("GET")
+                    .to_uppercase()
+                    .as_str(),
+            )
+            .uri(&req.url);
 
         for (k, v) in req.headers.iter() {
-            r = r.set(k, v);
+            r = r.header(k, v);
         }
 
+        // Set HTTP timeout to respect the manifest timeout
+        let timeout = data.time_remaining();
         let res = if body_offset > 0 {
             let handle = match data.memory_handle(body_offset) {
                 Some(h) => h,
-                None => anyhow::bail!("invalid handle offset: {http_req_offset}"),
+                None => {
+                    anyhow::bail!("invalid handle offset for http request body: {http_req_offset}")
+                }
             };
             let buf: &[u8] = data.memory_bytes(handle)?;
-            r.send_bytes(buf)
+            let agent = ureq::agent();
+            let config = agent
+                .configure_request(r.body(buf)?)
+                .http_status_as_error(false);
+            let req = config.timeout_global(timeout).build();
+            ureq::run(req)
         } else {
-            r.call()
+            let agent = ureq::agent();
+            let config = agent
+                .configure_request(r.body(())?)
+                .http_status_as_error(false);
+            let req = config.timeout_global(timeout).build();
+            ureq::run(req)
         };
 
+        if let Some(handle) = data.memory_handle(body_offset) {
+            data.memory_free(handle)?;
+        }
+
         let reader = match res {
             Ok(res) => {
-                data.http_status = res.status();
-                Some(res.into_reader())
+                if let Some(headers) = &mut data.http_headers {
+                    for (name, h) in res.headers() {
+                        if let Ok(h) = h.to_str() {
+                            headers.insert(name.as_str().to_string(), h.to_string());
+                        }
+                    }
+                }
+                data.http_status = res.status().as_u16();
+                Some(res.into_body().into_reader())
             }
             Err(e) => {
-                if let Some(res) = e.into_response() {
-                    data.http_status = res.status();
-                    Some(res.into_reader())
-                } else {
+                // Catch timeout and return
+                if let Some(d) = data.time_remaining() {
+                    if matches!(e, ureq::Error::Timeout(_)) && d.as_nanos() == 0 {
+                        anyhow::bail!("timeout");
+                    }
+                }
+                let msg = e.to_string();
+                if let ureq::Error::StatusCode(res) = e {
+                    data.http_status = res;
                     None
+                } else {
+                    return Err(Error::msg(msg));
                 }
             }
         };
 
         if let Some(reader) = reader {
             let mut buf = Vec::new();
-            reader
-                .take(1024 * 1024 * 50) // TODO: make this limit configurable
-                .read_to_end(&mut buf)?;
+            let max = if let Some(max) = &data.manifest.memory.max_http_response_bytes {
+                reader.take(*max + 1).read_to_end(&mut buf)?;
+                *max
+            } else {
+                reader.take(1024 * 1024 * 50 + 1).read_to_end(&mut buf)?;
+                1024 * 1024 * 50
+            };
+
+            if buf.len() > max as usize {
+                anyhow::bail!("HTTP response exceeds the configured maximum number of bytes: {max}")
+            }
 
             let mem = data.memory_new(&buf)?;
             output[0] = Val::I64(mem.offset() as i64);
@@ -262,6 +342,24 @@ pub(crate) fn http_status_code(
     Ok(())
 }
 
+/// Get the HTTP response headers from the last HTTP request
+/// Params: none
+/// Returns: i64 (offset)
+pub(crate) fn http_headers(
+    mut caller: Caller<CurrentPlugin>,
+    _input: &[Val],
+    output: &mut [Val],
+) -> Result<(), Error> {
+    let data: &mut CurrentPlugin = caller.data_mut();
+    if let Some(h) = &data.http_headers {
+        let headers = serde_json::to_string(h)?;
+        data.memory_set_val(&mut output[0], headers)?;
+    } else {
+        output[0] = Val::I64(0);
+    }
+    Ok(())
+}
+
 pub fn log(
     level: tracing::Level,
     mut caller: Caller<CurrentPlugin>,
@@ -269,13 +367,22 @@ pub fn log(
     _output: &mut [Val],
 ) -> Result<(), Error> {
     let data: &mut CurrentPlugin = caller.data_mut();
+
     let offset = args!(input, 0, i64) as u64;
 
+    // Check if the current log level should be logged
+    let global_log_level = tracing::level_filters::LevelFilter::current();
+    if global_log_level == tracing::level_filters::LevelFilter::OFF || level > global_log_level {
+        if let Some(handle) = data.memory_handle(offset) {
+            data.memory_free(handle)?;
+        }
+        return Ok(());
+    }
+
     let handle = match data.memory_handle(offset) {
         Some(h) => h,
-        None => anyhow::bail!("invalid handle offset: {offset}"),
+        None => anyhow::bail!("invalid handle offset for log message: {offset}"),
     };
-
     let id = data.id.to_string();
     let buf = data.memory_str(handle);
 
@@ -299,12 +406,16 @@ pub fn log(
         },
         Err(_) => tracing::error!(plugin = id, "unable to log message: {:?}", buf),
     }
+
+    data.memory_free(handle)?;
     Ok(())
 }
 
 /// Write to logs (warning)
 /// Params: i64 (offset)
 /// Returns: none
+/// **Note**: this function takes ownership of the handle passed in
+/// the caller should not `free` this value
 pub(crate) fn log_warn(
     caller: Caller<CurrentPlugin>,
     input: &[Val],
@@ -316,6 +427,8 @@ pub(crate) fn log_warn(
 /// Write to logs (info)
 /// Params: i64 (offset)
 /// Returns: none
+/// **Note**: this function takes ownership of the handle passed in
+/// the caller should not `free` this value
 pub(crate) fn log_info(
     caller: Caller<CurrentPlugin>,
     input: &[Val],
@@ -327,6 +440,8 @@ pub(crate) fn log_info(
 /// Write to logs (debug)
 /// Params: i64 (offset)
 /// Returns: none
+/// **Note**: this function takes ownership of the handle passed in
+/// the caller should not `free` this value
 pub(crate) fn log_debug(
     caller: Caller<CurrentPlugin>,
     input: &[Val],
@@ -338,6 +453,8 @@ pub(crate) fn log_debug(
 /// Write to logs (error)
 /// Params: i64 (offset)
 /// Returns: none
+/// **Note**: this function takes ownership of the handle passed in
+/// the caller should not `free` this value
 pub(crate) fn log_error(
     caller: Caller<CurrentPlugin>,
     input: &[Val],
@@ -345,3 +462,46 @@ pub(crate) fn log_error(
 ) -> Result<(), Error> {
     log(tracing::Level::ERROR, caller, input, _output)
 }
+
+/// Write to logs (trace)
+/// Params: i64 (offset)
+/// Returns: none
+/// **Note**: this function takes ownership of the handle passed in
+/// the caller should not `free` this value
+pub(crate) fn log_trace(
+    caller: Caller<CurrentPlugin>,
+    input: &[Val],
+    _output: &mut [Val],
+) -> Result<(), Error> {
+    log(tracing::Level::TRACE, caller, input, _output)
+}
+
+/// Get the log level
+/// Params: none
+/// Returns: i32 (log level)
+pub(crate) fn get_log_level(
+    mut _caller: Caller<CurrentPlugin>,
+    _input: &[Val],
+    output: &mut [Val],
+) -> Result<(), Error> {
+    let level = tracing::level_filters::LevelFilter::current();
+    if level == tracing::level_filters::LevelFilter::OFF {
+        output[0] = Val::I32(i32::MAX)
+    } else {
+        output[0] = Val::I32(log_level_to_int(
+            level.into_level().unwrap_or(tracing::Level::ERROR),
+        ));
+    }
+    Ok(())
+}
+
+/// Convert log level to integer
+pub(crate) const fn log_level_to_int(level: tracing::Level) -> i32 {
+    match level {
+        tracing::Level::TRACE => 0,
+        tracing::Level::DEBUG => 1,
+        tracing::Level::INFO => 2,
+        tracing::Level::WARN => 3,
+        tracing::Level::ERROR => 4,
+    }
+}

runtime/src/plugin_builder.rs

@@ -1,35 +1,75 @@
+use std::path::PathBuf;
+
 use crate::{plugin::WasmInput, *};
 
-#[derive(Default, Clone)]
-pub(crate) struct DebugOptions {
-    pub(crate) profiling_strategy: Option<wasmtime::ProfilingStrategy>,
-    pub(crate) coredump: Option<std::path::PathBuf>,
-    pub(crate) memdump: Option<std::path::PathBuf>,
-    pub(crate) debug_info: bool,
+#[derive(Clone)]
+pub struct DebugOptions {
+    pub profiling_strategy: wasmtime::ProfilingStrategy,
+    pub coredump: Option<std::path::PathBuf>,
+    pub memdump: Option<std::path::PathBuf>,
+    pub debug_info: bool,
+}
+
+impl Default for DebugOptions {
+    fn default() -> Self {
+        let debug_info = std::env::var("EXTISM_DEBUG").is_ok();
+        let coredump = if let Ok(x) = std::env::var("EXTISM_COREDUMP") {
+            Some(std::path::PathBuf::from(x))
+        } else {
+            None
+        };
+        let memdump = if let Ok(x) = std::env::var("EXTISM_MEMDUMP") {
+            Some(std::path::PathBuf::from(x))
+        } else {
+            None
+        };
+        DebugOptions {
+            profiling_strategy: plugin::profiling_strategy(),
+            coredump,
+            memdump,
+            debug_info,
+        }
+    }
 }
 
 /// PluginBuilder is used to configure and create `Plugin` instances
+#[derive(Clone)]
 pub struct PluginBuilder<'a> {
-    source: std::borrow::Cow<'a, [u8]>,
-    wasi: bool,
-    functions: Vec<Function>,
-    debug_options: DebugOptions,
+    pub(crate) source: WasmInput<'a>,
+    pub(crate) config: Option<wasmtime::Config>,
+    pub(crate) options: PluginBuilderOptions,
+}
+
+#[derive(Clone)]
+pub(crate) struct PluginBuilderOptions {
+    pub(crate) wasi: bool,
+    pub(crate) functions: Vec<Function>,
+    pub(crate) debug_options: DebugOptions,
+    pub(crate) cache_config: Option<Option<PathBuf>>,
+    pub(crate) fuel: Option<u64>,
+    pub(crate) http_response_headers: bool,
 }
 
 impl<'a> PluginBuilder<'a> {
     /// Create a new `PluginBuilder` from a `Manifest` or raw Wasm bytes
-    pub fn new(plugin: impl WasmInput<'a>) -> Self {
+    pub fn new(plugin: impl Into<WasmInput<'a>>) -> Self {
         PluginBuilder {
             source: plugin.into(),
-            wasi: false,
-            functions: vec![],
-            debug_options: DebugOptions::default(),
+            config: None,
+            options: PluginBuilderOptions {
+                wasi: false,
+                functions: vec![],
+                debug_options: DebugOptions::default(),
+                cache_config: None,
+                fuel: None,
+                http_response_headers: false,
+            },
         }
     }
 
     /// Enables WASI if the argument is set to `true`
     pub fn with_wasi(mut self, wasi: bool) -> Self {
-        self.wasi = wasi;
+        self.options.wasi = wasi;
         self
     }
 
@@ -48,7 +88,8 @@ impl<'a> PluginBuilder<'a> {
             + Sync
             + Send,
     {
-        self.functions
+        self.options
+            .functions
             .push(Function::new(name, args, returns, user_data, f));
         self
     }
@@ -69,39 +110,97 @@ impl<'a> PluginBuilder<'a> {
             + Sync
             + Send,
     {
-        self.functions
+        self.options
+            .functions
             .push(Function::new(name, args, returns, user_data, f).with_namespace(namespace));
         self
     }
 
     /// Add multiple host functions
     pub fn with_functions(mut self, f: impl IntoIterator<Item = Function>) -> Self {
-        self.functions.extend(f);
+        self.options.functions.extend(f);
         self
     }
 
+    /// Set profiling strategy
     pub fn with_profiling_strategy(mut self, p: wasmtime::ProfilingStrategy) -> Self {
-        self.debug_options.profiling_strategy = Some(p);
+        self.options.debug_options.profiling_strategy = p;
         self
     }
 
-    pub fn with_coredump(mut self, path: impl AsRef<std::path::Path>) -> Self {
-        self.debug_options.coredump = Some(path.as_ref().to_path_buf());
+    /// Enable Wasmtime coredump on trap
+    pub fn with_coredump(mut self, path: impl Into<std::path::PathBuf>) -> Self {
+        self.options.debug_options.coredump = Some(path.into());
         self
     }
 
-    pub fn with_memdump(mut self, path: impl AsRef<std::path::Path>) -> Self {
-        self.debug_options.memdump = Some(path.as_ref().to_path_buf());
+    /// Enable Extism memory dump when plugin calls return an error
+    pub fn with_memdump(mut self, path: impl Into<std::path::PathBuf>) -> Self {
+        self.options.debug_options.memdump = Some(path.into());
         self
     }
 
+    /// Compile with debug info
     pub fn with_debug_info(mut self) -> Self {
-        self.debug_options.debug_info = true;
+        self.options.debug_options.debug_info = true;
+        self
+    }
+
+    /// Configure debug options
+    pub fn with_debug_options(mut self, options: DebugOptions) -> Self {
+        self.options.debug_options = options;
+        self
+    }
+
+    /// Set wasmtime compilation cache config path
+    pub fn with_cache_config(mut self, dir: impl Into<PathBuf>) -> Self {
+        self.options.cache_config = Some(Some(dir.into()));
+        self
+    }
+
+    /// Turn wasmtime compilation caching off
+    pub fn with_cache_disabled(mut self) -> Self {
+        self.options.cache_config = Some(None);
+        self
+    }
+
+    /// Limit the number of instructions that can be executed
+    pub fn with_fuel_limit(mut self, fuel: u64) -> Self {
+        self.options.fuel = Some(fuel);
+        self
+    }
+
+    /// Configure an initial wasmtime config to be passed to the plugin
+    ///
+    /// **Warning**: some values might be overwritten by the Extism runtime. In particular:
+    /// - async_support
+    /// - epoch_interruption
+    /// - debug_info
+    /// - coredump_on_trap
+    /// - profiler
+    /// - wasm_tail_call
+    /// - wasm_function_references
+    /// - wasm_gc
+    ///
+    /// See the implementation details of [PluginBuilder::build] and [Plugin::build_new] to verify which values are overwritten.
+    pub fn with_wasmtime_config(mut self, config: wasmtime::Config) -> Self {
+        self.config = Some(config);
+        self
+    }
+
+    /// Enables `http_response_headers`, which allows for plugins to access response headers when using `extism:host/env::http_request`
+    pub fn with_http_response_headers(mut self, allow: bool) -> Self {
+        self.options.http_response_headers = allow;
         self
     }
 
     /// Generate a new plugin with the configured settings
     pub fn build(self) -> Result<Plugin, Error> {
-        Plugin::build_new(self.source, self.functions, self.wasi, self.debug_options)
+        Plugin::new_from_compiled(&CompiledPlugin::new(self)?)
+    }
+
+    /// Build new `CompiledPlugin`
+    pub fn compile(self) -> Result<CompiledPlugin, Error> {
+        CompiledPlugin::new(self)
     }
 }

runtime/src/pool.rs

@@ -0,0 +1,195 @@
+use crate::{Error, FromBytesOwned, Plugin, ToBytes};
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::sync::RwLock;
+
+// `PoolBuilder` is used to configure and create `Pool`s
+#[derive(Debug, Clone)]
+pub struct PoolBuilder {
+    /// Max number of concurrent instances for a plugin - by default this is set to
+    /// the output of `std::thread::available_parallelism`
+    pub max_instances: usize,
+}
+
+impl PoolBuilder {
+    /// Create a `PoolBuilder` with default values
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    /// Set the max number of parallel instances
+    pub fn with_max_instances(mut self, n: usize) -> Self {
+        self.max_instances = n;
+        self
+    }
+
+    /// Create a new `Pool` with the given configuration
+    pub fn build<F: 'static + Fn() -> Result<Plugin, Error>>(self, source: F) -> Pool {
+        Pool::new_from_builder(source, self)
+    }
+}
+
+impl Default for PoolBuilder {
+    fn default() -> Self {
+        PoolBuilder {
+            max_instances: std::thread::available_parallelism()
+                .expect("available parallelism")
+                .into(),
+        }
+    }
+}
+
+/// `PoolPlugin` is used by the pool to track the number of live instances of a particular plugin
+#[derive(Clone, Debug)]
+pub struct PoolPlugin(std::rc::Rc<std::cell::RefCell<Plugin>>);
+
+impl PoolPlugin {
+    fn new(plugin: Plugin) -> Self {
+        Self(std::rc::Rc::new(std::cell::RefCell::new(plugin)))
+    }
+
+    /// Access the underlying plugin
+    pub fn plugin(&self) -> std::cell::RefMut<'_, Plugin> {
+        self.0.borrow_mut()
+    }
+
+    /// Helper to call a plugin function on the underlying plugin
+    pub fn call<'a, Input: ToBytes<'a>, Output: FromBytesOwned>(
+        &self,
+        name: impl AsRef<str>,
+        input: Input,
+    ) -> Result<Output, Error> {
+        self.plugin().call(name.as_ref(), input)
+    }
+
+    /// Helper to get the underlying plugin's ID
+    pub fn id(&self) -> uuid::Uuid {
+        self.plugin().id
+    }
+}
+
+type PluginSource = dyn Fn() -> Result<Plugin, Error>;
+
+struct PoolInner {
+    plugin_source: Box<PluginSource>,
+    instances: Vec<PoolPlugin>,
+}
+
+unsafe impl Send for PoolInner {}
+unsafe impl Sync for PoolInner {}
+
+/// `Pool` manages threadsafe access to a limited number of instances of multiple plugins
+#[derive(Clone)]
+pub struct Pool {
+    config: PoolBuilder,
+    inner: Arc<std::sync::Mutex<PoolInner>>,
+    existing_functions: Arc<RwLock<HashMap<String, bool>>>,
+}
+
+unsafe impl Send for Pool {}
+unsafe impl Sync for Pool {}
+
+impl Pool {
+    /// Create a new pool with the default configuration
+    pub fn new<F: 'static + Fn() -> Result<Plugin, Error>>(source: F) -> Self {
+        Self::new_from_builder(Box::new(source), PoolBuilder::default())
+    }
+
+    /// Create a new pool configured using a `PoolBuilder`
+    pub fn new_from_builder<F: 'static + Fn() -> Result<Plugin, Error>>(
+        source: F,
+        builder: PoolBuilder,
+    ) -> Self {
+        Pool {
+            config: builder,
+            inner: Arc::new(std::sync::Mutex::new(PoolInner {
+                plugin_source: Box::new(source),
+                instances: Default::default(),
+            })),
+            existing_functions: RwLock::new(HashMap::default()).into(),
+        }
+    }
+
+    fn find_available(&self) -> Result<Option<PoolPlugin>, Error> {
+        let pool = self.inner.lock().unwrap();
+        for instance in pool.instances.iter() {
+            if std::rc::Rc::strong_count(&instance.0) == 1 {
+                return Ok(Some(instance.clone()));
+            }
+        }
+        Ok(None)
+    }
+
+    /// Get the number of live instances for a plugin
+    pub fn count(&self) -> usize {
+        self.inner.lock().unwrap().instances.len()
+    }
+
+    /// Get access to a plugin, this will create a new instance if needed (and allowed by the specified
+    /// max_instances). `Ok(None)` is returned if the timeout is reached before an available plugin could be
+    /// acquired
+    pub fn get(&self, timeout: std::time::Duration) -> Result<Option<PoolPlugin>, Error> {
+        let start = std::time::Instant::now();
+        let max = self.config.max_instances;
+        if let Some(avail) = self.find_available()? {
+            return Ok(Some(avail));
+        }
+
+        {
+            let mut pool = self.inner.lock().unwrap();
+            if pool.instances.len() < max {
+                let plugin = (*pool.plugin_source)()?;
+                let instance = PoolPlugin::new(plugin);
+                pool.instances.push(instance);
+                return Ok(Some(pool.instances.last().unwrap().clone()));
+            }
+        }
+
+        loop {
+            if let Ok(Some(x)) = self.find_available() {
+                return Ok(Some(x));
+            }
+            if std::time::Instant::now() - start > timeout {
+                return Ok(None);
+            }
+
+            std::thread::sleep(std::time::Duration::from_millis(100));
+        }
+    }
+
+    /// Access a plugin in a callback function. This calls `Pool::get` then the provided
+    /// callback. `Ok(None)` is returned if the timeout is reached before an available
+    /// plugin could be acquired
+    pub fn with_plugin<T>(
+        &self,
+        timeout: std::time::Duration,
+        f: impl FnOnce(&mut Plugin) -> Result<T, Error>,
+    ) -> Result<Option<T>, Error> {
+        if let Some(plugin) = self.get(timeout)? {
+            return f(&mut plugin.plugin()).map(Some);
+        }
+        Ok(None)
+    }
+
+    /// Returns `true` if the given function exists, otherwise `false`. Results are cached
+    /// after the first call.
+    pub fn function_exists(&self, name: &str, timeout: std::time::Duration) -> Result<bool, Error> {
+        // read current value if any
+        let read = self.existing_functions.read().unwrap();
+        let exists_opt = read.get(name).cloned();
+        drop(read);
+        if let Some(exists) = exists_opt {
+            Ok(exists)
+        } else {
+            // load plugin and call function_exists
+            let plugin = self.get(timeout)?;
+            let exists = plugin.unwrap().0.borrow().function_exists(name);
+
+            // write result to hashmap
+            let mut write = self.existing_functions.write().unwrap();
+            write.insert(name.to_string(), exists);
+
+            Ok(exists)
+        }
+    }
+}

runtime/src/readonly_dir.rs

@@ -0,0 +1,109 @@
+use crate::*;
+
+use wasi_common::{Error, ErrorExt};
+
+pub struct ReadOnlyDir<D: wasi_common::WasiDir> {
+    inner: std::sync::Arc<D>,
+}
+
+impl<D: wasi_common::WasiDir> ReadOnlyDir<D> {
+    pub fn new(inner: D) -> Self {
+        ReadOnlyDir {
+            inner: std::sync::Arc::new(inner),
+        }
+    }
+}
+
+#[wiggle::async_trait]
+impl<D: wasi_common::WasiDir> wasi_common::WasiDir for ReadOnlyDir<D> {
+    fn as_any(&self) -> &dyn std::any::Any {
+        self.inner.as_any()
+    }
+
+    async fn open_file(
+        &self,
+        symlink_follow: bool,
+        path: &str,
+        oflags: wasi_common::file::OFlags,
+        read: bool,
+        write: bool,
+        fdflags: wasi_common::file::FdFlags,
+    ) -> Result<wasi_common::dir::OpenResult, Error> {
+        if write {
+            return Err(Error::not_supported());
+        }
+        self.inner
+            .open_file(symlink_follow, path, oflags, read, false, fdflags)
+            .await
+    }
+
+    async fn create_dir(&self, _path: &str) -> Result<(), Error> {
+        Err(Error::not_supported())
+    }
+
+    async fn readdir(
+        &self,
+        cursor: wasi_common::dir::ReaddirCursor,
+    ) -> Result<
+        Box<dyn Iterator<Item = Result<wasi_common::dir::ReaddirEntity, Error>> + Send>,
+        Error,
+    > {
+        self.inner.readdir(cursor).await
+    }
+
+    async fn symlink(&self, _old_path: &str, _new_path: &str) -> Result<(), Error> {
+        Err(Error::not_supported())
+    }
+
+    async fn remove_dir(&self, _path: &str) -> Result<(), Error> {
+        Err(Error::not_supported())
+    }
+
+    async fn unlink_file(&self, _path: &str) -> Result<(), Error> {
+        Err(Error::not_supported())
+    }
+
+    async fn read_link(&self, path: &str) -> Result<std::path::PathBuf, Error> {
+        self.inner.read_link(path).await
+    }
+
+    async fn get_filestat(&self) -> Result<wasi_common::file::Filestat, Error> {
+        self.inner.get_filestat().await
+    }
+
+    async fn get_path_filestat(
+        &self,
+        path: &str,
+        follow_symlinks: bool,
+    ) -> Result<wasi_common::file::Filestat, Error> {
+        self.inner.get_path_filestat(path, follow_symlinks).await
+    }
+
+    async fn rename(
+        &self,
+        _path: &str,
+        _dest_dir: &dyn wasi_common::WasiDir,
+        _dest_path: &str,
+    ) -> Result<(), Error> {
+        Err(wasi_common::Error::not_supported())
+    }
+
+    async fn hard_link(
+        &self,
+        _path: &str,
+        _target_dir: &dyn wasi_common::WasiDir,
+        _target_path: &str,
+    ) -> Result<(), Error> {
+        Err(wasi_common::Error::not_supported())
+    }
+
+    async fn set_times(
+        &self,
+        _path: &str,
+        _atime: std::option::Option<wasi_common::SystemTimeSpec>,
+        _mtime: std::option::Option<wasi_common::SystemTimeSpec>,
+        _follow_symlinks: bool,
+    ) -> Result<(), Error> {
+        Err(wasi_common::Error::not_supported())
+    }
+}

runtime/src/sdk.rs

@@ -1,6 +1,6 @@
 #![allow(clippy::missing_safety_doc)]
 
-use std::os::raw::c_char;
+use std::{os::raw::c_char, ptr::null_mut};
 
 use crate::*;
 
@@ -11,6 +11,12 @@ pub struct ExtismFunction(std::cell::Cell<Option<Function>>);
 /// The return code used to specify a successful plugin call
 pub static EXTISM_SUCCESS: i32 = 0;
 
+fn make_error_msg(s: String) -> Vec<u8> {
+    let mut s = s.into_bytes();
+    s.push(0);
+    s
+}
+
 /// A union type for host function argument/return values
 #[repr(C)]
 pub union ValUnion {
@@ -38,33 +44,36 @@ pub type ExtismFunctionType = extern "C" fn(
     data: *mut std::ffi::c_void,
 );
 
-impl From<&wasmtime::Val> for ExtismVal {
-    fn from(value: &wasmtime::Val) -> Self {
-        match value.ty() {
-            wasmtime::ValType::I32 => ExtismVal {
+/// Log drain callback
+pub type ExtismLogDrainFunctionType = extern "C" fn(data: *const std::ffi::c_char, size: Size);
+
+impl ExtismVal {
+    fn from_val(value: &wasmtime::Val, ctx: impl AsContext) -> Result<Self, Error> {
+        match value.ty(ctx)? {
+            wasmtime::ValType::I32 => Ok(ExtismVal {
                 t: ValType::I32,
                 v: ValUnion {
                     i32: value.unwrap_i32(),
                 },
-            },
-            wasmtime::ValType::I64 => ExtismVal {
+            }),
+            wasmtime::ValType::I64 => Ok(ExtismVal {
                 t: ValType::I64,
                 v: ValUnion {
                     i64: value.unwrap_i64(),
                 },
-            },
-            wasmtime::ValType::F32 => ExtismVal {
+            }),
+            wasmtime::ValType::F32 => Ok(ExtismVal {
                 t: ValType::F32,
                 v: ValUnion {
                     f32: value.unwrap_f32(),
                 },
-            },
-            wasmtime::ValType::F64 => ExtismVal {
+            }),
+            wasmtime::ValType::F64 => Ok(ExtismVal {
                 t: ValType::F64,
                 v: ValUnion {
                     f64: value.unwrap_f64(),
                 },
-            },
+            }),
             t => todo!("{}", t),
         }
     }
@@ -81,6 +90,24 @@ pub unsafe extern "C" fn extism_plugin_id(plugin: *mut Plugin) -> *const u8 {
     plugin.id.as_bytes().as_ptr()
 }
 
+/// Get the current plugin's associated host context data. Returns null if call was made without
+/// host context.
+#[no_mangle]
+pub unsafe extern "C" fn extism_current_plugin_host_context(
+    plugin: *mut CurrentPlugin,
+) -> *mut std::ffi::c_void {
+    if plugin.is_null() {
+        return std::ptr::null_mut();
+    }
+
+    let plugin = &mut *plugin;
+    if let Ok(CVoidContainer(ptr)) = plugin.host_context::<CVoidContainer>() {
+        *ptr
+    } else {
+        std::ptr::null_mut()
+    }
+}
+
 /// Returns a pointer to the memory of the currently running plugin
 /// NOTE: this should only be called from host functions.
 #[no_mangle]
@@ -153,7 +180,7 @@ pub unsafe extern "C" fn extism_current_plugin_memory_free(
 /// - `n_outputs`: number of return types
 /// - `func`: the function to call
 /// - `user_data`: a pointer that will be passed to the function when it's called
-///    this value should live as long as the function exists
+///   this value should live as long as the function exists
 /// - `free_user_data`: a callback to release the `user_data` value when the resulting
 ///   `ExtismFunction` is freed.
 ///
@@ -197,7 +224,11 @@ pub unsafe extern "C" fn extism_function_new(
         output_types.clone(),
         user_data,
         move |plugin, inputs, outputs, user_data| {
-            let inputs: Vec<_> = inputs.iter().map(ExtismVal::from).collect();
+            let store = &*plugin.store;
+            let inputs: Vec<_> = inputs
+                .iter()
+                .map(|x| ExtismVal::from_val(x, store).unwrap())
+                .collect();
             let mut output_tmp: Vec<_> = output_types
                 .iter()
                 .map(|t| ExtismVal {
@@ -206,12 +237,28 @@ pub unsafe extern "C" fn extism_function_new(
                 })
                 .collect();
 
+            // We cannot simply "get" the Vec's storage pointer because
+            // the underlying storage might be invalid when the Vec is empty.
+            // In that case, we return (null, 0).
+
+            let (inputs_ptr, inputs_len) = if inputs.is_empty() {
+                (core::ptr::null(), 0 as Size)
+            } else {
+                (inputs.as_ptr(), inputs.len() as Size)
+            };
+
+            let (output_ptr, output_len) = if output_tmp.is_empty() {
+                (null_mut(), 0 as Size)
+            } else {
+                (output_tmp.as_mut_ptr(), output_tmp.len() as Size)
+            };
+
             func(
                 plugin,
-                inputs.as_ptr(),
-                inputs.len() as Size,
-                output_tmp.as_mut_ptr(),
-                output_tmp.len() as Size,
+                inputs_ptr,
+                inputs_len,
+                output_ptr,
+                output_len,
                 user_data.as_ptr(),
             );
 
@@ -219,8 +266,8 @@ pub unsafe extern "C" fn extism_function_new(
                 match tmp.t {
                     ValType::I32 => *out = Val::I32(tmp.v.i32),
                     ValType::I64 => *out = Val::I64(tmp.v.i64),
-                    ValType::F32 => *out = Val::F32(tmp.v.f32 as u32),
-                    ValType::F64 => *out = Val::F64(tmp.v.f64 as u64),
+                    ValType::F32 => *out = Val::F32(tmp.v.f32.to_bits()),
+                    ValType::F64 => *out = Val::F64(tmp.v.f64.to_bits()),
                     _ => todo!(),
                 }
             }
@@ -255,6 +302,143 @@ pub unsafe extern "C" fn extism_function_set_namespace(
     }
 }
 
+/// Pre-compile an Extism plugin
+#[no_mangle]
+pub unsafe extern "C" fn extism_compiled_plugin_new(
+    wasm: *const u8,
+    wasm_size: Size,
+    functions: *mut *const ExtismFunction,
+    n_functions: Size,
+    with_wasi: bool,
+    errmsg: *mut *mut std::ffi::c_char,
+) -> *mut CompiledPlugin {
+    trace!("Call to extism_plugin_new with wasm pointer {:?}", wasm);
+    let data = std::slice::from_raw_parts(wasm, wasm_size as usize);
+
+    let mut builder = PluginBuilder::new(data).with_wasi(with_wasi);
+
+    if !functions.is_null() {
+        let funcs = (0..n_functions)
+            .map(|i| unsafe { *functions.add(i as usize) })
+            .map(|ptr| {
+                if ptr.is_null() {
+                    return Err("Cannot pass null pointer");
+                }
+
+                let ExtismFunction(func) = &*ptr;
+                let Some(func) = func.take() else {
+                    return Err("Function cannot be registered with multiple different Plugins");
+                };
+
+                Ok(func)
+            })
+            .collect::<Result<Vec<_>, _>>()
+            .unwrap_or_else(|e| {
+                if !errmsg.is_null() {
+                    let e = std::ffi::CString::new(e.to_string()).unwrap();
+                    *errmsg = e.into_raw();
+                }
+                Vec::new()
+            });
+
+        if funcs.len() != n_functions as usize {
+            return std::ptr::null_mut();
+        }
+
+        builder = builder.with_functions(funcs);
+    }
+
+    CompiledPlugin::new(builder)
+        .map(|v| Box::into_raw(Box::new(v)))
+        .unwrap_or_else(|e| {
+            if !errmsg.is_null() {
+                let e = std::ffi::CString::new(format!(
+                    "Unable to compile Extism plugin: {}",
+                    e.root_cause(),
+                ))
+                .unwrap();
+                *errmsg = e.into_raw();
+            }
+            std::ptr::null_mut()
+        })
+}
+
+/// Pre-compile an Extism plugin and set the number of instructions a plugin is allowed to execute
+#[no_mangle]
+pub unsafe extern "C" fn extism_compiled_plugin_new_with_fuel_limit(
+    wasm: *const u8,
+    wasm_size: Size,
+    functions: *mut *const ExtismFunction,
+    n_functions: Size,
+    with_wasi: bool,
+    fuel_limit: u64,
+    errmsg: *mut *mut std::ffi::c_char,
+) -> *mut CompiledPlugin {
+    trace!("Call to extism_plugin_new with wasm pointer {:?}", wasm);
+    let data = std::slice::from_raw_parts(wasm, wasm_size as usize);
+
+    let mut builder = PluginBuilder::new(data)
+        .with_wasi(with_wasi)
+        .with_fuel_limit(fuel_limit);
+
+    if !functions.is_null() {
+        let funcs = (0..n_functions)
+            .map(|i| unsafe { *functions.add(i as usize) })
+            .map(|ptr| {
+                if ptr.is_null() {
+                    return Err("Cannot pass null pointer");
+                }
+
+                let ExtismFunction(func) = &*ptr;
+                let Some(func) = func.take() else {
+                    return Err("Function cannot be registered with multiple different Plugins");
+                };
+
+                Ok(func)
+            })
+            .collect::<Result<Vec<_>, _>>()
+            .unwrap_or_else(|e| {
+                if !errmsg.is_null() {
+                    let e = std::ffi::CString::new(e.to_string()).unwrap();
+                    *errmsg = e.into_raw();
+                }
+                Vec::new()
+            });
+
+        if funcs.len() != n_functions as usize {
+            return std::ptr::null_mut();
+        }
+
+        builder = builder.with_functions(funcs);
+    }
+
+    CompiledPlugin::new(builder)
+        .map(|v| Box::into_raw(Box::new(v)))
+        .unwrap_or_else(|e| {
+            if !errmsg.is_null() {
+                let e = std::ffi::CString::new(format!(
+                    "Unable to compile Extism plugin: {}",
+                    e.root_cause(),
+                ))
+                .unwrap();
+                *errmsg = e.into_raw();
+            }
+            std::ptr::null_mut()
+        })
+}
+
+/// Free `ExtismCompiledPlugin`
+#[no_mangle]
+pub unsafe extern "C" fn extism_compiled_plugin_free(plugin: *mut CompiledPlugin) {
+    if plugin.is_null() {
+        return;
+    }
+
+    let plugin = Box::from_raw(plugin);
+    trace!("called extism_compiled_plugin_free");
+    drop(plugin)
+}
+
 /// Create a new plugin with host functions, the functions passed to this function no longer need to be manually freed using
 ///
 /// `wasm`: is a WASM module (wat or wasm) or a JSON encoded manifest
@@ -271,36 +455,70 @@ pub unsafe extern "C" fn extism_plugin_new(
     with_wasi: bool,
     errmsg: *mut *mut std::ffi::c_char,
 ) -> *mut Plugin {
-    trace!("Call to extism_plugin_new with wasm pointer {:?}", wasm);
     let data = std::slice::from_raw_parts(wasm, wasm_size as usize);
-    let mut funcs = vec![];
-
-    if !functions.is_null() {
-        for i in 0..n_functions {
-            unsafe {
-                let f = *functions.add(i as usize);
-                if f.is_null() {
-                    continue;
+    let funcs = if functions.is_null() {
+        vec![]
+    } else {
+        let funcs = (0..n_functions)
+            .map(|i| unsafe { *functions.add(i as usize) })
+            .map(|ptr| {
+                if ptr.is_null() {
+                    return Err("Cannot pass null pointer");
                 }
-                if let Some(f) = (*f).0.take() {
-                    funcs.push(f);
-                } else {
-                    let e = std::ffi::CString::new(
-                        "Function cannot be registered with multiple different Plugins",
-                    )
-                    .unwrap();
+
+                let ExtismFunction(func) = &*ptr;
+                let Some(func) = func.take() else {
+                    return Err("Function cannot be registered with multiple different Plugins");
+                };
+
+                Ok(func)
+            })
+            .collect::<Result<Vec<_>, _>>()
+            .unwrap_or_else(|e| {
+                if !errmsg.is_null() {
+                    let e = std::ffi::CString::new(e.to_string()).unwrap();
                     *errmsg = e.into_raw();
                 }
-            }
-        }
-    }
+                Vec::new()
+            });
 
-    let plugin = Plugin::new(data, funcs, with_wasi);
+        if funcs.len() != n_functions as usize {
+            return std::ptr::null_mut();
+        }
+
+        funcs
+    };
+
+    Plugin::new(data, funcs, with_wasi)
+        .map(|v| Box::into_raw(Box::new(v)))
+        .unwrap_or_else(|e| {
+            if !errmsg.is_null() {
+                let e = std::ffi::CString::new(format!(
+                    "Unable to compile Extism plugin: {}",
+                    e.root_cause(),
+                ))
+                .unwrap();
+                *errmsg = e.into_raw();
+            }
+            std::ptr::null_mut()
+        })
+}
+
+/// Create a new plugin from an `ExtismCompiledPlugin`
+#[no_mangle]
+pub unsafe extern "C" fn extism_plugin_new_from_compiled(
+    compiled: *const CompiledPlugin,
+    errmsg: *mut *mut std::ffi::c_char,
+) -> *mut Plugin {
+    let plugin = Plugin::new_from_compiled(&*compiled);
     match plugin {
         Err(e) => {
             if !errmsg.is_null() {
-                let e = std::ffi::CString::new(format!("Unable to create Extism plugin: {}", e))
-                    .unwrap();
+                let e = std::ffi::CString::new(format!(
+                    "Unable to create Extism plugin: {}",
+                    e.root_cause(),
+                ))
+                .unwrap();
                 *errmsg = e.into_raw();
             }
             std::ptr::null_mut()
@@ -309,6 +527,100 @@ pub unsafe extern "C" fn extism_plugin_new(
     }
 }
 
+/// Create a new plugin and set the number of instructions a plugin is allowed to execute
+#[no_mangle]
+pub unsafe extern "C" fn extism_plugin_new_with_fuel_limit(
+    wasm: *const u8,
+    wasm_size: Size,
+    functions: *mut *const ExtismFunction,
+    n_functions: Size,
+    with_wasi: bool,
+    fuel_limit: u64,
+    errmsg: *mut *mut std::ffi::c_char,
+) -> *mut Plugin {
+    trace!(
+        "Call to extism_plugin_new_with_fuel_limit with wasm pointer {:?}",
+        wasm
+    );
+    let data = std::slice::from_raw_parts(wasm, wasm_size as usize);
+    let funcs = if functions.is_null() {
+        vec![]
+    } else {
+        let funcs = (0..n_functions)
+            .map(|i| unsafe { *functions.add(i as usize) })
+            .map(|ptr| {
+                if ptr.is_null() {
+                    return Err("Cannot pass null pointer");
+                }
+
+                let ExtismFunction(func) = &*ptr;
+                let Some(func) = func.take() else {
+                    return Err("Function cannot be registered with multiple different Plugins");
+                };
+
+                Ok(func)
+            })
+            .collect::<Result<Vec<_>, _>>()
+            .unwrap_or_else(|e| {
+                if !errmsg.is_null() {
+                    let e = std::ffi::CString::new(e.to_string()).unwrap();
+                    *errmsg = e.into_raw();
+                }
+                Vec::new()
+            });
+
+        if funcs.len() != n_functions as usize {
+            return std::ptr::null_mut();
+        }
+
+        funcs
+    };
+
+    let compiled = match CompiledPlugin::new(
+        PluginBuilder::new(data)
+            .with_functions(funcs)
+            .with_wasi(with_wasi)
+            .with_fuel_limit(fuel_limit),
+    ) {
+        Ok(x) => x,
+        Err(e) => {
+            if !errmsg.is_null() {
+                let e = std::ffi::CString::new(format!(
+                    "Unable to compile Extism plugin: {}",
+                    e.root_cause(),
+                ))
+                .unwrap();
+                *errmsg = e.into_raw();
+            }
+            return std::ptr::null_mut();
+        }
+    };
+
+    let plugin = Plugin::new_from_compiled(&compiled);
+
+    match plugin {
+        Err(e) => {
+            if !errmsg.is_null() {
+                let e = std::ffi::CString::new(format!(
+                    "Unable to create Extism plugin: {}",
+                    e.root_cause(),
+                ))
+                .unwrap();
+                *errmsg = e.into_raw();
+            }
+            std::ptr::null_mut()
+        }
+        Ok(p) => Box::into_raw(Box::new(p)),
+    }
+}
+
+/// Enable HTTP response headers in plugins using `extism:host/env::http_request`
+#[no_mangle]
+pub unsafe extern "C" fn extism_plugin_allow_http_response_headers(plugin: *mut Plugin) {
+    let plugin = &mut *plugin;
+    plugin.store.data_mut().http_headers = Some(BTreeMap::new());
+}
+
 /// Free the error returned by `extism_plugin_new`, errors returned from `extism_plugin_error` don't need to be freed
 #[no_mangle]
 pub unsafe extern "C" fn extism_plugin_new_error_free(err: *mut std::ffi::c_char) {
@@ -318,7 +630,7 @@ pub unsafe extern "C" fn extism_plugin_new_error_free(err: *mut std::ffi::c_char
     drop(std::ffi::CString::from_raw(err))
 }
 
-/// Remove a plugin from the registry and free associated memory
+/// Free `ExtismPlugin`
 #[no_mangle]
 pub unsafe extern "C" fn extism_plugin_free(plugin: *mut Plugin) {
     if plugin.is_null() {
@@ -369,8 +681,6 @@ pub unsafe extern "C" fn extism_plugin_config(
         return false;
     }
     let plugin = &mut *plugin;
-    let _lock = plugin.instance.clone();
-    let mut lock = _lock.lock().unwrap();
 
     trace!(
         plugin = plugin.id.to_string(),
@@ -381,25 +691,11 @@ pub unsafe extern "C" fn extism_plugin_config(
     let json: std::collections::BTreeMap<String, Option<String>> =
         match serde_json::from_slice(data) {
             Ok(x) => x,
-            Err(e) => {
-                return plugin.return_error(&mut lock, e, false);
+            Err(_) => {
+                return false;
             }
         };
 
-    let wasi = &mut plugin.current_plugin_mut().wasi;
-    if let Some(Wasi { ctx, .. }) = wasi {
-        for (k, v) in json.iter() {
-            match v {
-                Some(v) => {
-                    let _ = ctx.push_env(k, v);
-                }
-                None => {
-                    let _ = ctx.push_env(k, "");
-                }
-            }
-        }
-    }
-
     let id = plugin.id;
     let config = &mut plugin.current_plugin_mut().manifest.config;
     for (k, v) in json.into_iter() {
@@ -429,9 +725,6 @@ pub unsafe extern "C" fn extism_plugin_function_exists(
         return false;
     }
     let plugin = &mut *plugin;
-    let _lock = plugin.instance.clone();
-    let mut lock = _lock.lock().unwrap();
-
     let name = std::ffi::CStr::from_ptr(func_name);
     trace!(
         plugin = plugin.id.to_string(),
@@ -441,8 +734,8 @@ pub unsafe extern "C" fn extism_plugin_function_exists(
 
     let name = match name.to_str() {
         Ok(x) => x,
-        Err(e) => {
-            return plugin.return_error(&mut lock, e, false);
+        Err(_) => {
+            return false;
         }
     };
 
@@ -461,6 +754,31 @@ pub unsafe extern "C" fn extism_plugin_call(
     func_name: *const c_char,
     data: *const u8,
     data_len: Size,
+) -> i32 {
+    extism_plugin_call_with_host_context(plugin, func_name, data, data_len, std::ptr::null_mut())
+}
+
+#[derive(Clone)]
+#[repr(transparent)]
+struct CVoidContainer(*mut std::ffi::c_void);
+
+// "You break it, you buy it."
+unsafe impl Send for CVoidContainer {}
+unsafe impl Sync for CVoidContainer {}
+
+/// Call a function with host context.
+///
+/// `func_name`: is the function to call
+/// `data`: is the input data
+/// `data_len`: is the length of `data`
+/// `host_context`: a pointer to context data that will be available in host functions
+#[no_mangle]
+pub unsafe extern "C" fn extism_plugin_call_with_host_context(
+    plugin: *mut Plugin,
+    func_name: *const c_char,
+    data: *const u8,
+    data_len: Size,
+    host_context: *mut std::ffi::c_void,
 ) -> i32 {
     if plugin.is_null() {
         return -1;
@@ -473,7 +791,10 @@ pub unsafe extern "C" fn extism_plugin_call(
     let name = std::ffi::CStr::from_ptr(func_name);
     let name = match name.to_str() {
         Ok(name) => name,
-        Err(e) => return plugin.return_error(&mut lock, e, -1),
+        Err(e) => {
+            plugin.error_msg = Some(make_error_msg(e.to_string()));
+            return -1;
+        }
     };
 
     trace!(
@@ -482,10 +803,17 @@ pub unsafe extern "C" fn extism_plugin_call(
         name
     );
     let input = std::slice::from_raw_parts(data, data_len as usize);
-    let res = plugin.raw_call(&mut lock, name, input);
-
+    let r = if host_context.is_null() {
+        None
+    } else {
+        Some(CVoidContainer(host_context))
+    };
+    let res = plugin.raw_call(&mut lock, name, input, r);
     match res {
-        Err((e, rc)) => plugin.return_error(&mut lock, e, rc),
+        Err((e, rc)) => {
+            plugin.error_msg = Some(make_error_msg(e.to_string()));
+            rc
+        }
         Ok(x) => x,
     }
 }
@@ -508,14 +836,26 @@ pub unsafe extern "C" fn extism_plugin_error(plugin: *mut Plugin) -> *const c_ch
     let _lock = _lock.lock().unwrap();
 
     if plugin.output.error_offset == 0 {
+        if let Some(err) = &plugin.error_msg {
+            return err.as_ptr() as *const _;
+        }
         trace!(plugin = plugin.id.to_string(), "error is NULL");
         return std::ptr::null();
     }
 
-    plugin
+    let offs = plugin.output.error_offset;
+
+    let ptr = plugin.current_plugin_mut().memory_ptr().add(offs as usize) as *const _;
+
+    let len = plugin
         .current_plugin_mut()
-        .memory_ptr()
-        .add(plugin.output.error_offset as usize) as *const _
+        .memory_length(offs)
+        .unwrap_or_default();
+
+    let mut data = std::slice::from_raw_parts(ptr, len as usize).to_vec();
+    data.push(0);
+    plugin.error_msg = Some(data);
+    plugin.error_msg.as_ref().unwrap().as_ptr() as *const _
 }
 
 /// Get the length of a plugin's output data
@@ -590,13 +930,12 @@ pub unsafe extern "C" fn extism_log_file(
 fn set_log_file(log_file: impl Into<std::path::PathBuf>, filter: &str) -> Result<(), Error> {
     let log_file = log_file.into();
     let s = log_file.to_str();
-
     let is_level = tracing::Level::from_str(filter).is_ok();
     let cfg = tracing_subscriber::FmtSubscriber::builder().with_env_filter({
         let x = tracing_subscriber::EnvFilter::builder()
             .with_default_directive(tracing::Level::ERROR.into());
         if is_level {
-            x.parse_lossy(format!("extism={}", filter))
+            x.parse_lossy(format!("extism={filter}"))
         } else {
             x.parse_lossy(filter)
         }
@@ -624,7 +963,7 @@ fn set_log_file(log_file: impl Into<std::path::PathBuf>, filter: &str) -> Result
     Ok(())
 }
 
-static mut LOG_BUFFER: Option<LogBuffer> = None;
+static LOG_BUFFER: std::sync::Mutex<Option<LogBuffer>> = std::sync::Mutex::new(None);
 
 /// Enable a custom log handler, this will buffer logs until `extism_log_drain` is called
 /// Log level should be one of: info, error, trace, debug, warn
@@ -641,6 +980,7 @@ pub unsafe extern "C" fn extism_log_custom(log_level: *const c_char) -> bool {
     } else {
         "error"
     };
+
     set_log_buffer(level).is_ok()
 }
 
@@ -650,13 +990,13 @@ unsafe fn set_log_buffer(filter: &str) -> Result<(), Error> {
         let x = tracing_subscriber::EnvFilter::builder()
             .with_default_directive(tracing::Level::ERROR.into());
         if is_level {
-            x.parse_lossy(format!("extism={}", filter))
+            x.parse_lossy(format!("extism={filter}"))
         } else {
             x.parse_lossy(filter)
         }
     });
-    LOG_BUFFER = Some(LogBuffer::default());
-    let buf = LOG_BUFFER.clone().unwrap();
+    *LOG_BUFFER.lock().unwrap() = Some(LogBuffer::default());
+    let buf = LOG_BUFFER.lock().unwrap().clone().unwrap();
     cfg.with_ansi(false)
         .with_writer(move || buf.clone())
         .try_init()
@@ -667,11 +1007,11 @@ unsafe fn set_log_buffer(filter: &str) -> Result<(), Error> {
 #[no_mangle]
 /// Calls the provided callback function for each buffered log line.
 /// This is only needed when `extism_log_custom` is used.
-pub unsafe extern "C" fn extism_log_drain(handler: extern "C" fn(*const std::ffi::c_char, usize)) {
-    if let Some(buf) = &mut LOG_BUFFER {
+pub unsafe extern "C" fn extism_log_drain(handler: ExtismLogDrainFunctionType) {
+    if let Some(buf) = LOG_BUFFER.lock().unwrap().as_mut() {
         if let Ok(mut buf) = buf.buffer.lock() {
             for (line, len) in buf.drain(..) {
-                handler(line.as_ptr(), len);
+                handler(line.as_ptr(), len as u64);
             }
         }
     }
@@ -702,6 +1042,30 @@ impl std::io::Write for LogBuffer {
     }
 }
 
+/// Reset the Extism runtime, this will invalidate all allocated memory
+#[no_mangle]
+pub unsafe extern "C" fn extism_plugin_reset(plugin: *mut Plugin) -> bool {
+    let plugin = &mut *plugin;
+
+    if let Err(e) = plugin.reset() {
+        error!(
+            plugin = plugin.id.to_string(),
+            "unable to reset plugin: {}",
+            e.to_string()
+        );
+        if let Err(e) = plugin.current_plugin_mut().set_error(e.to_string()) {
+            error!(
+                plugin = plugin.id.to_string(),
+                "unable to set error after failed plugin reset: {}",
+                e.to_string()
+            );
+        }
+        false
+    } else {
+        true
+    }
+}
+
 /// Get the Extism version string
 #[no_mangle]
 pub unsafe extern "C" fn extism_version() -> *const c_char {

runtime/src/tests/data/data.txt

@@ -0,0 +1 @@
+hello world!

runtime/src/tests/issues.rs

@@ -0,0 +1,57 @@
+use crate::*;
+
+const WASM_EMPTY: &[u8] = include_bytes!("../../../wasm/empty.wasm");
+const WASM_UNREACHABLE: &[u8] = include_bytes!("../../../wasm/unreachable.wasm");
+
+// https://github.com/extism/extism/issues/620
+#[test]
+fn test_issue_620() {
+    // Load and build plugin
+    let url = Wasm::data(WASM_EMPTY);
+    let manifest = Manifest::new([url]);
+    let mut plugin = PluginBuilder::new(manifest)
+        .with_wasi(true)
+        .build()
+        .unwrap();
+    // Call test method, this does not work
+    let p = plugin.call::<(), String>("test", ()).unwrap();
+
+    println!("{p}");
+}
+
+// https://github.com/extism/extism/issues/619
+host_fn!(
+    _resolve_file_path(path: &str) -> String {
+        let path = std::path::PathBuf::from(path);
+        let path = path.canonicalize()?;
+        Ok(path.display().to_string())
+    }
+);
+
+// https://github.com/extism/extism/issues/775
+#[test]
+fn test_issue_775() {
+    // Load and build plugin
+    let url = Wasm::data(WASM_UNREACHABLE);
+    let manifest = Manifest::new([url]);
+    let mut plugin = PluginBuilder::new(manifest)
+        .with_wasi(true)
+        .build()
+        .unwrap();
+    // Call test method
+    let lock = plugin.instance.clone();
+    let mut lock = lock.lock().unwrap();
+    let res = plugin.raw_call(&mut lock, "do_unreachable", b"", None::<()>);
+    let p = match res {
+        Err(e) => {
+            if e.1 == 0 {
+                Err(e.1)
+            } else {
+                Ok(e.1)
+            }
+        }
+        Ok(code) => Err(code),
+    }
+    .unwrap();
+    println!("{p}");
+}

runtime/src/tests/kernel.rs

@@ -1,23 +1,34 @@
 use crate::*;
+use quickcheck::*;
 
 const KERNEL: &[u8] = include_bytes!("../extism-runtime.wasm");
 
-fn extism_alloc<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance, n: u64) -> u64 {
+fn extism_alloc<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, n: u64) -> u64 {
     let out_alloc = &mut [Val::I64(0)];
     instance
-        .get_func(&mut store, "alloc")
+        .get_func(&mut *store, "alloc")
         .unwrap()
-        .call(&mut store, &[Val::I64(n as i64)], out_alloc)
+        .call(store, &[Val::I64(n as i64)], out_alloc)
         .unwrap();
     out_alloc[0].unwrap_i64() as u64
 }
 
-fn extism_length<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) -> u64 {
+fn extism_length<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) -> u64 {
     let out = &mut [Val::I64(0)];
     instance
-        .get_func(&mut store, "length")
+        .get_func(&mut *store, "length")
         .unwrap()
-        .call(&mut store, &[Val::I64(p as i64)], out)
+        .call(store, &[Val::I64(p as i64)], out)
+        .unwrap();
+    out[0].unwrap_i64() as u64
+}
+
+fn extism_length_unsafe<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) -> u64 {
+    let out = &mut [Val::I64(0)];
+    instance
+        .get_func(&mut *store, "length_unsafe")
+        .unwrap()
+        .call(store, &[Val::I64(p as i64)], out)
         .unwrap();
     out[0].unwrap_i64() as u64
 }
@@ -32,122 +43,96 @@ fn extism_load_u8<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance
     out[0].unwrap_i32() as u8
 }
 
-fn extism_load_u64<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) -> u64 {
+fn extism_load_u64<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) -> u64 {
     let out = &mut [Val::I32(0)];
     instance
-        .get_func(&mut store, "load_u64")
+        .get_func(&mut *store, "load_u64")
         .unwrap()
-        .call(&mut store, &[Val::I64(p as i64)], out)
+        .call(store, &[Val::I64(p as i64)], out)
         .unwrap();
     out[0].unwrap_i64() as u64
 }
 
-fn extism_input_load_u8<T>(
-    mut store: &mut wasmtime::Store<T>,
-    instance: &mut Instance,
-    p: u64,
-) -> u8 {
+fn extism_input_load_u8<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) -> u8 {
     let out = &mut [Val::I32(0)];
     instance
-        .get_func(&mut store, "input_load_u8")
+        .get_func(&mut *store, "input_load_u8")
         .unwrap()
-        .call(&mut store, &[Val::I64(p as i64)], out)
+        .call(store, &[Val::I64(p as i64)], out)
         .unwrap();
     out[0].unwrap_i32() as u8
 }
 
 fn extism_input_load_u64<T>(
-    mut store: &mut wasmtime::Store<T>,
+    store: &mut wasmtime::Store<T>,
     instance: &mut Instance,
     p: u64,
 ) -> u64 {
     let out = &mut [Val::I32(0)];
     instance
-        .get_func(&mut store, "input_load_u64")
+        .get_func(&mut *store, "input_load_u64")
         .unwrap()
-        .call(&mut store, &[Val::I64(p as i64)], out)
+        .call(store, &[Val::I64(p as i64)], out)
         .unwrap();
     out[0].unwrap_i64() as u64
 }
 
-fn extism_store_u8<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64, x: u8) {
+fn extism_store_u8<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64, x: u8) {
     instance
-        .get_func(&mut store, "store_u8")
+        .get_func(&mut *store, "store_u8")
         .unwrap()
-        .call(
-            &mut store,
-            &[Val::I64(p as i64), Val::I32(x as i32)],
-            &mut [],
-        )
+        .call(store, &[Val::I64(p as i64), Val::I32(x as i32)], &mut [])
         .unwrap();
 }
 
-fn extism_store_u64<T>(
-    mut store: &mut wasmtime::Store<T>,
-    instance: &mut Instance,
-    p: u64,
-    x: u64,
-) {
+fn extism_store_u64<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64, x: u64) {
     instance
-        .get_func(&mut store, "store_u64")
+        .get_func(&mut *store, "store_u64")
         .unwrap()
-        .call(
-            &mut store,
-            &[Val::I64(p as i64), Val::I64(x as i64)],
-            &mut [],
-        )
+        .call(store, &[Val::I64(p as i64), Val::I64(x as i64)], &mut [])
         .unwrap();
 }
 
-fn extism_free<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) {
+fn extism_free<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) {
     instance
-        .get_func(&mut store, "free")
+        .get_func(&mut *store, "free")
         .unwrap()
-        .call(&mut store, &[Val::I64(p as i64)], &mut [])
+        .call(store, &[Val::I64(p as i64)], &mut [])
         .unwrap();
 }
 
-fn extism_error_set<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) {
+fn extism_error_set<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) {
     instance
-        .get_func(&mut store, "error_set")
+        .get_func(&mut *store, "error_set")
         .unwrap()
-        .call(&mut store, &[Val::I64(p as i64)], &mut [])
+        .call(store, &[Val::I64(p as i64)], &mut [])
         .unwrap();
 }
 
-fn extism_error_get<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance) -> u64 {
+fn extism_error_get<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance) -> u64 {
     let out = &mut [Val::I64(0)];
     instance
-        .get_func(&mut store, "error_get")
+        .get_func(&mut *store, "error_get")
         .unwrap()
-        .call(&mut store, &[], out)
+        .call(store, &[], out)
         .unwrap();
 
     out[0].unwrap_i64() as u64
 }
 
-fn extism_reset<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance) {
+fn extism_reset<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance) {
     instance
-        .get_func(&mut store, "reset")
+        .get_func(&mut *store, "reset")
         .unwrap()
-        .call(&mut store, &[], &mut [])
+        .call(store, &[], &mut [])
         .unwrap();
 }
 
-fn extism_input_set<T>(
-    mut store: &mut wasmtime::Store<T>,
-    instance: &mut Instance,
-    p: u64,
-    l: u64,
-) {
+fn extism_input_set<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64, l: u64) {
     instance
-        .get_func(&mut store, "input_set")
+        .get_func(&mut *store, "input_set")
         .unwrap()
-        .call(
-            &mut store,
-            &[Val::I64(p as i64), Val::I64(l as i64)],
-            &mut [],
-        )
+        .call(store, &[Val::I64(p as i64), Val::I64(l as i64)], &mut [])
         .unwrap();
 }
 
@@ -168,11 +153,32 @@ fn test_kernel_allocations() {
     // Test allocations
     assert_eq!(extism_alloc(&mut store, instance, 0), 0);
 
+    // 512 bytes, test block re-use + splitting
+    let p = extism_alloc(&mut store, instance, 65535);
+    let first_alloc = p;
+    assert_eq!(extism_length(&mut store, instance, p), 65535);
+    assert_eq!(extism_length(&mut store, instance, p + 1), 0);
+    assert_eq!(extism_length(&mut store, instance, p + 2), 0);
+    assert_eq!(extism_length(&mut store, instance, p + 3), 0);
+    assert_eq!(extism_length(&mut store, instance, p + 4), 0);
+    extism_free(&mut store, instance, p);
+
+    // Should re-use the previous block
+    let q = extism_alloc(&mut store, instance, 65535);
+    assert_eq!(q, p);
+    assert_eq!(extism_length(&mut store, instance, q), 65535);
+    extism_free(&mut store, instance, q);
+
+    let r = extism_alloc(&mut store, instance, 65535 - 24);
+    assert_eq!(r, q);
+    assert_eq!(extism_length(&mut store, instance, q), 65535 - 24);
+    extism_free(&mut store, instance, r);
+
     // 1 byte
     let p = extism_alloc(&mut store, instance, 1);
-    let first_alloc = p;
     assert!(p > 0);
     assert_eq!(extism_length(&mut store, instance, p), 1);
+    assert_eq!(extism_length_unsafe(&mut store, instance, p), 1);
     extism_free(&mut store, instance, p);
 
     // 2 bytes
@@ -180,39 +186,17 @@ fn test_kernel_allocations() {
     assert!(x > 0);
     assert!(x != p);
     assert_eq!(extism_length(&mut store, instance, x), 2);
+    assert_eq!(extism_length_unsafe(&mut store, instance, x), 2);
     extism_free(&mut store, instance, x);
 
     for i in 0..64 {
         let p = extism_alloc(&mut store, instance, 64 - i);
         assert!(p > 0);
         assert_eq!(extism_length(&mut store, instance, p), 64 - i);
+        assert_eq!(extism_length_unsafe(&mut store, instance, p), 64 - i);
         extism_free(&mut store, instance, p);
-
-        // should re-use the last allocation
-        let q = extism_alloc(&mut store, instance, 64 - i);
-        assert_eq!(p, q);
-        assert_eq!(extism_length(&mut store, instance, q), 64 - i);
-        extism_free(&mut store, instance, q);
     }
 
-    // 512 bytes, test block re-use + splitting
-    let p = extism_alloc(&mut store, instance, 512);
-    assert_eq!(extism_length(&mut store, instance, p), 512);
-    extism_free(&mut store, instance, p);
-
-    // 128 bytes, should be split off the 512 byte block
-    let q = extism_alloc(&mut store, instance, 128);
-    assert!(p <= q && q < p + 512);
-    assert_eq!(extism_length(&mut store, instance, q), 128);
-    extism_free(&mut store, instance, q);
-
-    // 128 bytes, same as above
-    let r = extism_alloc(&mut store, instance, 128);
-    assert!(p <= r && r < p + 512);
-    assert!(r > p);
-    assert_eq!(extism_length(&mut store, instance, q), 128);
-    extism_free(&mut store, instance, q);
-
     // 100 pages
     let p = extism_alloc(&mut store, instance, 6553600);
     assert!(p > 0);
@@ -297,3 +281,158 @@ fn test_load_input() {
     // Out of bounds should return 0
     assert_eq!(extism_input_load_u64(&mut store, instance, 123457), 0);
 }
+
+#[test]
+fn test_failed_quickcheck1() {
+    let (mut store, mut instance) = init_kernel_test();
+    let allocs = [
+        20622, 23162, 58594, 32421, 25928, 44611, 26318, 24455, 5798, 60202, 42126, 64928, 57832,
+        50888, 63256, 37562, 46334, 47985, 60836, 28132, 65535, 37800, 33150, 48768, 38457, 57249,
+        5734, 58587, 26294, 26653, 24519, 1,
+    ];
+
+    extism_reset(&mut store, &mut instance);
+    for a in allocs {
+        println!("Alloc: {a}");
+        let n = extism_alloc(&mut store, &mut instance, a);
+        if n == 0 {
+            continue;
+        }
+        assert_eq!(a, extism_length(&mut store, &mut instance, n));
+    }
+}
+
+#[test]
+fn test_failed_quickcheck2() {
+    let (mut store, mut instance) = init_kernel_test();
+    let allocs = [352054710, 1248853976, 2678441931, 14567928];
+
+    extism_reset(&mut store, &mut instance);
+    for a in allocs {
+        println!("Alloc: {a}");
+        let n = extism_alloc(&mut store, &mut instance, a);
+        if n == 0 {
+            continue;
+        }
+        assert_eq!(a, extism_length(&mut store, &mut instance, n));
+    }
+}
+
+quickcheck! {
+    fn check_alloc(amounts: Vec<u16>) -> bool {
+        let (mut store, mut instance) = init_kernel_test();
+        let instance = &mut instance;
+        for a in amounts {
+            let ptr = extism_alloc(&mut store, instance, a as u64);
+            if ptr == 0 || ptr == u64::MAX {
+                continue
+            }
+            if extism_length(&mut store, instance, ptr) != a as u64 {
+                return false
+            }
+        }
+
+        true
+    }
+}
+
+quickcheck! {
+    fn check_large_alloc(amounts: Vec<u32>) -> bool {
+        let (mut store, mut instance) = init_kernel_test();
+        let instance = &mut instance;
+        for a in amounts {
+            let ptr = extism_alloc(&mut store, instance, a as u64);
+            if ptr == 0 {
+                continue
+            }
+            let len = extism_length_unsafe(&mut store, instance, ptr);
+            if len != a as u64 {
+                return false
+            }
+        }
+
+        true
+    }
+}
+
+quickcheck! {
+    fn check_alloc_with_frees(amounts: Vec<u16>) -> bool {
+        let (mut store, mut instance) = init_kernel_test();
+        let instance = &mut instance;
+        let mut prev = 0;
+        for a in amounts {
+            let ptr = extism_alloc(&mut store, instance, a as u64);
+            if ptr == 0 {
+                continue
+            }
+            if extism_length(&mut store, instance, ptr) != a as u64  {
+                return false
+            }
+
+            if a % 2 == 0 {
+                extism_free(&mut store, instance, ptr);
+            } else if a % 3 == 0 {
+                extism_free(&mut store, instance, prev);
+            }
+
+            prev = ptr;
+        }
+
+        true
+    }
+}
+
+quickcheck! {
+    fn check_large_alloc_with_frees(amounts: Vec<u32>) -> bool {
+        let (mut store, mut instance) = init_kernel_test();
+        let instance = &mut instance;
+        let mut prev = 0;
+        for a in amounts {
+            let ptr = extism_alloc(&mut store, instance, a as u64);
+            if ptr == 0 || ptr == u64::MAX {
+                continue
+            }
+            if extism_length(&mut store, instance, ptr) != a as u64 {
+                return false
+            }
+            if a % 2 == 0 {
+                extism_free(&mut store, instance, ptr);
+            } else if a % 3 == 0 {
+                extism_free(&mut store, instance, prev);
+            }
+
+            prev = ptr;
+
+        }
+
+        true
+    }
+}
+
+quickcheck! {
+    fn check_alloc_with_load_and_store(amounts: Vec<u16>) -> bool {
+        use rand::Rng;
+        let mut rng = rand::rng();
+        let (mut store, mut instance) = init_kernel_test();
+        let instance = &mut instance;
+        for a in amounts {
+            let ptr = extism_alloc(&mut store, instance, a as u64);
+            if ptr == 0 || ptr == u64::MAX {
+                continue
+            }
+            if extism_length(&mut store, instance, ptr) != a as u64 {
+                return false
+            }
+
+            for _ in 0..16 {
+                let i = rng.random_range(ptr..ptr+a as u64);
+                extism_store_u8(&mut store, instance, i, i as u8);
+                if extism_load_u8(&mut store, instance, i as u64) != i as u8 {
+                    return false
+                }
+            }
+        }
+
+        true
+    }
+}

runtime/src/tests/mod.rs

@@ -1,2 +1,4 @@
+mod issues;
 mod kernel;
+mod pool;
 mod runtime;

runtime/src/tests/pool.rs

@@ -0,0 +1,63 @@
+use crate::*;
+use std::time::Duration;
+
+fn run_thread(p: Pool, i: u64) -> std::thread::JoinHandle<()> {
+    std::thread::spawn(move || {
+        std::thread::sleep(Duration::from_millis(i));
+        let s: String = p
+            .get(Duration::from_secs(1))
+            .unwrap()
+            .unwrap()
+            .call("count_vowels", "abc")
+            .unwrap();
+        println!("{s}");
+    })
+}
+
+fn init(max_instances: usize) -> Pool {
+    let data = include_bytes!("../../../wasm/code.wasm");
+    let plugin_builder =
+        extism::PluginBuilder::new(extism::Manifest::new([extism::Wasm::data(data)]))
+            .with_wasi(true);
+    PoolBuilder::new()
+        .with_max_instances(max_instances)
+        .build(move || plugin_builder.clone().build())
+}
+
+#[test]
+fn test_threads() {
+    for i in 1..=3 {
+        let pool = init(i);
+        let threads = vec![
+            run_thread(pool.clone(), 1000),
+            run_thread(pool.clone(), 1000),
+            run_thread(pool.clone(), 1000),
+            run_thread(pool.clone(), 1000),
+            run_thread(pool.clone(), 1000),
+            run_thread(pool.clone(), 1000),
+            run_thread(pool.clone(), 500),
+            run_thread(pool.clone(), 500),
+            run_thread(pool.clone(), 500),
+            run_thread(pool.clone(), 500),
+            run_thread(pool.clone(), 500),
+            run_thread(pool.clone(), 0),
+        ];
+
+        for t in threads {
+            t.join().unwrap();
+        }
+
+        assert!(pool.count() <= i);
+    }
+}
+
+#[test]
+fn test_exists() -> Result<(), Error> {
+    let pool = init(1);
+    let timeout = Duration::from_secs(1);
+    assert!(pool.function_exists("count_vowels", timeout)?);
+    assert!(pool.function_exists("count_vowels", timeout)?);
+    assert!(!pool.function_exists("not_existing", timeout)?);
+    assert!(!pool.function_exists("not_existing", timeout)?);
+    Ok(())
+}

runtime/src/tests/runtime.rs

@@ -1,5 +1,7 @@
+use extism_manifest::{HttpRequest, MemoryOptions};
+
 use crate::*;
-use std::{io::Write, time::Instant};
+use std::{collections::HashMap, io::Write, time::Instant};
 
 const WASM: &[u8] = include_bytes!("../../../wasm/code-functions.wasm");
 const WASM_NO_FUNCTIONS: &[u8] = include_bytes!("../../../wasm/code.wasm");
@@ -7,8 +9,10 @@ const WASM_LOOP: &[u8] = include_bytes!("../../../wasm/loop.wasm");
 const WASM_GLOBALS: &[u8] = include_bytes!("../../../wasm/globals.wasm");
 const WASM_REFLECT: &[u8] = include_bytes!("../../../wasm/reflect.wasm");
 const WASM_HTTP: &[u8] = include_bytes!("../../../wasm/http.wasm");
+const WASM_HTTP_HEADERS: &[u8] = include_bytes!("../../../wasm/http_headers.wasm");
+const WASM_FS: &[u8] = include_bytes!("../../../wasm/read_write.wasm");
 
-host_fn!(hello_world (a: String) -> String { Ok(a) });
+host_fn!(pub hello_world (a: String) -> String { Ok(a) });
 
 // Which is the same as:
 // fn hello_world(
@@ -39,11 +43,12 @@ pub struct Count {
 
 #[test]
 fn it_works() {
-    tracing_subscriber::fmt()
+    let log = tracing_subscriber::fmt()
         .with_ansi(false)
         .with_env_filter("extism=debug")
         .with_writer(std::fs::File::create("test.log").unwrap())
-        .init();
+        .try_init()
+        .is_ok();
 
     let wasm_start = Instant::now();
 
@@ -128,10 +133,7 @@ fn it_works() {
         .unwrap();
     let native_avg: std::time::Duration = native_sum / native_num_tests as u32;
 
-    println!(
-        "native function call (avg, N = {}): {:?}",
-        native_num_tests, native_avg
-    );
+    println!("native function call (avg, N = {native_num_tests}): {native_avg:?}");
 
     let num_tests = test_times.len();
     let sum: std::time::Duration = test_times
@@ -140,11 +142,13 @@ fn it_works() {
         .unwrap();
     let avg: std::time::Duration = sum / num_tests as u32;
 
-    println!("wasm function call (avg, N = {}): {:?}", num_tests, avg);
+    println!("wasm function call (avg, N = {num_tests}): {avg:?}");
 
     // Check that log file was written to
-    let meta = std::fs::metadata("test.log").unwrap();
-    assert!(meta.len() > 0);
+    if log {
+        let meta = std::fs::metadata("test.log").unwrap();
+        assert!(meta.len() > 0);
+    }
 }
 
 #[test]
@@ -205,7 +209,7 @@ fn test_cancel() {
         let _output: Result<&[u8], Error> = plugin.call("loop_forever", "abc123");
         let end = std::time::Instant::now();
         let time = end - start;
-        println!("Cancelled plugin ran for {:?}", time);
+        println!("Cancelled plugin ran for {time:?}");
     }
 }
 
@@ -235,7 +239,69 @@ fn test_timeout() {
     assert!(err == "timeout");
 }
 
-typed_plugin!(TestTypedPluginGenerics {
+#[test]
+fn test_fuel() {
+    let manifest = Manifest::new([extism_manifest::Wasm::data(WASM_LOOP)]);
+    let mut plugin = PluginBuilder::new(manifest)
+        .with_wasi(true)
+        .with_fuel_limit(1)
+        .build()
+        .unwrap();
+    for _ in 0..10001 {
+        let output: Result<&[u8], Error> = plugin.call("loop_forever", "abc123");
+        let err = output.unwrap_err().root_cause().to_string();
+        println!("Fuel limited plugin exited with error: {:?}", &err);
+        assert!(err.contains("fuel"));
+    }
+}
+
+#[test]
+fn test_fuel_consumption() {
+    let manifest = Manifest::new([extism_manifest::Wasm::data(WASM_LOOP)]);
+    let mut plugin = PluginBuilder::new(manifest)
+        .with_wasi(true)
+        .with_fuel_limit(10000)
+        .build()
+        .unwrap();
+
+    let output: Result<&[u8], Error> = plugin.call("loop_forever", "abc123");
+    assert!(output.is_err());
+
+    let fuel_consumed = plugin.fuel_consumed().unwrap();
+    println!("Fuel consumed: {fuel_consumed}");
+    assert!(fuel_consumed > 0);
+}
+
+#[test]
+#[cfg(feature = "http")]
+fn test_http_timeout() {
+    let f = Function::new(
+        "hello_world",
+        [PTR],
+        [PTR],
+        UserData::default(),
+        hello_world,
+    );
+
+    let manifest = Manifest::new([extism_manifest::Wasm::data(WASM_HTTP)])
+        .with_timeout(std::time::Duration::from_millis(1))
+        .with_allowed_host("www.extism.org");
+    let mut plugin = Plugin::new(manifest, [f], true).unwrap();
+
+    let start = std::time::Instant::now();
+    let output: Result<&[u8], Error> =
+        plugin.call("http_request", r#"{"url": "https://www.extism.org"}"#);
+    let end = std::time::Instant::now();
+    let time = end - start;
+    let err = output.unwrap_err().root_cause().to_string();
+    println!(
+        "Timed out plugin ran for {:?}, with error: {:?}",
+        time, &err
+    );
+    assert!(err == "timeout");
+}
+
+typed_plugin!(pub TestTypedPluginGenerics {
     count_vowels<T: FromBytes<'a>>(&str) -> T
 });
 
@@ -283,8 +349,10 @@ fn test_multiple_instantiations() {
 #[test]
 fn test_globals() {
     let mut plugin = Plugin::new(WASM_GLOBALS, [], true).unwrap();
-    for i in 0..1000 {
-        let Json(count) = plugin.call::<_, Json<Count>>("globals", "").unwrap();
+    for i in 0..100001 {
+        let Json(count) = plugin
+            .call_with_host_context::<_, Json<Count>, _>("globals", "", ())
+            .unwrap();
         assert_eq!(count.count, i);
     }
 }
@@ -302,6 +370,42 @@ fn test_toml_manifest() {
     assert_eq!(count.get("count").unwrap().as_i64().unwrap(), 1);
 }
 
+#[test]
+fn test_call_with_host_context() {
+    #[derive(Clone)]
+    struct Foo {
+        message: String,
+    }
+
+    let f = Function::new(
+        "host_reflect",
+        [PTR],
+        [PTR],
+        UserData::default(),
+        |current_plugin, _val, ret, _user_data: UserData<()>| {
+            let foo = current_plugin.host_context::<Foo>()?.clone();
+            let hnd = current_plugin.memory_new(foo.message)?;
+            ret[0] = current_plugin.memory_to_val(hnd);
+            Ok(())
+        },
+    );
+
+    let mut plugin = Plugin::new(WASM_REFLECT, [f], true).unwrap();
+
+    let message = "hello world";
+    let output: String = plugin
+        .call_with_host_context(
+            "reflect",
+            "anything, really",
+            Foo {
+                message: message.to_string(),
+            },
+        )
+        .unwrap();
+
+    assert_eq!(output, message);
+}
+
 #[test]
 fn test_fuzz_reflect_plugin() {
     // assert!(set_log_file("stdout", Some(log::Level::Trace)));
@@ -333,7 +437,7 @@ fn test_memory_max() {
     assert!(output.is_err());
 
     let err = output.unwrap_err().root_cause().to_string();
-    println!("{:?}", err);
+    println!("{err:?}");
     assert_eq!(err, "oom");
 
     // Should pass with memory.max set to a large enough number
@@ -357,7 +461,7 @@ fn hello_world_set_error(
     _user_data: UserData<()>,
 ) -> Result<(), Error> {
     plugin.set_error("TEST")?;
-    outputs[0] = inputs[0].clone();
+    outputs[0] = inputs[0];
     Ok(())
 }
 
@@ -396,7 +500,7 @@ fn test_extism_error() {
     let mut plugin = Plugin::new(&manifest, [f], true).unwrap();
     let output: Result<String, Error> = plugin.call("count_vowels", "a".repeat(1024));
     assert!(output.is_err());
-    println!("{:?}", output);
+    println!("{output:?}");
     assert_eq!(output.unwrap_err().root_cause().to_string(), "TEST");
 }
 
@@ -454,7 +558,7 @@ fn hello_world_user_data(
     let mut data = data.lock().unwrap();
     let s = _plugin.memory_get_val(&inputs[0])?;
     data.write_all(s)?;
-    outputs[0] = inputs[0].clone();
+    outputs[0] = inputs[0];
     Ok(())
 }
 
@@ -539,3 +643,198 @@ fn test_http_post() {
     assert!(!res.is_empty());
     assert!(res.contains(&data));
 }
+
+#[test]
+fn test_disable_cache() {
+    // Warmup cache
+    let _plugin: CountVowelsPlugin = PluginBuilder::new(WASM_NO_FUNCTIONS)
+        .build()
+        .unwrap()
+        .try_into()
+        .unwrap();
+
+    // This should be fast
+    let start = std::time::Instant::now();
+    let mut plugin: CountVowelsPlugin = PluginBuilder::new(WASM_NO_FUNCTIONS)
+        .build()
+        .unwrap()
+        .try_into()
+        .unwrap();
+    let t = std::time::Instant::now() - start;
+    let _output: Json<Count> = plugin.count_vowels("abc123").unwrap();
+
+    // This should take longer than the first run
+    let start = std::time::Instant::now();
+    let mut plugin: CountVowelsPlugin = PluginBuilder::new(WASM_NO_FUNCTIONS)
+        .with_cache_disabled()
+        .build()
+        .unwrap()
+        .try_into()
+        .unwrap();
+    let t1 = std::time::Instant::now() - start;
+    let _output: Json<Count> = plugin.count_vowels("abc123").unwrap();
+
+    assert!(t < t1);
+}
+
+#[test]
+fn test_manifest_ptr_len() {
+    let manifest = serde_json::json!({
+        "wasm" : [
+            {
+                "data" : {
+                    "ptr" : WASM_NO_FUNCTIONS.as_ptr() as u64,
+                    "len" : WASM_NO_FUNCTIONS.len()
+                }
+            }
+        ]
+    });
+    let mut plugin = Plugin::new(manifest.to_string().as_bytes(), [], true).unwrap();
+    let output = plugin.call("count_vowels", "abc123").unwrap();
+    let count: serde_json::Value = serde_json::from_slice(output).unwrap();
+    assert_eq!(count.get("count").unwrap().as_i64().unwrap(), 1);
+}
+
+#[test]
+fn test_no_vars() {
+    let data = br#"
+(module
+    (import "extism:host/env" "var_set" (func $var_set (param i64 i64)))
+    (import "extism:host/env" "input_offset" (func $input_offset (result i64)))
+    (func (export "test") (result i32)
+        (call $input_offset)
+        (call $input_offset)
+        (call $var_set)
+        (i32.const 0)
+    )
+)
+    "#;
+    let manifest = Manifest::new([Wasm::data(data)])
+        .with_memory_options(MemoryOptions::new().with_max_var_bytes(1));
+    let mut plugin = Plugin::new(manifest, [], true).unwrap();
+    let output: Result<(), Error> = plugin.call("test", b"A".repeat(1024));
+    assert!(output.is_err());
+    let output: Result<(), Error> = plugin.call("test", vec![]);
+    assert!(output.is_ok());
+}
+
+#[test]
+fn test_linking() {
+    let manifest = Manifest::new([
+        Wasm::Data {
+            data: br#"
+                (module
+                    (import "wasi_snapshot_preview1" "random_get" (func $random (param i32 i32) (result i32)))
+                    (import "extism:host/env" "alloc" (func $alloc (param i64) (result i64)))
+                    (import "extism:host/user" "hello" (func $hello))
+                    (global $counter (mut i32) (i32.const 0))
+                    (func $start (export "_start")
+                        (global.set $counter (i32.add (global.get $counter) (i32.const 1)))
+                    )
+                    (func (export "read_counter") (result i32)
+                        (global.get $counter)
+                    )
+                    (start $start)
+                )
+            "#.to_vec(),
+            meta: WasmMetadata {
+                name: Some("commander".to_string()),
+                hash: None,
+            },
+        },
+        Wasm::Data {
+            data: br#"
+                (module
+                    (import "commander" "_start" (func $commander_start))
+                    (import "commander" "read_counter" (func $commander_read_counter (result i32)))
+                    (import "extism:host/env" "store_u64" (func $store_u64 (param i64 i64)))
+                    (import "extism:host/env" "alloc" (func $alloc (param i64) (result i64)))
+                    (import "extism:host/user" "hello" (func $hello))
+                    (import "extism:host/env" "output_set" (func $output_set (param i64 i64)))
+                    (func (export "run") (result i32)
+                        (local $output i64)
+                        (local.set $output (call $alloc (i64.const 8)))
+
+                        (call $commander_start)
+                        (call $commander_start)
+                        (call $commander_start)
+                        (call $commander_start)
+                        (call $hello)
+                        (call $store_u64 (local.get $output) (i64.extend_i32_u (call $commander_read_counter)))
+                        (call $output_set (local.get $output) (i64.const 8))
+                        i32.const 0
+                    )
+                )
+            "#.to_vec(),
+            meta: WasmMetadata {
+                name: Some("main".to_string()),
+                hash: None,
+            },
+        },
+    ]);
+    let mut plugin = PluginBuilder::new(manifest)
+        .with_wasi(true)
+        .with_function("hello", [], [], UserData::new(()), |_, _, _, _| {
+            eprintln!("hello!");
+            Ok(())
+        })
+        .build()
+        .unwrap();
+
+    for _ in 0..5 {
+        assert_eq!(plugin.call::<&str, i64>("run", "Hello, world!").unwrap(), 1);
+    }
+}
+
+#[test]
+fn test_readonly_dirs() {
+    let wasm = Wasm::data(WASM_FS);
+    let manifest = Manifest::new([wasm])
+        .with_allowed_path("ro:src/tests/data".to_string(), "/data")
+        .with_config_key("path", "/data/data.txt");
+
+    let mut plugin = PluginBuilder::new(manifest)
+        .with_wasi(true)
+        .build()
+        .unwrap();
+
+    let res = plugin.call::<&str, &str>("try_read", "").unwrap();
+    assert_eq!(res, "hello world!");
+
+    let line = "hello world 2";
+    let res2 = plugin.call::<&str, &str>("try_write", line);
+    assert!(
+        res2.is_err(),
+        "Expected try_write to fail, but it succeeded."
+    );
+}
+
+#[test]
+#[cfg(feature = "http")]
+fn test_http_response_headers() {
+    let mut plugin = PluginBuilder::new(
+        Manifest::new([Wasm::data(WASM_HTTP_HEADERS)]).with_allowed_host("extism.org"),
+    )
+    .with_http_response_headers(true)
+    .build()
+    .unwrap();
+    let req = HttpRequest::new("https://extism.org");
+    let Json(res): Json<HashMap<String, String>> = plugin.call("http_get", Json(req)).unwrap();
+    println!("{res:?}");
+    assert_eq!(res["content-type"], "text/html; charset=utf-8");
+}
+
+#[test]
+#[cfg(feature = "http")]
+fn test_http_response_headers_disabled() {
+    let mut plugin = PluginBuilder::new(
+        Manifest::new([Wasm::data(WASM_HTTP_HEADERS)]).with_allowed_host("extism.org"),
+    )
+    .with_http_response_headers(false)
+    .build()
+    .unwrap();
+    let req = HttpRequest::new("https://extism.org");
+    let Json(res): Json<HashMap<String, String>> = plugin.call("http_get", Json(req)).unwrap();
+    println!("{res:?}");
+    assert!(res.is_empty());
+}

runtime/src/timer.rs

@@ -22,18 +22,18 @@ pub(crate) struct Timer {
 
 #[cfg(not(target_family = "windows"))]
 extern "C" fn cleanup_timer() {
-    let mut timer = match unsafe { TIMER.lock() } {
+    let mut timer = match TIMER.lock() {
         Ok(x) => x,
         Err(e) => e.into_inner(),
     };
     drop(timer.take());
 }
 
-static mut TIMER: std::sync::Mutex<Option<Timer>> = std::sync::Mutex::new(None);
+static TIMER: std::sync::Mutex<Option<Timer>> = std::sync::Mutex::new(None);
 
 impl Timer {
     pub(crate) fn tx() -> std::sync::mpsc::Sender<TimerAction> {
-        let mut timer = match unsafe { TIMER.lock() } {
+        let mut timer = match TIMER.lock() {
             Ok(x) => x,
             Err(e) => e.into_inner(),
         };
@@ -92,25 +92,39 @@ impl Timer {
             loop {
                 if plugins.is_empty() {
                     if let Ok(x) = rx.recv() {
-                        handle!(x)
+                        handle!(x);
                     }
                 }
 
-                plugins = plugins
-                    .into_iter()
-                    .filter(|(_k, (engine, end))| {
-                        if let Some(end) = end {
-                            let now = std::time::Instant::now();
-                            if end <= &now {
-                                engine.increment_epoch();
-                                return false;
+                let mut timeout: Option<std::time::Duration> = None;
+
+                plugins.retain(|_k, (engine, end)| {
+                    if let Some(end) = end {
+                        let now = std::time::Instant::now();
+                        if *end <= now {
+                            engine.increment_epoch();
+                            return false;
+                        } else {
+                            let time_left =
+                                (*end - now).saturating_sub(std::time::Duration::from_millis(1));
+                            if let Some(t) = &timeout {
+                                if time_left < *t {
+                                    timeout = Some(time_left);
+                                }
+                            } else {
+                                timeout = Some(time_left);
                             }
                         }
-                        true
-                    })
-                    .collect();
+                    }
 
-                for x in rx.try_iter() {
+                    true
+                });
+
+                if let Some(timeout) = timeout {
+                    if let Ok(x) = rx.recv_timeout(timeout) {
+                        handle!(x)
+                    }
+                } else if let Ok(x) = rx.recv() {
                     handle!(x)
                 }
             }