refactor!: make UserData more generic

- Adds `memory.max_var_bytes` to the manifest to limit the number of
bytes allowed to be stored in Extism vars - if `max_var_bytes` is set to
0 then vars are disabled.
- Adds some builder functions to `MemoryOptions` struct
- Sets the default var store size to 1mb
- Includes a test to make sure `var_set` returns an error when the limit
is reached

.github/actions/extism/action.yml

@@ -7,12 +7,24 @@ runs:
   steps:
     - name: Checkout sources
       uses: actions/checkout@v3
-    - name: Download libextism
-      uses: actions/download-artifact@v3
+    - name: Install Rust
+      uses: actions-rs/toolchain@v1
       with:
-        name: libextism-${{ matrix.os }}
+        toolchain: stable
+        override: true
+    - name: Cache Rust environment
+      uses: Swatinem/rust-cache@v1
+    - name: Cache libextism
+      id: cache-libextism
+      uses: actions/cache@v3
+      with:
+        path: target/**
+        key: ${{ runner.os }}-libextism-${{ hashFiles('runtime/**') }}-${{ hashFiles('manifest/**') }}-${{ hashFiles('libextism/**') }}
+    - name: Build
+      if: steps.cache-libextism.outputs.cache-hit != 'true'
+      shell: bash
+      run: cargo build --release -p libextism
     - name: Install extism shared library
       shell: bash
       run: |
-        sudo cp libextism.* /usr/local/lib
-        sudo cp runtime/extism.h /usr/local/include
+        sudo make install

.github/workflows/ci.yml

@@ -1,11 +1,20 @@
-on: [pull_request, workflow_dispatch]
+on: 
+  pull_request:
+    paths:
+      - .github/actions/extism/**
+      - .github/workflows/ci-rust.yml
+      - convert/**
+      - manifest/**
+      - runtime/**
+      - rust/**
+      - libextism/**
+  workflow_dispatch:
 
-name: CI
+name: Rust CI
 
 env:
-  RUNTIME_MANIFEST: runtime/Cargo.toml
-  RUNTIME_CRATE: extism-runtime
-  RUST_SDK_CRATE: extism
+  RUNTIME_CRATE: extism
+  LIBEXTISM_CRATE: libextism
 
 jobs:
   lib:
@@ -31,17 +40,17 @@ jobs:
         uses: actions/cache@v3
         with:
           path: target/release/libextism.*
-          key: ${{ runner.os }}-libextism-${{ hashFiles('runtime/**') }}-${{ hashFiles('manifest/**') }}
+          key: ${{ runner.os }}-libextism-${{ hashFiles('runtime/**') }}-${{ hashFiles('manifest/**') }}-${{ hashFiles('convert/**') }}
       - name: Cache target
         id: cache-target
         uses: actions/cache@v3
         with:
           path: target/**
-          key: ${{ runner.os }}-target-${{ env.GITHUB_SHA }}
+          key: ${{ runner.os }}-target-${{ github.sha }}
       - name: Build
         if: steps.cache-libextism.outputs.cache-hit != 'true'
         shell: bash
-        run: cargo build --release -p ${{ env.RUNTIME_CRATE }}
+        run: cargo build --release -p ${{ env.LIBEXTISM_CRATE }}
       - name: Upload artifact
         uses: actions/upload-artifact@v3
         with:
@@ -71,301 +80,40 @@ jobs:
         uses: actions/cache@v3
         with:
           path: target/**
-          key: ${{ runner.os }}-target-${{ env.GITHUB_SHA }}
+          key: ${{ runner.os }}-target-${{ github.sha }}
       - name: Format
-        run: cargo fmt --check -p ${{ env.RUNTIME_CRATE }}
+        run: cargo fmt --check
       - name: Lint
-        run: cargo clippy --release --all-features --no-deps -p ${{ env.RUNTIME_CRATE }}
+        run: cargo clippy --all --release --all-features --no-deps -- -D "clippy::all"
       - name: Test
-        run: cargo test --all-features --release -p ${{ env.RUNTIME_CRATE }}
-
-  rust:
-    name: Rust
-    needs: lib
+        run: cargo test --release
+      - name: Test all features
+        run: cargo test --all-features --release
+      - name: Test no features
+        run: cargo test --no-default-features --release
+  bench:
+    name: Benchmarking
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        os: [ubuntu-latest]
         rust:
           - stable
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
-      - uses: ./.github/actions/extism
-      - name: Test Rust Host SDK
-        run: LD_LIBRARY_PATH=/usr/local/lib cargo test --release -p ${{ env.RUST_SDK_CRATE }}
-
-  elixir:
-    name: Elixir
-    needs: lib
-    runs-on: ${{ matrix.os }}
-    env:
-      MIX_ENV: test
-    strategy:
-      matrix:
-        os: [ubuntu-latest, macos-latest]
-        rust:
-          - stable
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
-      - uses: ./.github/actions/extism
-      - name: Setup Elixir Host SDK
-        if: ${{ runner.os != 'macOS' }}
-        uses: erlef/setup-beam@v1
+      - name: Install Rust
+        uses: actions-rs/toolchain@v1
         with:
-          experimental-otp: true
-          otp-version: '25.0.4'
-          elixir-version: '1.14.0'
-
-      - name: Test Elixir Host SDK
-        if: ${{ runner.os != 'macOS' }}
-        run: |
-         cd elixir
-         LD_LIBRARY_PATH=/usr/local/lib mix do deps.get, test
-
-  go:
-    name: Go
-    needs: lib
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest, macos-latest]
-        rust:
-          - stable
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
-      - uses: ./.github/actions/extism
-      - name: Setup Go env
-        uses: actions/setup-go@v3
-
-      - name: Test Go Host SDK
-        run: |
-          go version
-          cd go
-          LD_LIBRARY_PATH=/usr/local/lib go run main.go
-          LD_LIBRARY_PATH=/usr/local/lib go test
-
-  python:
-    name: Python
-    needs: lib
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest, macos-latest]
-        rust:
-          - stable
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
-      - uses: ./.github/actions/extism
-      - name: Setup Python env
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.9"
-          check-latest: true
-      - name: Install Poetry
-        uses: snok/install-poetry@v1
-      - name: Test Python Host SDK
-        run: |
-          cd python
-          poetry install
-          poetry run python example.py
-          poetry run python -m unittest discover
-
-  ruby:
-    name: Ruby
-    needs: lib
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest, macos-latest]
-        rust:
-          - stable
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
-      - uses: ./.github/actions/extism
-      - name: Setup Ruby env
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: "3.0"
-
-      - name: Test Ruby Host SDK
-        run: |
-          cd ruby
-          bundle install
-          ruby example.rb
-          rake test
-
-  node:
-    name: Node
-    needs: lib
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest, macos-latest]
-        rust:
-          - stable
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
-      - uses: ./.github/actions/extism
-      - name: Setup Node env
-        uses: actions/setup-node@v3
-        with:
-          node-version: 18
-
-      - name: Test Node Host SDK
-        run: |
-          cd node
-          npm i
-          LD_LIBRARY_PATH=/usr/local/lib npm run build
-          LD_LIBRARY_PATH=/usr/local/lib npm run example
-          LD_LIBRARY_PATH=/usr/local/lib npm run test
-
-  ocaml:
-    name: OCaml
-    needs: lib
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest, macos-latest]
-        rust:
-          - stable
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
-      - uses: ./.github/actions/extism
-      - name: Setup OCaml env
-        uses: ocaml/setup-ocaml@v2
-        with:
-          ocaml-compiler: ocaml-base-compiler.5.0.0~beta1
-      - name: Cache OCaml
-        id: cache-ocaml
+          toolchain: stable
+          override: true
+      - name: Cache Rust environment
+        uses: Swatinem/rust-cache@v1
+      - name: Cache target
+        id: cache-target
         uses: actions/cache@v3
         with:
-          path: _build
-          key: ${{ runner.os }}-ocaml-${{ hashFiles('ocaml/lib/**') }}-${{ hashFiles('ocaml/bin/**') }}-${{ hashFiles('dune-project') }}
-      - name: Build OCaml Host SDK
-        if: steps.cache-ocaml.outputs.cache-hit != 'true'
-        run: |
-          opam install -y --deps-only .
-          cd ocaml
-          LD_LIBRARY_PATH=/usr/local/lib opam exec -- dune build
-      - name: Test OCaml Host SDK
-        run: |
-          opam install -y --deps-only .
-          cd ocaml
-          LD_LIBRARY_PATH=/usr/local/lib opam exec -- dune exec ./bin/main.exe
-          LD_LIBRARY_PATH=/usr/local/lib opam exec -- dune runtest
-
-  haskell:
-    name: Haskell
-    needs: lib
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest, macos-latest]
-        rust:
-          - stable
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
-      - uses: ./.github/actions/extism
-      - name: Setup Haskell env
-        uses: haskell/actions/setup@v2
-        with:
-          enable-stack: true
-          stack-version: "latest"
-      - name: Cache Haskell
-        id: cache-haskell
-        uses: actions/cache@v3
-        with:
-          path: .stack-work
-          key: ${{ runner.os }}-haskell-${{ hashFiles('haskell/**') }}
-      - name: Build Haskell Host SDK
-        if: steps.cache-haskell.outputs.cache-hit != 'true'
-        run: |
-          cd haskell
-          LD_LIBRARY_PATH=/usr/local/lib stack build
-      - name: Test Haskell SDK
-        run: |
-          cd haskell
-          LD_LIBRARY_PATH=/usr/local/lib stack test
-
-  php:
-    name: PHP
-    needs: lib
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest, macos-latest]
-        rust:
-          - stable
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
-      - uses: ./.github/actions/extism
-      - name: Setup PHP env
-        uses: shivammathur/setup-php@v2
-        with:
-          php-version: "8.1"
-          extensions: ffi
-          tools: composer
-        env:
-          fail-fast: true
-
-      - name: Test PHP SDK
-        run: |
-          cd php/example
-          composer install
-          php index.php
-
-  cpp:
-    name: C++
-    needs: lib
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest, macos-latest]
-        rust:
-          - stable
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
-      - uses: ./.github/actions/extism
-      - name: Install C++ SDK deps
-        if: ${{ matrix.os == 'macos-latest' }}
-        run: |
-          brew install jsoncpp googletest pkg-config
-      - name: Install C++ SDK deps
-        if: ${{ matrix.os == 'ubuntu-latest' }}
-        run: |
-          sudo apt-get install g++ libjsoncpp-dev libgtest-dev pkg-config
-      - name: Run C++ tests
-        run: |
-          cd cpp
-          LD_LIBRARY_PATH=/usr/local/lib make example
-          LD_LIBRARY_PATH=/usr/local/lib make test
-
-  sdk_api_coverage:
-    name: SDK API Coverage Report
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
-      - name: Setup Python env
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.9"
-          check-latest: true
-      - name: Install dependencies
-        run: |
-          sudo apt-get install ripgrep
-          pip3 install pycparser
-      - name: Run coverage script
-        id: coverage
-        run: |
-          python scripts/sdk_coverage.py
+          path: target/**
+          key: ${{ runner.os }}-target-${{ github.sha }}
+      - run: cargo install cargo-criterion
+      - run: cargo criterion

.github/workflows/kernel.yml

@@ -0,0 +1,48 @@
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - kernel/**
+
+name: Kernel
+
+jobs:
+  kernel:
+    name: Build extism-runtime.wasm
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          profile: minimal
+          target: wasm32-unknown-unknown
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Install deps
+        run: |
+          sudo apt install wabt --yes
+
+      - name: Build kernel
+        shell: bash
+        continue-on-error: true
+        run: |
+          cd kernel
+          sh build.sh
+          git diff --exit-code
+          export GIT_EXIT_CODE=$?
+
+      - uses: peter-evans/create-pull-request@v5
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        if: ${{ env.GIT_EXIT_CODE }} != 0
+        with:
+          author: "zshipko <zshipko@users.noreply.github.com>"
+          title: "update(kernel): extism-runtime.wasm in ${{ github.event.pull_request.head.ref }}"
+          body: "Automated PR to update `runtime/src/extism-runtime.wasm` in PR #${{ github.event.number }}"
+          base: "${{ github.event.pull_request.head.ref }}"
+          branch: "update-kernel--${{ github.event.pull_request.head.ref }}"
+          commit-message: "update(kernel): extism-runtime.wasm in ${{ github.event.pull_request.head.ref }}"
+          delete-branch: true

.github/workflows/release-dotnet-native.yaml

@@ -0,0 +1,70 @@
+on:
+  workflow_dispatch:
+
+name: Release .NET Native NuGet Packages
+
+jobs:
+  release-runtimes:
+    name: release-dotnet-native
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          filter: tree:0
+
+      - name: Setup .NET Core SDK
+        uses: actions/setup-dotnet@v3.0.3
+        with:
+           dotnet-version: 7.x
+      - uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: release.yml
+          name: release-artifacts
+      - name: Extract Archive
+        run: |      
+          extract_archive() {
+              # Check if both pattern and destination are provided
+              if [ -z "$1" ] || [ -z "$2" ]; then
+                  echo "Usage: $0 <filename_pattern> <destination_directory>"
+                  return 1
+              fi
+          
+              # Set the filename pattern and destination directory
+              filename_pattern="$1"
+              destination_directory="$2"
+          
+              # Find the archive file with the specified pattern
+              archive_file=$(ls $filename_pattern 2>/dev/null)
+          
+              # Check if an archive file is found
+              if [ -n "$archive_file" ]; then
+                  echo "Found archive file: $archive_file"
+          
+                  # Create the destination directory if it doesn't exist
+                  mkdir -p "$destination_directory"
+          
+                  # Extract the archive to the specified destination
+                  tar -xzvf "$archive_file" -C "$destination_directory"
+          
+                  echo "Extraction complete. Contents placed in: $destination_directory"
+              else
+                  echo "No matching archive file found with the pattern: $filename_pattern"
+              fi
+          }
+          
+          extract_archive "libextism-x86_64-pc-windows-msvc-*.tar.gz" "nuget/runtimes/win-x64/native/"
+          extract_archive "libextism-aarch64-apple-darwin-*.tar.gz" "nuget/runtimes/osx-arm64/native/"
+          extract_archive "libextism-x86_64-apple-darwin-*.tar.gz" "nuget/runtimes/osx-x64/native/"
+          extract_archive "libextism-x86_64-unknown-linux-gnu-*.tar.gz" "nuget/runtimes/linux-x64/native/"
+          extract_archive "libextism-aarch64-unknown-linux-gnu-*.tar.gz" "nuget/runtimes/linux-arm64/native/"
+          extract_archive "libextism-aarch64-unknown-linux-musl-*.tar.gz" "nuget/runtimes/linux-musl-arm64/native/"
+
+      - name: Pack NuGet packages
+        run: |
+          find ./nuget -type f -name "*.csproj" -exec dotnet pack {} -o release-artifacts \;
+
+      - name: Publish NuGet packages
+        run: |
+          dotnet nuget push --source https://api.nuget.org/v3/index.json ./release-artifacts/*.nupkg --api-key ${{ secrets.NUGET_API_KEY }}

.github/workflows/release-elixir.yaml

@@ -1,38 +0,0 @@
-on:
-  workflow_dispatch:
-
-name: Release Elixir SDK
-
-jobs:
-  release-sdks:
-    name: release-elixir
-    runs-on: ubuntu-latest
-    env:
-      HEX_API_KEY: ${{ secrets.HEX_PM_API_TOKEN }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Setup Rust ENV (to compile NIF)
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: minimal
-          override: true
-          target: ${{ matrix.target }}
-
-      - name: Setup Elixir Env
-        uses: erlef/setup-beam@v1
-        with:
-          experimental-otp: true
-          otp-version: '25.0.4'
-          elixir-version: '1.14.0'
-
-      - name: Publish Elixir Host SDK to hex.pm
-        env:
-          HEX_API_KEY: ${{ secrets.HEX_PM_API_TOKEN }}
-        run: |
-          cd elixir
-          cp ../LICENSE .
-          make publish
-  

.github/workflows/release-node.yaml

@@ -1,30 +0,0 @@
-on:
-  workflow_dispatch:
-
-name: Release Node SDK
-
-jobs:
-  release-sdks:
-    name: release-node
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Setup Node Env
-        uses: actions/setup-node@v3
-        with:
-          node-version: 16
-          registry-url: "https://registry.npmjs.org"
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_API_TOKEN }}
-          CI: true
-
-      - name: Release Node Host SDK
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_API_TOKEN }}
-          CI: true
-        run: |
-          cd node
-          make publish
-  

.github/workflows/release-python.yaml

@@ -1,8 +1,9 @@
-on:
-  workflow_dispatch:
-
 name: Release Python SDK
 
+on:
+  release:
+    types: [published, edited]
+
 jobs:
   release-sdks:
     name: release-python
@@ -11,28 +12,30 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Setup Python env
-        uses: actions/setup-python@v4
+      - name: Install poetry
+        run: pipx install poetry
+
+      - uses: actions/setup-python@v4
         with:
-          python-version: "3.9"
-          check-latest: true
-      - name: Install Poetry
-        uses: abatilo/actions-poetry@v2
-      - name: Build Python Env
+          python-version: '3.10'
+
+      - name: install twine
         run: |
-          cd python
-          cp ../LICENSE .
-          cp ../README.md .
-          make clean
-          make prepare
-          poetry build
+          pip install twine
 
-      - name: Release Python Host SDK
-        uses: pypa/gh-action-pypi-publish@release/v1
+      - name: download release
+        run: |
+          tag='${{ github.ref }}'
+          tag="${tag/refs\/tags\//}"
+          mkdir dist
+          cd dist
+          gh release download "$tag" -p 'extism_sys-*'
         env:
-          INPUT_VERIFY_METADATA: false
-        with:
-          user: ${{ secrets.PYPI_API_USER }}
-          password: ${{ secrets.PYPI_API_TOKEN }}
-          packages_dir: python/dist/
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: upload release
+        run: |
+          twine upload dist/*
+        env:
+          TWINE_USERNAME: ${{ secrets.PYPI_API_USER }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/release-ruby.yaml

@@ -1,27 +0,0 @@
-on:
-  workflow_dispatch:
-
-name: Release Ruby SDK
-
-jobs:
-  release-sdks:
-    name: release-ruby
-    runs-on: ubuntu-latest
-    env:
-      RUBYGEMS_API_KEY: ${{ secrets.RUBYGEMS_API_TOKEN }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Setup Ruby Env
-        uses: actions/setup-ruby@v1
-        with:
-          ruby-version: '3.1' # Version range or exact version of a Ruby version to use, using semvers version range syntax.
-
-      - name: Publish Ruby Gem
-        env:
-          RUBYGEMS_API_KEY: ${{ secrets.RUBYGEMS_API_TOKEN }}
-        run: |
-          cd ruby
-          make publish
-      

.github/workflows/release-rust.yaml

@@ -1,17 +1,34 @@
 on:
+  release:
+    types: [published, edited]
   workflow_dispatch:
 
-name: Release Rust SDK
+name: Release Runtime/Rust SDK
 
 jobs:
-  release-sdks:
+  release-runtime:
     name: release-rust
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      
-      - name: Setup Rust Env
+        with:
+          ref: '${{ github.ref }}'
+
+      - name: Set version
+        shell: bash
+        run: |
+          version="${{ github.ref }}"
+          if [[ "$version" = "refs/heads/main" ]]; then
+            version="0.0.0-dev"
+          else
+            version="${version/refs\/tags\/v/}"
+          fi
+          sed -i -e "s/0.0.0+replaced-by-ci/${version}/g" Cargo.toml
+          pyproject="$(cat extism-maturin/pyproject.toml)"
+          <<<"$pyproject" >extism-maturin/pyproject.toml sed -e 's/^version = "0.0.0.replaced-by-ci"/version = "'"$version"'"/g'
+
+      - name: Setup Rust env
         uses: actions-rs/toolchain@v1
         with:
           toolchain: stable
@@ -19,17 +36,50 @@ jobs:
           override: true
           target: ${{ matrix.target }}
 
-      - name: Release Rust Host SDK
+      - name: Release Rust convert-macros Crate
         env:
           CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_API_TOKEN }}
         run: |
-          # order of crate publication matter: manifest, runtime, rust
+          version=$(cargo metadata --format-version=1 | jq -r '.packages[] | select(.name == "extism") | .version')
 
-          cargo publish --manifest-path manifest/Cargo.toml
-          # allow for crates.io to update so dependant crates can locate extism-manifest
-          sleep 5 
+          if ! &>/dev/null curl -sLIf https://crates.io/api/v1/crates/extism-convert-macros/${version}/download; then
+            cargo publish --manifest-path convert-macros/Cargo.toml --allow-dirty
+          else
+            echo "already published ${version}"
+          fi
 
-          cargo publish --manifest-path runtime/Cargo.toml --no-verify
-          cargo publish --manifest-path rust/Cargo.toml
-      
-  
+      - name: Release Rust Convert Crate
+        env:
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_API_TOKEN }}
+        run: |
+          version=$(cargo metadata --format-version=1 | jq -r '.packages[] | select(.name == "extism") | .version')
+
+          if ! &>/dev/null curl -sLIf https://crates.io/api/v1/crates/extism-convert/${version}/download; then
+            cargo publish --manifest-path convert/Cargo.toml --allow-dirty
+          else
+            echo "already published ${version}"
+          fi
+
+      - name: Release Rust Manifest Crate
+        env:
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_API_TOKEN }}
+        run: |
+          version=$(cargo metadata --format-version=1 | jq -r '.packages[] | select(.name == "extism") | .version')
+
+          if ! &>/dev/null curl -sLIf https://crates.io/api/v1/crates/extism-manifest/${version}/download; then
+            cargo publish --manifest-path manifest/Cargo.toml --allow-dirty
+          else
+            echo "already published ${version}"
+          fi
+
+      - name: Release Runtime
+        env:
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_API_TOKEN }}
+        run: |
+          version=$(cargo metadata --format-version=1 | jq -r '.packages[] | select(.name == "extism") | .version')
+
+          if ! &>/dev/null curl -sLIf https://crates.io/api/v1/crates/extism/${version}/download; then
+            cargo publish --manifest-path runtime/Cargo.toml --allow-dirty
+          else
+            echo "already published ${version}"
+          fi

.github/workflows/release.yml

@@ -1,97 +1,102 @@
 on:
-  release:
-    types: [created]
   workflow_dispatch:
+  push:
+    branches: [ main, "v*" ]
+    tags:
+      - 'v*'
 
 name: Release
 
 env:
   RUNTIME_MANIFEST: runtime/Cargo.toml
-  RUNTIME_CRATE: extism-runtime
+  RUNTIME_CRATE: libextism
   RUSTFLAGS: -C target-feature=-crt-static
   ARTIFACT_DIR: release-artifacts
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
-  release-linux:
-    name: linux
-    runs-on: ubuntu-latest
+  release:
+    name: ${{ matrix.os }} ${{ matrix.target }}
+    runs-on: ${{ matrix.os }}-latest
     strategy:
       matrix:
-        target:
-          [
-            aarch64-unknown-linux-gnu,
-            aarch64-unknown-linux-musl,
-            x86_64-unknown-linux-gnu,
-          ]
-          # i686-unknown-linux-gnu,
-    if: always()
+        include:
+          - os: 'macos'
+            target: 'x86_64-apple-darwin'
+            artifact: 'libextism.dylib'
+            static-artifact: 'libextism.a'
+            static-dll-artifact: ''
+            pc-in: 'extism.pc.in'
+            static-pc-in: 'extism-static.pc.in'
+          - os: 'macos'
+            target: 'aarch64-apple-darwin'
+            artifact: 'libextism.dylib'
+            static-artifact: 'libextism.a'
+            static-dll-artifact: ''
+            pc-in: 'extism.pc.in'
+            static-pc-in: 'extism-static.pc.in'
+          - os: 'ubuntu'
+            target: 'aarch64-unknown-linux-gnu'
+            artifact: 'libextism.so'
+            static-artifact: 'libextism.a'
+            static-dll-artifact: ''
+            pc-in: 'extism.pc.in'
+            static-pc-in: 'extism-static.pc.in'
+          - os: 'ubuntu'
+            target: 'aarch64-unknown-linux-musl'
+            artifact: 'libextism.so'
+            static-artifact: 'libextism.a'
+            static-dll-artifact: ''
+            pc-in: 'extism.pc.in'
+            static-pc-in: 'extism-static.pc.in'
+          - os: 'ubuntu'
+            target: 'x86_64-unknown-linux-gnu'
+            artifact: 'libextism.so'
+            static-artifact: 'libextism.a'
+            static-dll-artifact: ''
+            pc-in: 'extism.pc.in'
+            static-pc-in: 'extism-static.pc.in'
+          - os: 'ubuntu'
+            target: 'x86_64-unknown-linux-musl'
+            artifact: ''
+            static-artifact: 'libextism.a'
+            static-dll-artifact: ''
+            pc-in: ''
+            static-pc-in: 'extism-static.pc.in'
+          - os: 'windows'
+            target: 'x86_64-pc-windows-gnu'
+            artifact: 'extism.dll'
+            static-artifact: 'libextism.a'
+            static-dll-artifact: 'libextism.dll.a'
+            pc-in: 'extism.pc.in'
+            static-pc-in: 'extism-static.pc.in'
+          - os: 'windows'
+            target: 'x86_64-pc-windows-msvc'
+            artifact: 'extism.dll'
+            static-artifact: 'extism.lib'
+            static-dll-artifact: 'extism.dll.lib'
+            pc-in: ''
+            static-pc-in: ''
+
     steps:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Install Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: minimal
-          override: true
-          target: ${{ matrix.target }}
-
-      - name: Build Target (${{ matrix.target }})
-        uses: actions-rs/cargo@v1
-        with:
-          use-cross: true
-          command: build
-          args: --release --target ${{ matrix.target }} -p ${{ env.RUNTIME_CRATE }}
-
-      - name: Prepare Artifact
+      - name: Set version
+        shell: bash
         run: |
-          EXT=so
-          SRC_DIR=target/${{ matrix.target }}/release
-          DEST_DIR=${{ env.ARTIFACT_DIR }}
-          RELEASE_NAME=libextism-${{ matrix.target }}-${{ github.ref_name }}
-          ARCHIVE=${RELEASE_NAME}.tar.gz
-          CHECKSUM=${RELEASE_NAME}.checksum.txt
-
-          # compress the shared library & create checksum
-          cp runtime/extism.h ${SRC_DIR}
-          cp LICENSE ${SRC_DIR}
-          tar -C ${SRC_DIR} -czvf ${ARCHIVE} libextism.${EXT} extism.h
-          ls -ll ${ARCHIVE}
-          shasum -a 256 ${ARCHIVE} > ${CHECKSUM}
-
-          # copy archive and checksum into release artifact directory
-          mkdir -p ${DEST_DIR}
-          cp ${ARCHIVE} ${DEST_DIR}
-          cp ${CHECKSUM} ${DEST_DIR}
-
-          ls ${DEST_DIR}
-
-      - name: Upload Artifact to Summary
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ env.ARTIFACT_DIR }}
-          path: |
-            *.tar.gz
-            *.txt
-
-      - name: Upload Artifact to Release
-        uses: softprops/action-gh-release@v1
-        with:
-          files: |
-            *.tar.gz
-            *.txt
-
-  release-macos:
-    name: macos
-    runs-on: macos-latest
-    strategy:
-      matrix:
-        target: [x86_64-apple-darwin, aarch64-apple-darwin]
-    if: always()
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
+          version="${{ github.ref }}"
+          if [[ "$version" = "refs/heads/main" ]]; then
+            version="0.0.0-dev"
+          else
+            version="${version/refs\/tags\/v/}"
+          fi
+          sed -i -e "s/0.0.0+replaced-by-ci/${version}/g" Cargo.toml
+          pyproject="$(cat extism-maturin/pyproject.toml)"
+          <<<"$pyproject" >extism-maturin/pyproject.toml sed -e 's/^version = "0.0.0.replaced-by-ci"/version = "'"$version"'"/g'
 
       - name: Install Rust
         uses: actions-rs/toolchain@v1
@@ -101,84 +106,55 @@ jobs:
           override: true
           target: ${{ matrix.target }}
 
-      - name: Build Target (${{ matrix.target }})
+      - uses: Swatinem/rust-cache@v2
+        with:
+          prefix-key: "${{matrix.os}}-${{matrix.target}}"
+          save-if: ${{ github.ref == 'refs/heads/main' }}
+          cache-on-failure: "true"
+
+      - name: Build Target (${{ matrix.os }} ${{ matrix.target }})
         uses: actions-rs/cargo@v1
         with:
-          use-cross: true
+          use-cross: ${{ matrix.os != 'windows' }}
           command: build
           args: --release --target ${{ matrix.target }} -p ${{ env.RUNTIME_CRATE }}
 
-      - name: Prepare Artifact
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        # maturin's cffi integration struggles with gnu headers on windows.
+        # there's partial work towards fixing this in `extism-maturin/build.rs`, but it's
+        # not sufficient to get it to work. omit it for now!
+        if: ${{ matrix.target != 'x86_64-pc-windows-gnu' && matrix.target != 'aarch64-unknown-linux-gnu' }}
+        with:
+          target: ${{ matrix.target }}
+          args: --release --out dist --find-interpreter -m extism-maturin/Cargo.toml
+          sccache: 'true'
+          manylinux: auto
+
+      - name: Build GNU Linux wheels
+        uses: PyO3/maturin-action@v1
+        # One of our deps, "ring", needs a newer sysroot than what "manylinux: auto" provides.
+        if: ${{ matrix.target == 'aarch64-unknown-linux-gnu' }}
+        with:
+          target: ${{ matrix.target }}
+          args: --release --out dist --find-interpreter -m extism-maturin/Cargo.toml
+          sccache: 'true'
+          manylinux: 2_28
+
+      - name: Add pkg-config files except on MSVC
+        if: ${{ matrix.target != 'x86_64-pc-windows-msvc' }}
+        shell: bash
         run: |
-          EXT=dylib
           SRC_DIR=target/${{ matrix.target }}/release
-          DEST_DIR=${{ env.ARTIFACT_DIR }}
-          RELEASE_NAME=libextism-${{ matrix.target }}-${{ github.ref_name }}
-          ARCHIVE=${RELEASE_NAME}.tar.gz
-          CHECKSUM=${RELEASE_NAME}.checksum.txt
-
-          # compress the shared library & create checksum
-          cp runtime/extism.h ${SRC_DIR}
-          cp LICENSE ${SRC_DIR}
-          tar -C ${SRC_DIR} -czvf ${ARCHIVE} libextism.${EXT} extism.h
-          ls -ll ${ARCHIVE}
-          shasum -a 256 ${ARCHIVE} > ${CHECKSUM}
-
-          # copy archive and checksum into release artifact directory
-          mkdir -p ${DEST_DIR}
-          cp ${ARCHIVE} ${DEST_DIR}
-          cp ${CHECKSUM} ${DEST_DIR}
-
-          ls ${DEST_DIR}
-
-      - name: Upload Artifact to Summary
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ env.ARTIFACT_DIR }}
-          path: |
-            *.tar.gz
-            *.txt
-
-      - name: Upload Artifact to Release
-        uses: softprops/action-gh-release@v1
-        with:
-          files: |
-            *.tar.gz
-            *.txt
-
-  release-windows:
-    name: windows
-    runs-on: windows-latest
-    strategy:
-      matrix:
-        target:
-          [x86_64-pc-windows-gnu, x86_64-pc-windows-msvc]
-          # i686-pc-windows-gnu,
-          # i686-pc-windows-msvc,
-          # aarch64-pc-windows-msvc
-    if: always()
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Install Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: minimal
-          override: true
-          target: ${{ matrix.target }}
-
-      - name: Build Target (${{ matrix.target }})
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --release --target ${{ matrix.target }} -p ${{ env.RUNTIME_CRATE }}
+          cp libextism/extism*.pc.in ${SRC_DIR}
 
       - name: Prepare Artifact
         shell: bash
         run: |
-          EXT=dll
           SRC_DIR=target/${{ matrix.target }}/release
           DEST_DIR=${{ env.ARTIFACT_DIR }}
           RELEASE_NAME=libextism-${{ matrix.target }}-${{ github.ref_name }}
@@ -188,29 +164,63 @@ jobs:
           # compress the shared library & create checksum
           cp runtime/extism.h ${SRC_DIR}
           cp LICENSE ${SRC_DIR}
-          tar -C ${SRC_DIR} -czvf ${ARCHIVE} extism.${EXT} extism.h
+          tar -C ${SRC_DIR} -czvf ${ARCHIVE} extism.h \
+            ${{ matrix.artifact }} ${{ matrix.static-artifact }}  \
+            ${{ matrix.pc-in }} ${{ matrix.static-pc-in }} \
+            ${{ matrix.static-dll-artifact }}
           ls -ll ${ARCHIVE}
 
-          certutil -hashfile ${ARCHIVE} SHA256 >${CHECKSUM}
+          if &>/dev/null which shasum; then
+            shasum -a 256 ${ARCHIVE} > ${CHECKSUM}
+          else
+            # windows doesn't have shasum available, so we use certutil instead.
+            certutil -hashfile ${ARCHIVE} SHA256 >${CHECKSUM}
+          fi
 
           # copy archive and checksum into release artifact directory
           mkdir -p ${DEST_DIR}
           cp ${ARCHIVE} ${DEST_DIR}
           cp ${CHECKSUM} ${DEST_DIR}
 
-          ls ${DEST_DIR}
+          # copy any built wheels.
+          if [ -e dist/*.whl ]; then
+            cp dist/*.whl ${DEST_DIR}
+          fi
+
+          ls -ll ${DEST_DIR}
 
       - name: Upload Artifact to Summary
         uses: actions/upload-artifact@v3
         with:
           name: ${{ env.ARTIFACT_DIR }}
-          path: |
-            *.tar.gz
-            *.txt
+          path: ${{ env.ARTIFACT_DIR }}
 
-      - name: Upload Artifact to Release
+      - name: Upload Artifact to Draft Release
         uses: softprops/action-gh-release@v1
         with:
+          draft: true
+          files: |
+            ${{ env.ARTIFACT_DIR }}/*
+        if: startsWith(github.ref, 'refs/tags/')
+
+  release-latest:
+    name: create latest release
+    runs-on: ubuntu-latest
+    needs: [release]
+    if: github.ref == 'refs/heads/main'
+    steps:
+      - uses: actions/download-artifact@v3
+        with:
+          name: ${{ env.ARTIFACT_DIR }}
+
+      - uses: "marvinpinto/action-automatic-releases@latest"
+        with:
+          repo_token: "${{ secrets.GITHUB_TOKEN }}"
+          automatic_release_tag: "latest"
+          prerelease: true
+          title: "Development Build"
           files: |
             *.tar.gz
             *.txt
+            *.whl
+        if: github.ref == 'refs/heads/main'

.gitignore

@@ -15,6 +15,8 @@ python/poetry.lock
 c/main
 cpp/test/test
 cpp/example
+.dub
+dub.selections.json
 go/main
 ruby/.bundle/
 ruby/.yardoc
@@ -27,9 +29,23 @@ ruby/tmp/
 ruby/Gemfile.lock
 rust/target
 rust/test.log
-ocaml/duniverse
-ocaml/_build
+duniverse
+_build
 php/Extism.php
+python/docs
 dist-newstyle
 .stack-work
-vendor
+vendor
+zig/zig-*
+zig/example-out/
+zig/*.log
+java/*.iml
+java/*.log
+java/.idea
+java/.DS_Store
+extism-maturin/src/extism.h
+runtime/*.log
+libextism/example
+libextism/extism*.pc
+*.cwasm
+test-cache

Cargo.toml

@@ -1,10 +1,18 @@
 [workspace]
-members = [
-  "manifest",
-  "runtime",
-  "rust",
-]
-exclude = [
-  "elixir/native/extism_nif"
-]
+resolver = "2"
+members = ["extism-maturin", "manifest", "runtime", "libextism", "convert", "convert-macros"]
+exclude = ["kernel"]
 
+[workspace.package]
+edition = "2021"
+authors = ["The Extism Authors", "oss@extism.org"]
+license = "BSD-3-Clause"
+homepage = "https://extism.org"
+repository = "https://github.com/extism/extism"
+version = "0.0.0+replaced-by-ci"
+
+[workspace.dependencies]
+extism = { path = "./runtime", version = "0.0.0+replaced-by-ci" }
+extism-convert = { path = "./convert", version = "0.0.0+replaced-by-ci" }
+extism-convert-macros = { path = "./convert-macros", version = "0.0.0+replaced-by-ci" }
+extism-manifest = { path = "./manifest", version = "0.0.0+replaced-by-ci" }

DEVELOPING.md

@@ -0,0 +1,117 @@
+# HACKING 
+
+## cutting releases
+
+### goals
+
+Cutting a release should be a boring, rote process with as little excitement as
+possible. Following the processes in this document, we should be able to cut a
+release at any time without worrying about producing bad artifacts. Our process
+should let us resolve build issues without affecting library users.
+
+### branching
+
+1. The `main` branch represents the next major version of the library.
+2. Previous major versions should be tracked using `v0.x`, `v1.x`, `v2.x`, used
+   for backporting changes as necessary.
+3. Libraries should generate a `latest` release using, e.g.,
+   `marvinpinto/action-automatic-releases` on changes to the `main` branch.
+
+### tag and release process
+
+1. Pick a target semver value. Prepend the semver value with `v`: `v1.2.3`.
+   Increment the minor version for additive changes and patch for bugfixes.
+   - For trickier changes, consider using release candidates: `rc0`, `rc1`, etc.
+2. Create an empty git commit for the tag to point at: `git commit -m 'v1.2.3-rc1' --allow-empty`.
+3. Create a new tag against that commit: `v1.2.3-rc1`.
+4. Push the changes to the library: `git push origin main v1.2.3-rc1`.
+    - You can separate these steps: `git push origin main` followed by `git push origin v1.2.3-rc1`,
+      if you want to make absolutely sure the commit you're pushing builds correctly before tagging it.
+5. Wait for the tag `build` workflow to complete.
+    - The `build` workflow should create a _draft_ release (using `softprops/action-gh-release` with `draft`
+      set to `true`) and upload built artifacts to the release.
+6. Once the workflow is complete, do whatever testing is necessary using the artifacts.
+    - TODO: We can add automation to this step so that we test on downstream deps automatically: e.g., if we
+      build a new kernel, we _should_ be able to trigger tests in the `python-sdk` _using_ that new kernel.
+7. Once we're confident the release is good, go to the releases page for the library and edit the draft release.
+    - If the release is a release candidate (`rc0..N`), make sure to mark the release as a "prerelease".
+    - Publish the draft release.
+    - This kicks off the publication workflow: taking the artifacts built during the `build` workflow and publishing
+      them to any necessary registry or repository.
+        - In extism, this publishes `extism-maturin` to PyPI as `extism-sys` and the dotnet packages to nuget.
+        - In `python-sdk`, this publishes `extism` to PyPI.
+        - In `js-sdk`, this publishes `@extism/extism` (and `extism`) to NPM.
+
+> **Note**
+> If you're at all worried about a release, use a private fork of the target library repo to test the release first (e.g., `extism/dev-extism`.)
+
+#### CLI flow
+
+For official releases:
+
+```
+$ git commit -m 'v9.9.9' --allow-empty
+$ git tag v9.9.9
+$ git push origin main v9.9.9
+$ gh run watch
+$ gh release edit v9.9.9 --tag v9.9.9 --title 'v9.9.9' --draft=false
+$ gh run watch
+```
+
+For prereleases:
+
+```
+$ git commit -m 'v9.9.9' --allow-empty
+$ git tag v9.9.9
+$ git push origin main v9.9.9
+$ gh run watch
+$ gh release edit v9.9.9 --tag v9.9.9 --title 'v9.9.9' --draft=false --prerelease
+$ gh run watch
+```
+
+### implementation
+
+Libraries should:
+
+- Provide a `ci` workflow, triggered on PR and `workflow_dispatch`.
+    - This workflow should exercise the tests, linting, and documentation generation of the library.
+- Provide a `build` workflow, triggered on `v*` tags and merges to `main`
+    - This workflow should produce artifacts and attach them to a draft release (if operating on a tag) or a `latest` release (if operating on `main`.)
+    - Artifacts include: source tarballs, checksums, shared objects, and documentation.
+- Provide a `release` workflow, triggered on github releases:
+    - This workflow should expect artifacts from the draft release to be available.
+    - Artifacts from the release should be published to their final destination as part of this workflow: tarballs to NPM, documentation to Cloudflare R2/Amazon S3/$yourFavoriteBucket.
+
+### A rough list of libraries and downstreams
+
+```mermaid
+flowchart TD;
+    A["runtime"] --> B["libextism"];
+    B --> C["extism-maturin"];
+    B --> X["nuget-extism"];
+    C --> D["python-sdk"];
+    B --> E["ruby-sdk"];
+    A --> F["go-sdk"];
+    G["plugins"] --> B;
+    G --> D;
+    G --> E;
+    G --> F;
+    G --> H["js-sdk"];
+    F --> I["cli"];
+    G --> J["dotnet-sdk"];
+    X --> J;
+    G --> K["cpp-sdk"];
+    G --> L["zig-sdk"];
+    B --> L;
+    G --> M["haskell-sdk"];
+    B --> M;
+    G --> N["php-sdk"];
+    B --> N;
+    G --> O["elixir-sdk"];
+    B --> O;
+    G --> P["d-sdk"];
+    B --> P;
+    G --> Q["ocaml-sdk"];
+    B --> Q;
+```
+

Makefile

@@ -1,7 +1,9 @@
 DEST?=/usr/local
 SOEXT=so
+AEXT=a
 FEATURES?=default
 DEFAULT_FEATURES?=yes
+RUST_TARGET?=
 
 UNAME := $(shell uname -s)
 ifeq ($(UNAME),Darwin)
@@ -18,20 +20,43 @@ else
 	FEATURE_FLAGS=--features $(FEATURES)
 endif
 
-
-.PHONY: build
-
-lint:
-	cargo clippy --release --no-deps --manifest-path runtime/Cargo.toml
+ifeq ($(RUST_TARGET),)
+	TARGET_FLAGS=
+else
+	TARGET_FLAGS=--target $(RUST_TARGET)
+endif
 
 build:
-	cargo build --release $(FEATURE_FLAGS) --manifest-path runtime/Cargo.toml
+	cargo build --release $(FEATURE_FLAGS) --manifest-path libextism/Cargo.toml $(TARGET_FLAGS)
+	sed -e "s%@CMAKE_INSTALL_PREFIX@%$(DEST)%" libextism/extism.pc.in > libextism/extism.pc
+	sed -e "s%@CMAKE_INSTALL_PREFIX@%$(DEST)%" libextism/extism-static.pc.in > libextism/extism-static.pc
+
+bench:
+	@(cargo criterion $(TARGET_FLAGS) || echo 'For nicer output use cargo-criterion: `cargo install cargo-criterion` - using `cargo bench`') && cargo bench $(TARGET_FLAGS)
+
+.PHONY: kernel
+kernel:
+	cd kernel && bash build.sh
+
+lint:
+	cargo clippy --release --no-deps --manifest-path runtime/Cargo.toml $(TARGET_FLAGS)
+
+debug:
+	RUSTFLAGS=-g RUST_TARGET=$(RUST_TARGET) $(MAKE) build
 
 install:
-	install runtime/extism.h $(DEST)/include
-	install target/release/libextism.$(SOEXT) $(DEST)/lib
+	echo $(RUST_TARGET)
+	mkdir -p $(DEST)/lib $(DEST)/include $(DEST)/lib/pkgconfig
+	install runtime/extism.h $(DEST)/include/extism.h
+	if [ -f target/$(RUST_TARGET)/release/libextism.$(SOEXT) ]; then \
+	    install target/$(RUST_TARGET)/release/libextism.$(SOEXT) $(DEST)/lib/libextism.$(SOEXT); \
+	fi
+	install target/$(RUST_TARGET)/release/libextism.$(AEXT) $(DEST)/lib/libextism.$(AEXT)
+	install libextism/extism.pc $(DEST)/lib/pkgconfig/extism.pc
+	install libextism/extism-static.pc $(DEST)/lib/pkgconfig/extism-static.pc
 
 uninstall:
-	rm -f $(DEST)/include/extism.h $(DEST)/lib/libextism.$(SOEXT)
+	rm -f $(DEST)/include/extism.h $(DEST)/lib/libextism.$(SOEXT) $(DEST)/lib/libextism.$(AEXT) \
+	$(DEST)/lib/pkgconfig/extism*.pc
 
 

README.md

@@ -1,50 +1,111 @@
-### _Welcome!_
+<div align="center">
+    <a href="https://extism.org">
+    <picture>
+        <source media="(prefers-color-scheme: dark)" srcset=".github/assets/logo-horizontal-darkmode.png">
+        <img alt="Extism - the WebAssembly framework" width="75%" style="max-width: 600px" src=".github/assets/logo-horizontal.png">
+    </picture>
+    </a>
 
-**Please note:** this project still under active development. It's usable, but expect some rough edges while work is underway. If you're interested in working on or building with Extism, please join our [Discord](https://discord.gg/cx3usBCWnc) and let us know - we are happy to help get you started.
+[![Discord](https://img.shields.io/discord/1011124058408112148?color=%23404eed&label=Community%20Chat&logo=Discord&logoColor=%23404eed)](https://extism.org/discord)
+![GitHub Org's stars](https://img.shields.io/github/stars/extism)
+![Downloads](https://img.shields.io/crates/d/extism)
+![GitHub License](https://img.shields.io/github/license/extism/extism)
+![GitHub release (with filter)](https://img.shields.io/github/v/release/extism/extism)
 
-[![Discord](https://img.shields.io/discord/1011124058408112148?color=%23404eed&label=Community%20Chat&logo=Discord&logoColor=%23404eed)](https://discord.gg/cx3usBCWnc)
+</div>
 
-# [Extism](https://extism.org)
+# Overview
 
-The universal plug-in system. Run WebAssembly extensions inside your app. Use idiomatic Host SDKs for [Go](https://extism.org/docs/integrate-into-your-codebase/go-host-sdk), 
-[Ruby](https://extism.org/docs/integrate-into-your-codebase/ruby-host-sdk), [Python](https://extism.org/docs/integrate-into-your-codebase/python-host-sdk), 
-[Node](https://extism.org/docs/integrate-into-your-codebase/node-host-sdk), [Rust](https://extism.org/docs/integrate-into-your-codebase/rust-host-sdk), 
-[C](https://extism.org/docs/integrate-into-your-codebase/c-host-sdk), [C++](https://extism.org/docs/integrate-into-your-codebase/cpp-host-sdk), 
-[OCaml](https://extism.org/docs/integrate-into-your-codebase/ocaml-host-sdk), [Haskell](https://extism.org/docs/integrate-into-your-codebase/haskell-host-sdk), [PHP](https://extism.org/docs/integrate-into-your-codebase/php-host-sdk), [Elixir/Erlang](https://extism.org/docs/integrate-into-your-codebase/elixir-or-erlang-host-sdk) &amp; more (others coming soon). 
+Extism is a lightweight framework for building with WebAssembly (Wasm). It
+supports running Wasm code on servers, the edge, CLIs, IoT, browsers and
+everything in between. Extism is designed to be "universal" in that it supports
+a common interface, no matter where it runs.
 
-Plug-in development kits (PDK) for plug-in authors supported in [Rust](https://github.com/extism/rust-pdk), [AssemblyScript](https://github.com/extism/assemblyscript-pdk), [Go](https://github.com/extism/go-pdk), [C/C++](https://github.com/extism/c-pdk).
+> **Note:** One of the primary use cases for Extism is **building extensible
+> software & plugins**. You want to be able to execute arbitrary, untrusted code
+> from your users? Extism makes this safe and practical to do.
 
-<p align="center">
-  <img style="width: 70%;" src="https://user-images.githubusercontent.com/7517515/200043015-ddfe5833-0252-43a8-bc9e-5b3f829c37d1.png" alt="Extism embedded SDK language support"/>
-</p>
+Additionally, Extism adds some extra utilities on top of standard Wasm runtimes.
+For example, we support persistent memory/module-scope variables, secure &
+host-controlled HTTP without WASI, runtime limiters & timers, simpler host
+function linking, and more. Extism users build:
 
+- plug-in systems
+- FaaS platforms
+- code generators
+- web applications
+- & much more...
 
+# Run WebAssembly In Your App
 
-Add a flexible, secure, and _bLaZiNg FaSt_ plug-in system to your project. Server, desktop, mobile, web, database -- you name it. Enable users to write and execute safe extensions to your software in **3 easy steps:**
+Pick a SDK to import into your program, and refer to the documentation to get
+started:
 
-### 1. Import
+| Type        | Language                                                                                       | Source Code                                                             | Package                                                                 |
+| ----------- | ---------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------- | ----------------------------------------------------------------------- |
+| Rust SDK    | <img alt="Rust SDK" src="https://extism.org/img/sdk-languages/rust.svg" width="50px"/>         | https://github.com/extism/extism/tree/main/runtime                      | [Crates.io](https://crates.io/crates/extism)                            |
+| JS SDK      | <img alt="JS SDK" src="https://extism.org/img/sdk-languages/js.svg" width="50px"/>             | https://github.com/extism/js-sdk <br/>(supports Web, Node, Deno & Bun!) | [NPM](https://www.npmjs.com/package/@extism/extism)                     |
+| Elixir SDK  | <img alt="Elixir SDK" src="https://extism.org/img/sdk-languages/elixir.svg" width="50px"/>     | https://github.com/extism/elixir-sdk                                    | [Hex](https://hex.pm/packages/extism)                                   |
+| Go SDK      | <img alt="Go SDK" src="https://extism.org/img/sdk-languages/go.svg" width="50px"/>             | https://github.com/extism/go-sdk                                        | [Go mod](https://pkg.go.dev/github.com/extism/go-sdk)                   |
+| Haskell SDK | <img alt="Haskell SDK" src="https://extism.org/img/sdk-languages/haskell.svg" width="50px"/>   | https://github.com/extism/haskell-sdk                                   | [Hackage](https://hackage.haskell.org/package/extism)                   |
+| Java SDK    | <img alt="Java SDK" src="https://extism.org/img/sdk-languages/java-android.svg" width="50px"/> | https://github.com/extism/java-sdk                                      | [Sonatype](https://central.sonatype.com/artifact/org.extism.sdk/extism) |
+| .NET SDK    | <img alt=".NET SDK" src="https://extism.org/img/sdk-languages/dotnet.svg" width="50px"/>       | https://github.com/extism/dotnet-sdk <br/>(supports C# & F#!)           | [Nuget](https://www.nuget.org/packages/Extism.Sdk)                      |
+| OCaml SDK   | <img alt="OCaml SDK" src="https://extism.org/img/sdk-languages/ocaml.svg" width="50px"/>       | https://github.com/extism/ocaml-sdk                                     | [opam](https://opam.ocaml.org/packages/extism/)                         |
+| PHP SDK     | <img alt="PHP SDK" src="https://extism.org/img/sdk-languages/php.svg" width="50px"/>           | https://github.com/extism/php-sdk                                       | [Packagist](https://packagist.org/packages/extism/extism)               |
+| Python SDK  | <img alt="Python SDK" src="https://extism.org/img/sdk-languages/python.svg" width="50px"/>     | https://github.com/extism/python-sdk                                    | [PyPi](https://pypi.org/project/extism/)                                |
+| Ruby SDK    | <img alt="Ruby SDK" src="https://extism.org/img/sdk-languages/ruby.svg" width="50px"/>         | https://github.com/extism/ruby-sdk                                      | [RubyGems](https://rubygems.org/gems/extism)                            |
+| Zig SDK     | <img alt="Zig SDK" src="https://extism.org/img/sdk-languages/zig.svg" width="50px"/>           | https://github.com/extism/zig-sdk                                       | N/A                                                                     |
+| C SDK       | <img alt="C SDK" src="https://extism.org/img/sdk-languages/c.svg" width="50px"/>               | https://github.com/extism/extism/tree/main/libextism                    | N/A                                                                     |
+| C++ SDK     | <img alt="C++ SDK" src="https://extism.org/img/sdk-languages/cpp.svg" width="50px"/>           | https://github.com/extism/cpp-sdk                                       | N/A                                                                     |
 
-Import an Extism Host SDK into your code as a library dependency.
+# Compile WebAssembly to run in Extism Hosts
 
-### 2. Integrate 
+Extism Hosts (running the SDK) must execute WebAssembly code that has a PDK
+library compiled in to the `.wasm` binary. PDKs make it easy for plug-in /
+extension code authors to read input from the host and return data back, read
+provided configuration, set/get variables, make outbound HTTP calls if allowed,
+and more.
 
-Identify the place(s) in your code where some arbitrary logic should run (the plug-in!), returning your code some results.
+Pick a PDK to import into your Wasm program, and refer to the documentation to
+get started:
 
-### 3. Execute
+| Type               | Language                                                                                                   | Source Code                                                   | Package                                                   |
+| ------------------ | ---------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------- | --------------------------------------------------------- |
+| Rust PDK           | <img alt="Rust PDK" src="https://extism.org/img/sdk-languages/rust.svg" width="50px"/>                     | https://github.com/extism/rust-pdk                            | [Crates.io](https://crates.io/crates/extism-pdk)          |
+| JS PDK             | <img alt="JS PDK" src="https://extism.org/img/sdk-languages/js.svg" width="50px"/>                         | https://github.com/extism/js-pdk                              | N/A                                                       |
+| Go PDK             | <img alt="Go PDK" src="https://extism.org/img/sdk-languages/go.svg" width="50px"/>                         | https://github.com/extism/go-pdk                              | [Go mod](https://pkg.go.dev/github.com/extism/go-pdk)     |
+| Haskell PDK        | <img alt="Haskell PDK" src="https://extism.org/img/sdk-languages/haskell.svg" width="50px"/>               | https://github.com/extism/haskell-pdk                         | [Hackage](https://hackage.haskell.org/package/extism-pdk) |
+| AssemblyScript PDK | <img alt="AssemblyScript PDK" src="https://extism.org/img/sdk-languages/assemblyscript.svg" width="50px"/> | https://github.com/extism/assemblyscript-pdk                  | [NPM](https://www.npmjs.com/package/@extism/as-pdk)       |
+| .NET PDK           | <img alt=".NET PDK" src="https://extism.org/img/sdk-languages/dotnet.svg" width="50px"/>                   | https://github.com/extism/dotnet-pdk <br/>(supports C# & F#!) | https://www.nuget.org/packages/Extism.Pdk                 |
+| C PDK              | <img alt="C PDK" src="https://extism.org/img/sdk-languages/c.svg" width="50px"/>                           | https://github.com/extism/c-pdk                               | N/A                                                       |
+| Zig PDK            | <img alt="Zig PDK" src="https://extism.org/img/sdk-languages/zig.svg" width="50px"/>                       | https://github.com/extism/zig-pdk                             | N/A                                                       |
 
-Load WebAssembly modules at any time in your app's lifetime and Extism will execute them in a secure sandbox, fully isolated from your program's memory.
+# Support
 
---- 
+## Discord
+
+If you experience any problems or have any questions, please join our
+[Discord](https://extism.org/discord) and let us know. Our community is very
+responsive and happy to help get you started.
 
 ## Usage
 
-Head to the [project website](https://extism.org) for more information and docs. Also, consider reading an [overview](https://extism.org/docs/overview) of Extism and its goals & approach.
+Head to the [project website](https://extism.org) for more information and docs.
+Also, consider reading an [overview](https://extism.org/docs/overview) of Extism
+and its goals & approach.
 
 ## Contribution
 
-Thank you for considering a contribution to Extism, we are happy to help you make a PR or find something to work on! 
+Thank you for considering a contribution to Extism, we are happy to help you
+make a PR or find something to work on!
 
-The easiest way to start would be to join the [Discord](https://discord.gg/cx3usBCWnc) or open an issue on the [`extism/proposals`](https://github.com/extism/proposals) issue tracker, which can eventually become an Extism Improvement Proposal (EIP).
+The easiest way to start would be to join the
+[Discord](https://extism.org/discord) or open an issue on the
+[`extism/proposals`](https://github.com/extism/proposals) issue tracker, which
+can eventually become an Extism Improvement Proposal (EIP).
+
+For more information, please read the
+[Contributing](https://extism.org/docs/concepts/contributing) guide.
 
 ---
 
@@ -53,9 +114,8 @@ The easiest way to start would be to join the [Discord](https://discord.gg/cx3us
 Extism is an open-source product from the team at:
 
 <p align="left">
-  <a href="https://dylib.so" _target="blanks"><img width="200px" src="https://user-images.githubusercontent.com/7517515/198204119-5afdebb9-a5d8-4322-bd2a-46179c8d7b24.svg"/></a>
+  <a href="https://dylibso.com" _target="blanks"><img width="200px" src="https://user-images.githubusercontent.com/7517515/198204119-5afdebb9-a5d8-4322-bd2a-46179c8d7b24.svg"/></a>
 </p>
 
-
-
-_Reach out and tell us what you're building! We'd love to help._
+_Reach out and tell us what you're building! We'd love to help:_
+<a href="mailto:hello@dylibso.com">hello@dylibso.com</a>

c/Makefile

@@ -1,2 +0,0 @@
-build:
-	clang -o main main.c -lextism -L .

c/main.c

@@ -1,60 +0,0 @@
-#include "../runtime/extism.h"
-
-#include <assert.h>
-#include <errno.h>
-#include <fcntl.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <unistd.h>
-
-uint8_t *read_file(const char *filename, size_t *len) {
-
-  FILE *fp = fopen(filename, "rb");
-  if (fp == NULL) {
-    return NULL;
-  }
-  fseek(fp, 0, SEEK_END);
-  size_t length = ftell(fp);
-  fseek(fp, 0, SEEK_SET);
-
-  uint8_t *data = malloc(length);
-  if (data == NULL) {
-    return NULL;
-  }
-
-  assert(fread(data, 1, length, fp) == length);
-  fclose(fp);
-
-  *len = length;
-  return data;
-}
-
-int main(int argc, char *argv[]) {
-  if (argc < 2) {
-    fputs("Not enough arguments\n", stderr);
-    exit(1);
-  }
-
-  ExtismContext *ctx = extism_context_new();
-
-  size_t len = 0;
-  uint8_t *data = read_file("../wasm/code.wasm", &len);
-  ExtismPlugin plugin = extism_plugin_new(ctx, data, len, false);
-  free(data);
-  if (plugin < 0) {
-    exit(1);
-  }
-
-  assert(extism_plugin_call(ctx, plugin, "count_vowels", (uint8_t *)argv[1],
-                            strlen(argv[1])) == 0);
-  ExtismSize out_len = extism_plugin_output_length(ctx, plugin);
-  const uint8_t *output = extism_plugin_output_data(ctx, plugin);
-  write(STDOUT_FILENO, output, out_len);
-  write(STDOUT_FILENO, "\n", 1);
-
-  extism_plugin_free(ctx, plugin);
-  extism_context_free(ctx);
-  return 0;
-}

composer.json

@@ -1,50 +0,0 @@
-{
-    "name": "extism/extism",
-    "description": "Make your software programmable. Run WebAssembly extensions in your app using the first off-the-shelf, universal plug-in system.",
-    "license": "BSD-3-Clause",
-    "type": "library",
-    "keywords": [
-        "WebAssembly",
-        "plugin-system",
-        "runtime",
-        "plug-in"
-    ],
-    "authors": [
-        {
-            "name": "The Extism Authors",
-            "email": "oss@extism.org",
-            "homepage": "https://extism.org"
-        },
-        {
-            "name": "Dylibso, Inc.",
-            "email": "oss@dylib.so",
-            "homepage": "https://dylib.so"
-        }
-    ],
-    "require": {
-        "php": "^7.4 || ^8",
-        "ircmaxell/ffime": "dev-master"
-    },
-    "suggest": {},
-    "minimum-stability": "dev",
-    "prefer-stable": true,
-    "autoload": {
-        "psr-4": {
-            "Extism\\": "php/src/"
-        },
-        "files": [
-            "php/src/Plugin.php",
-            "php/src/generate.php",
-            "php/src/extism.h"
-        ]
-    },
-    "autoload-dev": {
-        "psr-4": {}
-    },
-    "config": {
-        "sort-packages": true
-    },
-    "extra": {},
-    "scripts": {},
-    "scripts-descriptions": {}
-}

composer.lock

@@ -1,163 +0,0 @@
-{
-    "_readme": [
-        "This file locks the dependencies of your project to a known state",
-        "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
-        "This file is @generated automatically"
-    ],
-    "content-hash": "0e0352cd3a96e03fd9c964888deedb29",
-    "packages": [
-        {
-            "name": "ircmaxell/ffime",
-            "version": "dev-master",
-            "source": {
-                "type": "git",
-                "url": "https://github.com/ircmaxell/FFIMe.git",
-                "reference": "5f648f95ecf23262a2e58f4e4c9001bd1b5f9c98"
-            },
-            "dist": {
-                "type": "zip",
-                "url": "https://api.github.com/repos/ircmaxell/FFIMe/zipball/5f648f95ecf23262a2e58f4e4c9001bd1b5f9c98",
-                "reference": "5f648f95ecf23262a2e58f4e4c9001bd1b5f9c98",
-                "shasum": ""
-            },
-            "require": {
-                "ircmaxell/php-c-parser": "dev-master",
-                "ircmaxell/php-object-symbolresolver": "dev-master",
-                "php": ">=8.0"
-            },
-            "require-dev": {
-                "phpunit/phpunit": "^8.0"
-            },
-            "default-branch": true,
-            "type": "library",
-            "autoload": {
-                "psr-4": {
-                    "FFIMe\\": "lib/"
-                }
-            },
-            "notification-url": "https://packagist.org/downloads/",
-            "license": [
-                "MIT"
-            ],
-            "authors": [
-                {
-                    "name": "Anthony Ferrara",
-                    "email": "ircmaxell@gmail.com"
-                }
-            ],
-            "description": "Make life easy when working with 7.4's FFI",
-            "support": {
-                "issues": "https://github.com/ircmaxell/FFIMe/issues",
-                "source": "https://github.com/ircmaxell/FFIMe/tree/master"
-            },
-            "time": "2022-09-01T18:56:19+00:00"
-        },
-        {
-            "name": "ircmaxell/php-c-parser",
-            "version": "dev-master",
-            "source": {
-                "type": "git",
-                "url": "https://github.com/ircmaxell/php-c-parser.git",
-                "reference": "55e0a4fdf88d6e955d928860e1e107a68492c1cf"
-            },
-            "dist": {
-                "type": "zip",
-                "url": "https://api.github.com/repos/ircmaxell/php-c-parser/zipball/55e0a4fdf88d6e955d928860e1e107a68492c1cf",
-                "reference": "55e0a4fdf88d6e955d928860e1e107a68492c1cf",
-                "shasum": ""
-            },
-            "require": {
-                "php": ">=7.4"
-            },
-            "require-dev": {
-                "ircmaxell/php-yacc": "dev-master",
-                "phpunit/phpunit": "^8.0"
-            },
-            "default-branch": true,
-            "type": "library",
-            "autoload": {
-                "psr-4": {
-                    "PHPCParser\\": "lib/"
-                }
-            },
-            "notification-url": "https://packagist.org/downloads/",
-            "license": [
-                "MIT"
-            ],
-            "authors": [
-                {
-                    "name": "Anthony Ferrara",
-                    "email": "ircmaxell@gmail.com"
-                },
-                {
-                    "name": "Bob Weinand",
-                    "email": "bobwei9@hotmail.com"
-                }
-            ],
-            "description": "Parse C when using PHP",
-            "support": {
-                "issues": "https://github.com/ircmaxell/php-c-parser/issues",
-                "source": "https://github.com/ircmaxell/php-c-parser/tree/master"
-            },
-            "time": "2022-08-27T17:37:14+00:00"
-        },
-        {
-            "name": "ircmaxell/php-object-symbolresolver",
-            "version": "dev-master",
-            "source": {
-                "type": "git",
-                "url": "https://github.com/ircmaxell/php-object-symbolresolver.git",
-                "reference": "3734df2b22d7c8273ee6f6f2155fddde6056d057"
-            },
-            "dist": {
-                "type": "zip",
-                "url": "https://api.github.com/repos/ircmaxell/php-object-symbolresolver/zipball/3734df2b22d7c8273ee6f6f2155fddde6056d057",
-                "reference": "3734df2b22d7c8273ee6f6f2155fddde6056d057",
-                "shasum": ""
-            },
-            "require": {
-                "php": ">=7.4"
-            },
-            "default-branch": true,
-            "type": "library",
-            "autoload": {
-                "psr-4": {
-                    "PHPObjectSymbolResolver\\": "lib/"
-                }
-            },
-            "notification-url": "https://packagist.org/downloads/",
-            "license": [
-                "MIT"
-            ],
-            "authors": [
-                {
-                    "name": "Anthony Ferrara",
-                    "email": "ircmaxell@gmail.com"
-                },
-                {
-                    "name": "Bob Weinand",
-                    "email": "bobwei9@hotmail.com"
-                }
-            ],
-            "description": "An object file (ELF, Mach-O) parser",
-            "support": {
-                "issues": "https://github.com/ircmaxell/php-object-symbolresolver/issues",
-                "source": "https://github.com/ircmaxell/php-object-symbolresolver/tree/master"
-            },
-            "time": "2022-08-14T19:30:20+00:00"
-        }
-    ],
-    "packages-dev": [],
-    "aliases": [],
-    "minimum-stability": "dev",
-    "stability-flags": {
-        "ircmaxell/ffime": 20
-    },
-    "prefer-stable": true,
-    "prefer-lowest": false,
-    "platform": {
-        "php": "^7.4 || ^8"
-    },
-    "platform-dev": [],
-    "plugin-api-version": "2.3.0"
-}

convert-macros/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "extism-convert-macros"
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+homepage.workspace = true
+repository.workspace = true
+version.workspace = true
+description = "Macros to remove boilerplate with Extism"
+
+[lib]
+proc-macro = true
+
+[features]
+extism-path = []
+extism-pdk-path = []
+
+[dependencies]
+manyhow.version = "0.11.0"
+proc-macro-crate = "3.1.0"
+proc-macro2 = "1.0.78"
+quote = "1.0.35"
+syn = { version = "2.0.48", features = ["derive"] }
+
+[dev-dependencies]
+trybuild = "1.0.89"

convert-macros/src/lib.rs

@@ -0,0 +1,108 @@
+use std::iter;
+
+use manyhow::{ensure, error_message, manyhow, Result};
+use proc_macro_crate::{crate_name, FoundCrate};
+use quote::{format_ident, quote, ToTokens};
+use syn::{parse_quote, Attribute, DeriveInput, Path};
+
+/// Tries to resolve the path to `extism_convert` dynamically, falling back to feature flags when unsuccessful.
+fn convert_path() -> Path {
+    match (
+        crate_name("extism"),
+        crate_name("extism-convert"),
+        crate_name("extism-pdk"),
+    ) {
+        (Ok(FoundCrate::Name(name)), ..) => {
+            let ident = format_ident!("{name}");
+            parse_quote!(::#ident::convert)
+        }
+        (_, Ok(FoundCrate::Name(name)), ..) | (.., Ok(FoundCrate::Name(name))) => {
+            let ident = format_ident!("{name}");
+            parse_quote!(::#ident)
+        }
+        (Ok(FoundCrate::Itself), ..) => parse_quote!(::extism::convert),
+        (_, Ok(FoundCrate::Itself), ..) => parse_quote!(::extism_convert),
+        (.., Ok(FoundCrate::Itself)) => parse_quote!(::extism_pdk),
+        _ if cfg!(feature = "extism-path") => parse_quote!(::extism::convert),
+        _ if cfg!(feature = "extism-pdk-path") => parse_quote!(::extism_pdk),
+        _ => parse_quote!(::extism_convert),
+    }
+}
+
+fn extract_encoding(attrs: &[Attribute]) -> Result<Path> {
+    let encodings: Vec<_> = attrs
+        .iter()
+        .filter(|attr| attr.path().is_ident("encoding"))
+        .collect();
+    ensure!(!encodings.is_empty(), "encoding needs to be specified"; try = "`#[encoding(ToJson)]`");
+    ensure!(encodings.len() < 2, encodings[1], "only one encoding can be specified"; try = "remove `{}`", encodings[1].to_token_stream());
+
+    Ok(encodings[0].parse_args().map_err(
+        |e| error_message!(e.span(), "{e}"; note= "expects a path"; try = "`#[encoding(ToJson)]`"),
+    )?)
+}
+
+#[manyhow]
+#[proc_macro_derive(ToBytes, attributes(encoding))]
+pub fn to_bytes(
+    DeriveInput {
+        attrs,
+        ident,
+        generics,
+        ..
+    }: DeriveInput,
+) -> Result {
+    let encoding = extract_encoding(&attrs)?;
+    let convert = convert_path();
+
+    let (_, type_generics, _) = generics.split_for_impl();
+
+    let mut generics = generics.clone();
+    generics.make_where_clause().predicates.push(
+        parse_quote!(for<'__to_bytes_b> #encoding<&'__to_bytes_b Self>: #convert::ToBytes<'__to_bytes_b>)
+    );
+    generics.params = iter::once(parse_quote!('__to_bytes_a))
+        .chain(generics.params)
+        .collect();
+    let (impl_generics, _, where_clause) = generics.split_for_impl();
+
+    Ok(quote! {
+        impl #impl_generics #convert::ToBytes<'__to_bytes_a> for #ident #type_generics #where_clause
+        {
+            type Bytes = ::std::vec::Vec<u8>;
+
+            fn to_bytes(&self) -> Result<Self::Bytes, #convert::Error> {
+                #convert::ToBytes::to_bytes(&#encoding(self)).map(|__bytes| __bytes.as_ref().to_vec())
+            }
+        }
+
+    })
+}
+
+#[manyhow]
+#[proc_macro_derive(FromBytes, attributes(encoding))]
+pub fn from_bytes(
+    DeriveInput {
+        attrs,
+        ident,
+        mut generics,
+        ..
+    }: DeriveInput,
+) -> Result {
+    let encoding = extract_encoding(&attrs)?;
+    let convert = convert_path();
+    generics
+        .make_where_clause()
+        .predicates
+        .push(parse_quote!(#encoding<Self>: #convert::FromBytesOwned));
+    let (impl_generics, type_generics, where_clause) = generics.split_for_impl();
+    Ok(quote! {
+        impl #impl_generics #convert::FromBytesOwned for #ident #type_generics #where_clause
+        {
+            fn from_bytes_owned(__data: &[u8]) -> Result<Self, #convert::Error> {
+                <#encoding<Self> as #convert::FromBytesOwned>::from_bytes_owned(__data).map(|__encoding| __encoding.0)
+            }
+        }
+
+    })
+}

convert-macros/tests/ui.rs

@@ -0,0 +1,5 @@
+#[test]
+fn ui() {
+    let t = trybuild::TestCases::new();
+    t.compile_fail("tests/ui/*.rs");
+}

convert-macros/tests/ui/invalid-encoding.rs

@@ -0,0 +1,23 @@
+use extism_convert_macros::ToBytes;
+
+#[derive(ToBytes)]
+struct MissingEncoding;
+
+#[derive(ToBytes)]
+#[encoding]
+struct EmptyAttr;
+
+#[derive(ToBytes)]
+#[encoding = "string"]
+struct EqNoParen;
+
+#[derive(ToBytes)]
+#[encoding(something, else)]
+struct NotAPath;
+
+#[derive(ToBytes)]
+#[encoding(Multiple)]
+#[encoding(Encodings)]
+struct MultipleEncodings;
+
+fn main() {}

convert-macros/tests/ui/invalid-encoding.stderr

@@ -0,0 +1,44 @@
+error: encoding needs to be specified
+
+         = try: `#[encoding(ToJson)]`
+ --> tests/ui/invalid-encoding.rs:3:10
+  |
+3 | #[derive(ToBytes)]
+  |          ^^^^^^^
+  |
+  = note: this error originates in the derive macro `ToBytes` (in Nightly builds, run with -Z macro-backtrace for more info)
+
+error: expected attribute arguments in parentheses: #[encoding(...)]
+
+         = note: expects a path
+         = try: `#[encoding(ToJson)]`
+ --> tests/ui/invalid-encoding.rs:7:3
+  |
+7 | #[encoding]
+  |   ^^^^^^^^
+
+error: expected parentheses: #[encoding(...)]
+
+         = note: expects a path
+         = try: `#[encoding(ToJson)]`
+  --> tests/ui/invalid-encoding.rs:11:12
+   |
+11 | #[encoding = "string"]
+   |            ^
+
+error: unexpected token
+
+         = note: expects a path
+         = try: `#[encoding(ToJson)]`
+  --> tests/ui/invalid-encoding.rs:15:21
+   |
+15 | #[encoding(something, else)]
+   |                     ^
+
+error: only one encoding can be specified
+
+         = try: remove `#[encoding(Encodings)]`
+  --> tests/ui/invalid-encoding.rs:20:1
+   |
+20 | #[encoding(Encodings)]
+   | ^^^^^^^^^^^^^^^^^^^^^^

convert/Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+name = "extism-convert"
+readme = "./README.md"
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+homepage.workspace = true
+repository.workspace = true
+version.workspace = true
+description = "Traits to make Rust types usable with Extism"
+
+[dependencies]
+anyhow = "1.0.75"
+base64 = "~0.22"
+bytemuck = {version = "1.14.0", optional = true }
+prost = { version = "0.12.0", optional = true }
+protobuf = { version = "3.2.0", optional = true }
+rmp-serde = { version = "1.1.2", optional = true }
+serde = "1.0.186"
+serde_json = "1.0.105"
+extism-convert-macros.workspace = true
+
+[dev-dependencies]
+serde = { version = "1.0.186", features = ["derive"] }
+
+[features]
+default = ["msgpack", "prost", "raw"]
+msgpack = ["rmp-serde"]
+raw = ["bytemuck"]
+extism-path = ["extism-convert-macros/extism-path"]
+extism-pdk-path = ["extism-convert-macros/extism-pdk-path"]

convert/README.md

@@ -0,0 +1,12 @@
+# extism-convert
+
+The [extism-convert](https://crates.io/crates/extism-convert) crate is used by the [Rust SDK](https://crates.io/crates/extism) and [Rust PDK](https://crates.io/crates/extism-pdk) to provide a shared interface for
+encoding and decoding values that can be passed to Extism function calls.
+
+A set of types (Json, Msgpack, Protobuf) that can be used to specify a serde encoding are also provided. These are
+similar to [axum extractors](https://docs.rs/axum/latest/axum/extract/index.html#intro) - they are
+implemented as a tuple struct with a single field that is meant to be extracted using pattern matching.
+
+## Documentation
+
+See [extism-convert on docs.rs](https://docs.rs/extism-convert/latest/extism_convert/) for in-depth documentation.

convert/src/encoding.rs

@@ -0,0 +1,187 @@
+use crate::*;
+
+use base64::Engine;
+
+/// The `encoding` macro can be used to create newtypes that implement a particular encoding for the
+/// inner value.
+///
+/// For example, the following line creates a new JSON encoding using serde_json:
+///
+/// ```
+/// extism_convert::encoding!(MyJson, serde_json::to_vec, serde_json::from_slice);
+/// ```
+///
+/// This will create a struct `struct MyJson<T>(pub T)` and implement [`ToBytes`] using [`serde_json::to_vec`]
+/// and [`FromBytesOwned`] using [`serde_json::from_slice`]
+#[macro_export]
+macro_rules! encoding {
+    ($pub:vis $name:ident, $to_vec:expr, $from_slice:expr) => {
+        #[doc = concat!(stringify!($name), " encoding")]
+        #[derive(Debug)]
+        $pub struct $name<T>(pub T);
+
+        impl<T> $name<T> {
+            pub fn into_inner(self) -> T {
+                self.0
+            }
+        }
+
+        impl<T> From<T> for $name<T> {
+            fn from(data: T) -> Self {
+                Self(data)
+            }
+        }
+
+        impl<T: serde::de::DeserializeOwned> $crate::FromBytesOwned for $name<T> {
+            fn from_bytes_owned(data: &[u8]) -> std::result::Result<Self, $crate::Error> {
+                let x = $from_slice(data)?;
+                std::result::Result::Ok($name(x))
+            }
+        }
+
+        impl<'a, T: serde::Serialize> $crate::ToBytes<'a> for $name<T> {
+            type Bytes = Vec<u8>;
+
+            fn to_bytes(&self) -> std::result::Result<Self::Bytes, $crate::Error> {
+                let enc = $to_vec(&self.0)?;
+                std::result::Result::Ok(enc)
+            }
+        }
+    };
+}
+
+encoding!(pub Json, serde_json::to_vec, serde_json::from_slice);
+
+#[cfg(feature = "msgpack")]
+encoding!(pub Msgpack, rmp_serde::to_vec, rmp_serde::from_slice);
+
+impl<'a> ToBytes<'a> for serde_json::Value {
+    type Bytes = Vec<u8>;
+
+    fn to_bytes(&self) -> Result<Self::Bytes, Error> {
+        Ok(serde_json::to_vec(self)?)
+    }
+}
+
+impl FromBytesOwned for serde_json::Value {
+    fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
+        Ok(serde_json::from_slice(data)?)
+    }
+}
+
+/// Base64 conversion
+///
+/// When using `Base64` with `ToBytes` any type that implement `AsRef<[T]>` may be used as the inner value,
+/// but only `Base64<String>` and `Base64<Vec>` may be used with `FromBytes`
+///
+/// A value wrapped in `Base64` will automatically be encoded/decoded using base64, the inner value should not
+/// already be base64 encoded.
+#[derive(Debug)]
+pub struct Base64<T: AsRef<[u8]>>(pub T);
+
+impl<T: AsRef<[u8]>> From<T> for Base64<T> {
+    fn from(data: T) -> Self {
+        Self(data)
+    }
+}
+
+impl<'a, T: AsRef<[u8]>> ToBytes<'a> for Base64<T> {
+    type Bytes = String;
+
+    fn to_bytes(&self) -> Result<Self::Bytes, Error> {
+        Ok(base64::engine::general_purpose::STANDARD.encode(&self.0))
+    }
+}
+
+impl FromBytesOwned for Base64<Vec<u8>> {
+    fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
+        Ok(Base64(
+            base64::engine::general_purpose::STANDARD.decode(data)?,
+        ))
+    }
+}
+
+impl FromBytesOwned for Base64<String> {
+    fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
+        Ok(Base64(String::from_utf8(
+            base64::engine::general_purpose::STANDARD.decode(data)?,
+        )?))
+    }
+}
+
+/// Protobuf encoding
+///
+/// Allows for `prost` Protobuf messages to be used as arguments to Extism plugin calls
+#[cfg(feature = "prost")]
+#[derive(Debug)]
+pub struct Prost<T: prost::Message>(pub T);
+
+#[cfg(feature = "prost")]
+impl<T: prost::Message> From<T> for Prost<T> {
+    fn from(data: T) -> Self {
+        Self(data)
+    }
+}
+
+#[cfg(feature = "prost")]
+impl<'a, T: prost::Message> ToBytes<'a> for Prost<T> {
+    type Bytes = Vec<u8>;
+
+    fn to_bytes(&self) -> Result<Self::Bytes, Error> {
+        Ok(self.0.encode_to_vec())
+    }
+}
+
+#[cfg(feature = "prost")]
+impl<T: Default + prost::Message> FromBytesOwned for Prost<T> {
+    fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
+        Ok(Prost(T::decode(data)?))
+    }
+}
+
+/// Protobuf encoding
+///
+/// Allows for `rust-protobuf` Protobuf messages to be used as arguments to Extism plugin calls
+#[cfg(feature = "protobuf")]
+pub struct Protobuf<T: protobuf::Message>(pub T);
+
+#[cfg(feature = "protobuf")]
+impl<'a, T: protobuf::Message> ToBytes<'a> for Protobuf<T> {
+    type Bytes = Vec<u8>;
+
+    fn to_bytes(&self) -> Result<Self::Bytes, Error> {
+        Ok(self.0.write_to_bytes()?)
+    }
+}
+
+#[cfg(feature = "protobuf")]
+impl<T: Default + protobuf::Message> FromBytesOwned for Protobuf<T> {
+    fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
+        Ok(Protobuf(T::parse_from_bytes(data)?))
+    }
+}
+
+/// Raw does no conversion, it just copies the memory directly.
+/// Note: This will only work for types that implement [bytemuck::Pod](https://docs.rs/bytemuck/latest/bytemuck/trait.Pod.html)
+#[cfg(all(feature = "raw", target_endian = "little"))]
+pub struct Raw<'a, T: bytemuck::Pod>(pub &'a T);
+
+#[cfg(all(feature = "raw", target_endian = "little"))]
+impl<'a, T: bytemuck::Pod> ToBytes<'a> for Raw<'a, T> {
+    type Bytes = &'a [u8];
+
+    fn to_bytes(&self) -> Result<Self::Bytes, Error> {
+        Ok(bytemuck::bytes_of(self.0))
+    }
+}
+
+#[cfg(all(feature = "raw", target_endian = "little"))]
+impl<'a, T: bytemuck::Pod> FromBytes<'a> for Raw<'a, T> {
+    fn from_bytes(data: &'a [u8]) -> Result<Self, Error> {
+        let x = bytemuck::try_from_bytes(data).map_err(|x| Error::msg(x.to_string()))?;
+        Ok(Raw(x))
+    }
+}
+
+#[cfg(all(feature = "raw", target_endian = "big"))]
+compile_error!("The raw feature is only supported on little endian targets");

convert/src/from_bytes.rs

@@ -0,0 +1,164 @@
+use crate::*;
+
+pub use extism_convert_macros::FromBytes;
+
+/// `FromBytes` is used to define how a type should be decoded when working with
+/// Extism memory. It is used for plugin output and host function input.
+///
+/// `FromBytes` can be derived by delegating encoding to generic type implementing
+/// `FromBytes`, e.g., [`Json`], [`Msgpack`].
+///
+/// ```
+/// use extism_convert::{Json, FromBytes};
+/// use serde::Deserialize;
+///
+/// #[derive(FromBytes, Deserialize, PartialEq, Debug)]
+/// #[encoding(Json)]
+/// struct Struct {
+///     hello: String,
+/// }
+///
+/// assert_eq!(Struct::from_bytes(br#"{"hello":"hi"}"#)?, Struct { hello: "hi".into() });
+/// # Ok::<(), extism_convert::Error>(())
+/// ```
+///
+/// Custom encodings can also be used, through new-types with a single generic
+/// argument, i.e., `Type<T>(T)`, that implement `FromBytesOwned` for the struct.
+///
+/// ```
+/// use std::str::{self, FromStr};
+/// use std::convert::Infallible;
+/// use extism_convert::{Error, FromBytes, FromBytesOwned};
+///
+/// // Custom serialization using `FromStr`
+/// struct StringEnc<T>(T);
+/// impl<T: FromStr> FromBytesOwned for StringEnc<T> where Error: From<<T as FromStr>::Err> {
+///     fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
+///         Ok(Self(str::from_utf8(data)?.parse()?))
+///     }
+/// }
+///
+/// #[derive(FromBytes, PartialEq, Debug)]
+/// #[encoding(StringEnc)]
+/// struct Struct {
+///     hello: String,
+/// }
+///
+/// impl FromStr for Struct {
+///     type Err = Infallible;
+///     fn from_str(s: &str) -> Result<Self, Infallible> {
+///         Ok(Self { hello: s.to_owned() })
+///     }
+/// }
+///
+/// assert_eq!(Struct::from_bytes(b"hi")?, Struct { hello: "hi".into() });
+/// # Ok::<(), extism_convert::Error>(())
+/// ```
+pub trait FromBytes<'a>: Sized {
+    /// Decode a value from a slice of bytes
+    fn from_bytes(data: &'a [u8]) -> Result<Self, Error>;
+}
+
+/// `FromBytesOwned` is similar to [`FromBytes`] but it doesn't borrow from the input slice.
+/// [`FromBytes`] is automatically implemented for all types that implement `FromBytesOwned`.
+///
+/// `FromBytesOwned` can be derived through [`#[derive(FromBytes)]`](FromBytes).
+pub trait FromBytesOwned: Sized {
+    /// Decode a value from a slice of bytes, the resulting value should not borrow the input
+    /// data.
+    fn from_bytes_owned(data: &[u8]) -> Result<Self, Error>;
+}
+
+impl<'a> FromBytes<'a> for &'a [u8] {
+    fn from_bytes(data: &'a [u8]) -> Result<Self, Error> {
+        Ok(data)
+    }
+}
+
+impl<'a> FromBytes<'a> for &'a str {
+    fn from_bytes(data: &'a [u8]) -> Result<Self, Error> {
+        Ok(std::str::from_utf8(data)?)
+    }
+}
+
+impl<'a, T: FromBytesOwned> FromBytes<'a> for T {
+    fn from_bytes(data: &'a [u8]) -> Result<Self, Error> {
+        T::from_bytes_owned(data)
+    }
+}
+
+impl FromBytesOwned for Box<[u8]> {
+    fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
+        Ok(data.to_vec().into_boxed_slice())
+    }
+}
+
+impl FromBytesOwned for Vec<u8> {
+    fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
+        Ok(data.to_vec())
+    }
+}
+
+impl FromBytesOwned for String {
+    fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
+        Ok(std::str::from_utf8(data)?.to_string())
+    }
+}
+
+impl FromBytesOwned for f64 {
+    fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
+        Ok(Self::from_le_bytes(data.try_into()?))
+    }
+}
+
+impl FromBytesOwned for f32 {
+    fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
+        Ok(Self::from_le_bytes(data.try_into()?))
+    }
+}
+
+impl FromBytesOwned for i64 {
+    fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
+        Ok(Self::from_le_bytes(data.try_into()?))
+    }
+}
+
+impl FromBytesOwned for i32 {
+    fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
+        Ok(Self::from_le_bytes(data.try_into()?))
+    }
+}
+
+impl FromBytesOwned for u64 {
+    fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
+        Ok(Self::from_le_bytes(data.try_into()?))
+    }
+}
+
+impl FromBytesOwned for u32 {
+    fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
+        Ok(Self::from_le_bytes(data.try_into()?))
+    }
+}
+
+impl FromBytesOwned for () {
+    fn from_bytes_owned(_: &[u8]) -> Result<Self, Error> {
+        Ok(())
+    }
+}
+
+impl<'a, T: FromBytes<'a>> FromBytes<'a> for std::io::Cursor<T> {
+    fn from_bytes(data: &'a [u8]) -> Result<Self, Error> {
+        Ok(std::io::Cursor::new(T::from_bytes(data)?))
+    }
+}
+
+impl<'a, T: FromBytes<'a>> FromBytes<'a> for Option<T> {
+    fn from_bytes(data: &'a [u8]) -> Result<Self, Error> {
+        if data.is_empty() {
+            return Ok(None);
+        }
+
+        T::from_bytes(data).map(Some)
+    }
+}

convert/src/lib.rs

@@ -0,0 +1,38 @@
+//! The [extism-convert](https://crates.io/crates/extism-convert) crate is used by the [Rust SDK](https://crates.io/crates/extism) and [Rust PDK](https://crates.io/crates/extism-pdk) to provide a shared interface for
+//! encoding and decoding values that can be passed to Extism function calls.
+//!
+//! A set of types (Json, Msgpack) that can be used to specify a serde encoding are also provided. These are
+//! similar to [axum extractors](https://docs.rs/axum/latest/axum/extract/index.html#intro) - they are
+//! implemented as a tuple struct with a single field that is meant to be extracted using pattern matching.
+
+// Makes proc-macros able to resolve `::extism_convert` correctly
+extern crate self as extism_convert;
+
+pub use anyhow::Error;
+
+mod encoding;
+
+mod from_bytes;
+mod memory_handle;
+mod to_bytes;
+
+pub use encoding::{Base64, Json};
+
+#[cfg(feature = "msgpack")]
+pub use encoding::Msgpack;
+
+#[cfg(feature = "prost")]
+pub use encoding::Prost;
+
+#[cfg(feature = "protobuf")]
+pub use encoding::Protobuf;
+
+#[cfg(all(feature = "raw", target_endian = "little"))]
+pub use encoding::Raw;
+
+pub use from_bytes::{FromBytes, FromBytesOwned};
+pub use memory_handle::MemoryHandle;
+pub use to_bytes::ToBytes;
+
+#[cfg(test)]
+mod tests;

convert/src/memory_handle.rs

@@ -0,0 +1,42 @@
+/// `MemoryHandle` describes where in memory a block of data is stored
+#[derive(Debug, Clone, Copy, PartialEq, PartialOrd, Eq, Ord)]
+pub struct MemoryHandle {
+    /// The offset of the region in Extism linear memory
+    pub offset: u64,
+
+    /// The length of the memory region
+    pub length: u64,
+}
+
+impl MemoryHandle {
+    /// Create a new `MemoryHandle` from an offset in memory and length
+    ///
+    /// # Safety
+    /// This function is unsafe because the specified memory region may not be valid.
+    pub unsafe fn new(offset: u64, length: u64) -> MemoryHandle {
+        MemoryHandle { offset, length }
+    }
+
+    /// `NULL` equivalent
+    pub fn null() -> MemoryHandle {
+        MemoryHandle {
+            offset: 0,
+            length: 0,
+        }
+    }
+
+    /// Get the offset of a memory handle
+    pub fn offset(&self) -> u64 {
+        self.offset
+    }
+
+    /// Get the length of the memory region
+    pub fn len(&self) -> usize {
+        self.length as usize
+    }
+
+    /// Returns `true` when the length is 0
+    pub fn is_empty(&self) -> bool {
+        self.length == 0
+    }
+}

convert/src/tests.rs

@@ -0,0 +1,90 @@
+use crate::*;
+
+#[derive(serde::Serialize, serde::Deserialize, Debug, PartialEq)]
+struct Testing {
+    a: String,
+    b: i64,
+    c: f32,
+}
+
+#[test]
+fn roundtrip_json() {
+    let x = Testing {
+        a: "foobar".to_string(),
+        b: 123,
+        c: 456.7,
+    };
+    let bytes = Json(&x).to_bytes().unwrap();
+    let Json(y): Json<Testing> = FromBytes::from_bytes(&bytes).unwrap();
+    assert_eq!(x, y);
+}
+
+#[test]
+#[cfg(feature = "msgpack")]
+fn roundtrip_msgpack() {
+    let x = Testing {
+        a: "foobar".to_string(),
+        b: 123,
+        c: 456.7,
+    };
+    let bytes = Msgpack(&x).to_bytes().unwrap();
+    let Msgpack(y): Msgpack<Testing> = FromBytes::from_bytes(&bytes).unwrap();
+    assert_eq!(x, y);
+}
+
+#[test]
+fn roundtrip_base64() {
+    let bytes = Base64("this is a test").to_bytes().unwrap();
+    let Base64(s): Base64<String> = FromBytes::from_bytes(bytes.as_bytes()).unwrap();
+    assert_eq!(s, "this is a test");
+}
+
+#[test]
+fn rountrip_option() {
+    // `None` case
+    let e0: Option<Json<Testing>> = FromBytes::from_bytes(&[]).unwrap();
+    let b = e0.to_bytes().unwrap();
+    let e1: Option<Json<Testing>> = FromBytes::from_bytes(&b).unwrap();
+    assert!(e0.is_none());
+    assert_eq!(e0.is_none(), e1.is_none());
+
+    // `Some` case
+    let x = Testing {
+        a: "foobar".to_string(),
+        b: 123,
+        c: 456.7,
+    };
+    let bytes = Json(&x).to_bytes().unwrap();
+    let y: Option<Json<Testing>> = FromBytes::from_bytes(&bytes).unwrap();
+    let b = ToBytes::to_bytes(&y).unwrap();
+    let z: Option<Json<Testing>> = FromBytes::from_bytes(&b).unwrap();
+    assert_eq!(y.unwrap().0, z.unwrap().0);
+}
+
+#[cfg(all(feature = "raw", target_endian = "little"))]
+mod tests {
+    use crate::*;
+
+    #[test]
+    fn test_raw() {
+        #[derive(Debug, Clone, Copy, PartialEq)]
+        struct TestRaw {
+            a: i32,
+            b: f64,
+            c: bool,
+        }
+        unsafe impl bytemuck::Pod for TestRaw {}
+        unsafe impl bytemuck::Zeroable for TestRaw {}
+        let x = TestRaw {
+            a: 123,
+            b: 45678.91011,
+            c: true,
+        };
+        let raw = Raw(&x).to_bytes().unwrap();
+        let y = Raw::from_bytes(&raw).unwrap();
+        assert_eq!(&x, y.0);
+
+        let y: Result<Raw<[u8; std::mem::size_of::<TestRaw>()]>, Error> = Raw::from_bytes(&raw);
+        assert!(y.is_ok());
+    }
+}

convert/src/to_bytes.rs

@@ -0,0 +1,176 @@
+use crate::*;
+
+pub use extism_convert_macros::ToBytes;
+
+/// `ToBytes` is used to define how a type should be encoded when working with
+/// Extism memory. It is used for plugin input and host function output.
+///
+/// `ToBytes` can be derived by delegating encoding to generic type implementing
+/// `ToBytes`, e.g., [`Json`], [`Msgpack`].
+///
+/// ```
+/// use extism_convert::{Json, ToBytes};
+/// use serde::Serialize;
+///
+/// #[derive(ToBytes, Serialize)]
+/// #[encoding(Json)]
+/// struct Struct {
+///     hello: String,
+/// }
+///
+/// assert_eq!(Struct { hello: "hi".into() }.to_bytes()?, br#"{"hello":"hi"}"#);
+/// # Ok::<(), extism_convert::Error>(())
+/// ```
+///
+/// But custom types can also be used, as long as they are new-types with a single
+/// generic argument, i.e., `Type<T>(T)`, that implement `ToBytes` for the struct.
+///
+/// ```
+/// use extism_convert::{Error, ToBytes};
+///
+/// // Custom serialization using `ToString`
+/// struct StringEnc<T>(T);
+/// impl<T: ToString> ToBytes<'_> for StringEnc<&T> {
+///     type Bytes = String;
+///     
+///     fn to_bytes(&self) -> Result<String, Error> {
+///         Ok(self.0.to_string())
+///     }
+/// }
+///
+/// #[derive(ToBytes)]
+/// #[encoding(StringEnc)]
+/// struct Struct {
+///     hello: String,
+/// }
+///
+/// impl ToString for Struct {
+///    fn to_string(&self) -> String {
+///        self.hello.clone()     
+///    }
+/// }
+///
+/// assert_eq!(Struct { hello: "hi".into() }.to_bytes()?, b"hi");
+/// # Ok::<(), Error>(())
+/// ```
+pub trait ToBytes<'a> {
+    /// A configurable byte slice representation, allows any type that implements `AsRef<[u8]>`
+    type Bytes: AsRef<[u8]>;
+
+    /// `to_bytes` converts a value into `Self::Bytes`
+    fn to_bytes(&self) -> Result<Self::Bytes, Error>;
+}
+
+impl<'a> ToBytes<'a> for () {
+    type Bytes = [u8; 0];
+    fn to_bytes(&self) -> Result<Self::Bytes, Error> {
+        Ok([])
+    }
+}
+
+impl<'a> ToBytes<'a> for Vec<u8> {
+    type Bytes = Vec<u8>;
+    fn to_bytes(&self) -> Result<Self::Bytes, Error> {
+        Ok(self.clone())
+    }
+}
+
+impl<'a> ToBytes<'a> for String {
+    type Bytes = String;
+    fn to_bytes(&self) -> Result<Self::Bytes, Error> {
+        Ok(self.clone())
+    }
+}
+
+impl<'a> ToBytes<'a> for &'a [u8] {
+    type Bytes = &'a [u8];
+    fn to_bytes(&self) -> Result<Self::Bytes, Error> {
+        Ok(self)
+    }
+}
+
+impl<'a> ToBytes<'a> for &'a str {
+    type Bytes = &'a str;
+    fn to_bytes(&self) -> Result<Self::Bytes, Error> {
+        Ok(self)
+    }
+}
+
+impl<'a> ToBytes<'a> for f64 {
+    type Bytes = [u8; 8];
+
+    fn to_bytes(&self) -> Result<Self::Bytes, Error> {
+        Ok(self.to_le_bytes())
+    }
+}
+
+impl<'a> ToBytes<'a> for f32 {
+    type Bytes = [u8; 4];
+
+    fn to_bytes(&self) -> Result<Self::Bytes, Error> {
+        Ok(self.to_le_bytes())
+    }
+}
+
+impl<'a> ToBytes<'a> for i64 {
+    type Bytes = [u8; 8];
+
+    fn to_bytes(&self) -> Result<Self::Bytes, Error> {
+        Ok(self.to_le_bytes())
+    }
+}
+
+impl<'a> ToBytes<'a> for i32 {
+    type Bytes = [u8; 4];
+
+    fn to_bytes(&self) -> Result<Self::Bytes, Error> {
+        Ok(self.to_le_bytes())
+    }
+}
+
+impl<'a> ToBytes<'a> for u64 {
+    type Bytes = [u8; 8];
+
+    fn to_bytes(&self) -> Result<Self::Bytes, Error> {
+        Ok(self.to_le_bytes())
+    }
+}
+
+impl<'a> ToBytes<'a> for u32 {
+    type Bytes = [u8; 4];
+
+    fn to_bytes(&self) -> Result<Self::Bytes, Error> {
+        Ok(self.to_le_bytes())
+    }
+}
+
+impl<'a, T: ToBytes<'a>> ToBytes<'a> for &'a T {
+    type Bytes = T::Bytes;
+
+    fn to_bytes(&self) -> Result<Self::Bytes, Error> {
+        <T as ToBytes>::to_bytes(self)
+    }
+}
+
+impl<'a, T: ToBytes<'a>> ToBytes<'a> for Option<T> {
+    type Bytes = Vec<u8>;
+
+    fn to_bytes(&self) -> Result<Self::Bytes, Error> {
+        match self {
+            Some(x) => x.to_bytes().map(|x| x.as_ref().to_vec()),
+            None => Ok(vec![]),
+        }
+    }
+}
+
+#[test]
+fn test() {
+    use extism_convert::{Json, ToBytes};
+    use serde::Serialize;
+
+    #[derive(ToBytes, Serialize)]
+    #[encoding(Json)]
+    struct Struct {
+        hello: String,
+    }
+}

cpp/Makefile

@@ -1,15 +0,0 @@
-FLAGS=`pkg-config --cflags --libs jsoncpp gtest` -lextism -lpthread
-
-build-example:
-	$(CXX) -std=c++11 -o example -I. example.cpp $(FLAGS)
-	
-.PHONY: example
-example: build-example
-	./example
-
-build-test:
-	$(CXX) -std=c++11 -o test/test -I. test/test.cpp $(FLAGS)
-
-.PHONY: test
-test: build-test
-	cd test && ./test

cpp/example.cpp

@@ -1,28 +0,0 @@
-#define EXTISM_NO_JSON
-#include "extism.hpp"
-
-#include <cstring>
-#include <fstream>
-#include <iostream>
-
-using namespace extism;
-
-std::vector<uint8_t> read(const char *filename) {
-  std::ifstream file(filename, std::ios::binary);
-  return std::vector<uint8_t>((std::istreambuf_iterator<char>(file)),
-                              std::istreambuf_iterator<char>());
-}
-
-int main(int argc, char *argv[]) {
-  auto wasm = read("../wasm/code.wasm");
-  Context context = Context();
-
-  Plugin plugin = context.plugin(wasm);
-
-  const char *input = argc > 1 ? argv[1] : "this is a test";
-  ExtismSize length = strlen(input);
-
-  extism::Buffer output = plugin.call("count_vowels", (uint8_t *)input, length);
-  std::cout << (char *)output.data << std::endl;
-  return 0;
-}

cpp/extism.hpp

@@ -1,303 +0,0 @@
-#pragma once
-
-#include <map>
-#include <memory>
-#include <string>
-#include <vector>
-
-#ifndef EXTISM_NO_JSON
-#if __has_include(<jsoncpp/json/json.h>)
-#include <jsoncpp/json/json.h>
-#else
-#include <json/json.h>
-#endif
-#endif // EXTISM_NO_JSON
-
-extern "C" {
-#include <extism.h>
-}
-
-namespace extism {
-
-typedef std::map<std::string, std::string> Config;
-class Wasm {
-public:
-  std::string path;
-  std::string url;
-  // TODO: add base64 encoded raw data
-  std::string hash;
-
-#ifndef EXTISM_NO_JSON
-  Json::Value json() const {
-    Json::Value doc;
-
-    if (!this->path.empty()) {
-      doc["path"] = this->path;
-    }
-
-    if (!this->url.empty()) {
-      doc["url"] = this->url;
-    }
-
-    if (!this->hash.empty()) {
-      doc["hash"] = this->hash;
-    }
-
-    return doc;
-  }
-#endif
-};
-
-class Manifest {
-public:
-  Config config;
-  std::vector<Wasm> wasm;
-  std::vector<std::string> allowed_hosts;
-
-  static Manifest path(std::string s, std::string hash = std::string()) {
-    Manifest m;
-    m.add_wasm_path(s, hash);
-    return m;
-  }
-
-  static Manifest url(std::string s, std::string hash = std::string()) {
-    Manifest m;
-    m.add_wasm_url(s, hash);
-    return m;
-  }
-
-#ifndef EXTISM_NO_JSON
-  std::string json() const {
-    Json::Value doc;
-    Json::Value wasm;
-    for (auto w : this->wasm) {
-      wasm.append(w.json());
-    }
-
-    doc["wasm"] = wasm;
-
-    if (!this->config.empty()) {
-      Json::Value conf;
-
-      for (auto k : this->config) {
-        conf[k.first] = k.second;
-      }
-      doc["config"] = conf;
-    }
-
-    if (!this->allowed_hosts.empty()) {
-      Json::Value h;
-
-      for (auto s : this->allowed_hosts) {
-        h.append(s);
-      }
-      doc["allowed_hosts"] = h;
-    }
-
-    Json::FastWriter writer;
-    return writer.write(doc);
-  }
-#endif
-
-  void add_wasm_path(std::string s, std::string hash = std::string()) {
-    Wasm w;
-    w.path = s;
-    w.hash = hash;
-    this->wasm.push_back(w);
-  }
-
-  void add_wasm_url(std::string u, std::string hash = std::string()) {
-    Wasm w;
-    w.url = u;
-    w.hash = hash;
-    this->wasm.push_back(w);
-  }
-
-  void allow_host(std::string host) { this->allowed_hosts.push_back(host); }
-
-  void set_config(std::string k, std::string v) { this->config[k] = v; }
-};
-
-class Error : public std::exception {
-private:
-  std::string message;
-
-public:
-  Error(std::string msg) : message(msg) {}
-  const char *what() { return message.c_str(); }
-};
-
-class Buffer {
-public:
-  Buffer(const uint8_t *ptr, ExtismSize len) : data(ptr), length(len) {}
-  const uint8_t *data;
-  ExtismSize length;
-
-  std::string string() { return (std::string)(*this); }
-
-  std::vector<uint8_t> vector() { return (std::vector<uint8_t>)(*this); }
-
-  operator std::string() { return std::string((const char *)data, length); }
-  operator std::vector<uint8_t>() {
-    return std::vector<uint8_t>(data, data + length);
-  }
-};
-
-class Plugin {
-  std::shared_ptr<ExtismContext> context;
-  ExtismPlugin plugin;
-
-public:
-  Plugin(std::shared_ptr<ExtismContext> ctx, const uint8_t *wasm,
-         ExtismSize length, bool with_wasi = false) {
-    this->plugin = extism_plugin_new(ctx.get(), wasm, length, with_wasi);
-    if (this->plugin < 0) {
-      const char *err = extism_error(ctx.get(), -1);
-      throw Error(err == nullptr ? "Unable to load plugin" : err);
-    }
-    this->context = ctx;
-  }
-
-#ifndef EXTISM_NO_JSON
-  Plugin(std::shared_ptr<ExtismContext> ctx, const Manifest &manifest,
-         bool with_wasi = false) {
-    auto buffer = manifest.json();
-    this->plugin = extism_plugin_new(ctx.get(), (const uint8_t *)buffer.c_str(),
-                                     buffer.size(), with_wasi);
-    if (this->plugin < 0) {
-      const char *err = extism_error(ctx.get(), -1);
-      throw Error(err == nullptr ? "Unable to load plugin from manifest" : err);
-    }
-    this->context = ctx;
-  }
-#endif
-
-  ~Plugin() {
-    extism_plugin_free(this->context.get(), this->plugin);
-    this->plugin = -1;
-  }
-
-  ExtismPlugin id() const { return this->plugin; }
-
-  ExtismContext *get_context() const { return this->context.get(); }
-
-  void update(const uint8_t *wasm, size_t length, bool with_wasi = false) {
-    bool b = extism_plugin_update(this->context.get(), this->plugin, wasm,
-                                  length, with_wasi);
-    if (!b) {
-      const char *err = extism_error(this->context.get(), -1);
-      throw Error(err == nullptr ? "Unable to update plugin" : err);
-    }
-  }
-
-#ifndef EXTISM_NO_JSON
-  void update(const Manifest &manifest, bool with_wasi = false) {
-    auto buffer = manifest.json();
-    bool b = extism_plugin_update(this->context.get(), this->plugin,
-                                  (const uint8_t *)buffer.c_str(),
-                                  buffer.size(), with_wasi);
-    if (!b) {
-      const char *err = extism_error(this->context.get(), -1);
-      throw Error(err == nullptr ? "Unable to update plugin" : err);
-    }
-  }
-
-  void config(const Config &data) {
-    Json::Value conf;
-
-    for (auto k : data) {
-      conf[k.first] = k.second;
-    }
-
-    Json::FastWriter writer;
-    auto s = writer.write(conf);
-    this->config(s);
-  }
-#endif
-
-  void config(const char *json, size_t length) {
-    bool b = extism_plugin_config(this->context.get(), this->plugin,
-                                  (const uint8_t *)json, length);
-    if (!b) {
-      const char *err = extism_error(this->context.get(), this->plugin);
-      throw Error(err == nullptr ? "Unable to update plugin config" : err);
-    }
-  }
-
-  void config(const std::string &json) {
-    this->config(json.c_str(), json.size());
-  }
-
-  Buffer call(const std::string &func, const uint8_t *input,
-              ExtismSize input_length) const {
-    int32_t rc = extism_plugin_call(this->context.get(), this->plugin,
-                                    func.c_str(), input, input_length);
-    if (rc != 0) {
-      const char *error = extism_error(this->context.get(), this->plugin);
-      if (error == nullptr) {
-        throw Error("extism_call failed");
-      }
-
-      throw Error(error);
-    }
-
-    ExtismSize length =
-        extism_plugin_output_length(this->context.get(), this->plugin);
-    const uint8_t *ptr =
-        extism_plugin_output_data(this->context.get(), this->plugin);
-    return Buffer(ptr, length);
-  }
-
-  Buffer call(const std::string &func,
-              const std::vector<uint8_t> &input) const {
-    return this->call(func, input.data(), input.size());
-  }
-
-  Buffer call(const std::string &func, const std::string &input) const {
-    return this->call(func, (const uint8_t *)input.c_str(), input.size());
-  }
-
-  bool function_exists(const std::string &func) const {
-    return extism_plugin_function_exists(this->context.get(), this->plugin,
-                                         func.c_str());
-  }
-};
-
-class Context {
-public:
-  std::shared_ptr<ExtismContext> pointer;
-  Context() {
-    this->pointer = std::shared_ptr<ExtismContext>(extism_context_new(),
-                                                   extism_context_free);
-  }
-
-  Plugin plugin(const uint8_t *wasm, size_t length,
-                bool with_wasi = false) const {
-    return Plugin(this->pointer, wasm, length, with_wasi);
-  }
-
-  Plugin plugin(const std::string &str, bool with_wasi = false) const {
-    return Plugin(this->pointer, (const uint8_t *)str.c_str(), str.size(),
-                  with_wasi);
-  }
-
-  Plugin plugin(const std::vector<uint8_t> &data,
-                bool with_wasi = false) const {
-    return Plugin(this->pointer, data.data(), data.size(), with_wasi);
-  }
-
-#ifndef EXTISM_NO_JSON
-  Plugin plugin(const Manifest &manifest, bool with_wasi = false) const {
-    return Plugin(this->pointer, manifest, with_wasi);
-  }
-#endif
-
-  void reset() { extism_context_reset(this->pointer.get()); }
-};
-
-inline bool set_log_file(const char *filename, const char *level) {
-  return extism_log_file(filename, level);
-}
-
-inline std::string version() { return std::string(extism_version()); }
-} // namespace extism

cpp/test/test.cpp

@@ -1,73 +0,0 @@
-#include "../extism.hpp"
-
-#include <fstream>
-
-#include <gtest/gtest.h>
-
-std::vector<uint8_t> read(const char *filename) {
-  std::ifstream file(filename, std::ios::binary);
-  return std::vector<uint8_t>((std::istreambuf_iterator<char>(file)),
-                              std::istreambuf_iterator<char>());
-}
-
-namespace {
-using namespace extism;
-
-TEST(Context, Basic) {
-  Context context;
-  ASSERT_NE(context.pointer, nullptr);
-}
-
-TEST(Plugin, Manifest) {
-  Context context;
-  Manifest manifest = Manifest::path("code.wasm");
-  manifest.set_config("a", "1");
-
-  ASSERT_NO_THROW(Plugin plugin = context.plugin(manifest));
-  Plugin plugin = context.plugin(manifest);
-
-  Buffer buf = plugin.call("count_vowels", "this is a test");
-  ASSERT_EQ((std::string)buf, "{\"count\": 4}");
-}
-
-TEST(Plugin, BadManifest) {
-  Context context;
-  Manifest manifest;
-  ASSERT_THROW(Plugin plugin = context.plugin(manifest), Error);
-}
-
-TEST(Plugin, Bytes) {
-  Context context;
-  auto wasm = read("code.wasm");
-  ASSERT_NO_THROW(Plugin plugin = context.plugin(wasm));
-  Plugin plugin = context.plugin(wasm);
-
-  Buffer buf = plugin.call("count_vowels", "this is another test");
-  ASSERT_EQ(buf.string(), "{\"count\": 6}");
-}
-
-TEST(Plugin, UpdateConfig) {
-  Context context;
-  auto wasm = read("code.wasm");
-  Plugin plugin = context.plugin(wasm);
-
-  Config config;
-  config["abc"] = "123";
-  ASSERT_NO_THROW(plugin.config(config));
-}
-
-TEST(Plugin, FunctionExists) {
-  Context context;
-  auto wasm = read("code.wasm");
-  Plugin plugin = context.plugin(wasm);
-
-  ASSERT_FALSE(plugin.function_exists("bad_function"));
-  ASSERT_TRUE(plugin.function_exists("count_vowels"));
-}
-
-}; // namespace
-
-int main(int argc, char **argv) {
-  testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}

dune

@@ -1 +0,0 @@
-(dirs ocaml)

dune-project

@@ -1,24 +0,0 @@
-(lang dune 3.2)
-
-(name extism)
-
-(generate_opam_files true)
-
-(source
- (github extism/extism))
-
-(authors "Extism Authors <oss@extism.org>")
-
-(maintainers "Extism Authors <oss@extism.org>")
-
-(license BSD-3-Clause)
-
-(documentation https://github.com/extism/extism)
-
-(package
- (name extism)
- (synopsis "Extism bindings")
- (description "Bindings to Extism, the universal plugin system")
- (depends ocaml dune ctypes-foreign bigstringaf ppx_yojson_conv base64 ppx_inline_test)
- (tags
-  (topics wasm plugin)))

elixir/.formatter.exs

@@ -1,4 +0,0 @@
-# Used by "mix format"
-[
-  inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"]
-]

elixir/.gitignore

@@ -1,28 +0,0 @@
-# The directory Mix will write compiled artifacts to.
-/_build/
-
-# If you run "mix test --cover", coverage assets end up here.
-/cover/
-
-# The directory Mix downloads your dependencies sources to.
-/deps/
-
-# Where third-party dependencies like ExDoc output generated docs.
-/doc/
-
-# Ignore .fetch files in case you like to edit your project deps locally.
-/.fetch
-
-# If the VM crashes, it generates a dump, let's ignore it too.
-erl_crash.dump
-
-# Also ignore archive artifacts (built via "mix archive.build").
-*.ez
-
-# Ignore package tarball (built via "mix hex.build").
-extism-*.tar
-
-# Temporary files, for example, from tests.
-/tmp/
-
-/priv/

elixir/Makefile

@@ -1,27 +0,0 @@
-.PHONY: test
-
-prepare:
-	mix deps.get
-	mix compile
-
-test: prepare
-	mix test
-
-clean:
-	mix clean
-
-publish: clean prepare
-	mix hex.build
-	mix hex.publish --yes
-
-format:
-	mix format
-
-lint:
-	mix format --check-formatted
-
-docs:
-	mix docs
-
-show-docs: docs
-	open doc/index.html

elixir/README.md

@@ -1,73 +0,0 @@
-# Extism
-
-Extism Host SDK for Elixir and Erlang
-
-## Docs
-
-Read the [docs on hexdocs.pm](https://hexdocs.pm/extism/).
-
-## Installation
-
-You can find this package on [hex.pm](https://hex.pm/packages/extism).
-
-```elixir
-def deps do
-  [
-    {:extism, "~> 0.0.1-rc.6"}
-  ]
-end
-```
-
-## Getting Started
-
-### Example
-
-```elixir
-# Create a context for which plugins can be allocated and cleaned
-ctx = Extism.Context.new()
-
-# point to some wasm code, this is the count_vowels example that ships with extism
-manifest = %{ wasm: [ %{ path: "/Users/ben/code/extism/wasm/code.wasm" } ]}
-{:ok, plugin} = Extism.Context.new_plugin(ctx, manifest, false)
-# {:ok,
-# %Extism.Plugin{
-#   resource: 0,
-#   reference: #Reference<0.520418104.1263009793.80956>
-# }}
-{:ok, output} = Extism.Plugin.call(plugin, "count_vowels", "this is a test")
-# {:ok, "{\"count\": 4}"}
-{:ok, result} = JSON.decode(output)
-# {:ok, %{"count" => 4}}
-
-# free up the context and any plugins we allocated
-Extism.Context.free(ctx)
-```
-
-### Modules
-
-The two primary modules you should learn are:
-
-* [Extism.Context](Extism.Context.html)
-* [Extism.Plugin](Extism.Plugin.html)
-
-#### Context
-
-The [Context](Extism.Context.html) can be thought of as a session. You need a context to interact with the Extism runtime. The context holds your plugins and when you free the context, it frees your plugins. It's important to free up your context and plugins when you are done with them.
-
-```elixir
-ctx = Extism.Context.new()
-# frees all the plugins 
-Extism.Context.reset(ctx)
-# frees the context and all its plugins
-Extism.Context.free(ctx)
-```
-
-#### Plugin
-
-The [Plugin](Extism.Plugin.html) represents an instance of your WASM program from the given manifest.
-The key method to know here is [Extism.Plugin#call](Extism.Plugin.html#call/3) which takes a function name to invoke and some input data, and returns the results from the plugin.
-
-```elixir
-{:ok, plugin} = Extism.Context.new_plugin(ctx, manifest, false)
-{:ok, output} = Extism.Plugin.call(plugin, "count_vowels", "this is a test")
-```

elixir/lib/extism.ex

@@ -1,5 +0,0 @@
-defmodule Extism do
-  def set_log_file(filepath, level) do
-    Extism.Native.set_log_file(filepath, level)
-  end
-end

elixir/lib/extism/context.ex

@@ -1,64 +0,0 @@
-defmodule Extism.Context do
-  @moduledoc """
-  A Context is needed to create plugins. The Context is where your plugins
-  live. Freeing the context frees all of the plugins in its scope.
-  """
-
-  defstruct [
-    # The actual NIF Resource. A pointer in this case
-    ptr: nil
-  ]
-
-  def wrap_resource(ptr) do
-    %__MODULE__{
-      ptr: ptr
-    }
-  end
-
-  @doc """
-  Creates a new context.
-  """
-  def new() do
-    ptr = Extism.Native.context_new()
-    Extism.Context.wrap_resource(ptr)
-  end
-
-  @doc """
-  Resets the context. This has the effect of freeing all the plugins created so far.
-  """
-  def reset(ctx) do
-    Extism.Native.context_reset(ctx.ptr)
-  end
-
-  @doc """
-  Frees the context from memory and all of its plugins.
-  """
-  def free(ctx) do
-    Extism.Native.context_free(ctx.ptr)
-  end
-
-  @doc """
-  Create a new plugin from a WASM module or manifest
-
-  ## Examples:
-
-      iex> ctx = Extism.Context.new()
-      iex> manifest = %{ wasm: [ %{ path: "/Users/ben/code/extism/wasm/code.wasm" } ]}
-      iex> {:ok, plugin} =  Extism.Context.new_plugin(ctx, manifest, false)
-
-  ## Parameters
-
-    - ctx: The Context to manage this plugin
-    - manifest: The String or Map of the WASM module or [manifest](https://extism.org/docs/concepts/manifest)
-    - wasi: A bool you set to true if you want WASI support
-
-  """
-  def new_plugin(ctx, manifest, wasi \\ false) do
-    {:ok, manifest_payload} = JSON.encode(manifest)
-
-    case Extism.Native.plugin_new_with_manifest(ctx.ptr, manifest_payload, wasi) do
-      {:error, err} -> {:error, err}
-      res -> {:ok, Extism.Plugin.wrap_resource(ctx, res)}
-    end
-  end
-end

elixir/lib/extism/native.ex

@@ -1,21 +0,0 @@
-defmodule Extism.Native do
-  @moduledoc """
-  This module represents the Native Extism runtime API and is for internal use.
-  Do not use or rely on this this module.
-  """
-  use Rustler,
-    otp_app: :extism,
-    crate: :extism_nif
-
-  def context_new(), do: error()
-  def context_reset(_ctx), do: error()
-  def context_free(_ctx), do: error()
-  def plugin_new_with_manifest(_ctx, _manifest, _wasi), do: error()
-  def plugin_call(_ctx, _plugin_id, _name, _input), do: error()
-  def plugin_update_manifest(_ctx, _plugin_id, _manifest, _wasi), do: error()
-  def plugin_has_function(_ctx, _plugin_id, _function_name), do: error()
-  def plugin_free(_ctx, _plugin_id), do: error()
-  def set_log_file(_filename, _level), do: error()
-
-  defp error, do: :erlang.nif_error(:nif_not_loaded)
-end

elixir/lib/extism/plugin.ex

@@ -1,91 +0,0 @@
-defmodule Extism.Plugin do
-  @moduledoc """
-  A Plugin represents an instance of your WASM program from the given manifest.
-  """
-  defstruct [
-    # The actual NIF Resource. PluginIndex and the context
-    plugin_id: nil,
-    ctx: nil
-  ]
-
-  def wrap_resource(ctx, plugin_id) do
-    %__MODULE__{
-      ctx: ctx,
-      plugin_id: plugin_id
-    }
-  end
-
-  @doc """
-  Call a plugin's function by name
-
-  ## Examples
-
-      iex> {:ok, plugin} = Extism.Context.new_plugin(ctx, manifest, false)
-      iex> {:ok, output} = Extism.Plugin.call(plugin, "count_vowels", "this is a test")
-         # {:ok, "{\"count\": 4}"}
-
-  ## Parameters
-
-    - plugin: The plugin
-    - name: The name of the function as a string
-    - input: The input data as a string
-
-  ## Returns
-
-    A string representation of the functions output
-
-  """
-  def call(plugin, name, input) do
-    case Extism.Native.plugin_call(plugin.ctx.ptr, plugin.plugin_id, name, input) do
-      {:error, err} -> {:error, err}
-      res -> {:ok, res}
-    end
-  end
-
-  @doc """
-  Updates the manifest of the given plugin
-
-  ## Parameters
-
-    - ctx: The Context to manage this plugin
-    - manifest: The String or Map of the WASM module or [manifest](https://extism.org/docs/concepts/manifest)
-    - wasi: A bool you set to true if you want WASI support
-
-
-  """
-  def update(plugin, manifest, wasi) when is_map(manifest) do
-    {:ok, manifest_payload} = JSON.encode(manifest)
-
-    case Extism.Native.plugin_update_manifest(
-           plugin.ctx.ptr,
-           plugin.plugin_id,
-           manifest_payload,
-           wasi
-         ) do
-      {:error, err} -> {:error, err}
-      _ -> :ok
-    end
-  end
-
-  @doc """
-  Frees the plugin
-  """
-  def free(plugin) do
-    Extism.Native.plugin_free(plugin.ctx.ptr, plugin.plugin_id)
-  end
-
-  @doc """
-  Returns true if the given plugin responds to the given function name
-  """
-  def has_function(plugin, function_name) do
-    Extism.Native.plugin_has_function(plugin.ctx.ptr, plugin.plugin_id, function_name)
-  end
-end
-
-defimpl Inspect, for: Extim.Plugin do
-  import Inspect.Algebra
-
-  def inspect(dict, opts) do
-    concat(["#Extism.Plugin<", to_doc(dict.plugin_id, opts), ">"])
-  end
-end

elixir/mix.exs

@@ -1,59 +0,0 @@
-defmodule Extism.MixProject do
-  use Mix.Project
-
-  def project do
-    [
-      app: :extism,
-      version: "0.0.1-rc.6",
-      elixir: "~> 1.14",
-      start_permanent: Mix.env() == :prod,
-      deps: deps(),
-      package: package(),
-      aliases: aliases(),
-      docs: docs()
-    ]
-  end
-
-  # Run "mix help compile.app" to learn about applications.
-  def application do
-    [
-      extra_applications: [:logger]
-    ]
-  end
-
-  defp deps do
-    [
-      {:rustler, "~> 0.26.0"},
-      {:json, "~> 1.4"},
-      {:ex_doc, "~> 0.21", only: :dev, runtime: false}
-    ]
-  end
-
-  defp aliases do
-    [
-      fmt: [
-        "format",
-        "cmd cargo fmt --manifest-path native/io/Cargo.toml"
-      ]
-    ]
-  end
-
-  defp package do
-    [
-      licenses: ["BSD-3-Clause"],
-      description: "Extism Host SDK for Elixir and Erlang",
-      name: "extism",
-      files: ~w(lib native priv .formatter.exs mix.exs README.md LICENSE),
-      links: %{"GitHub" => "https://github.com/extism/extism"}
-    ]
-  end
-
-  defp docs do
-    [
-      main: "Extism",
-      logo: "./logo.png",
-      main: "readme",
-      extras: ["README.md"]
-    ]
-  end
-end

elixir/mix.lock

@@ -1,12 +0,0 @@
-%{
-  "earmark_parser": {:hex, :earmark_parser, "1.4.29", "149d50dcb3a93d9f3d6f3ecf18c918fb5a2d3c001b5d3305c926cddfbd33355b", [:mix], [], "hexpm", "4902af1b3eb139016aed210888748db8070b8125c2342ce3dcae4f38dcc63503"},
-  "ex_doc": {:hex, :ex_doc, "0.29.0", "4a1cb903ce746aceef9c1f9ae8a6c12b742a5461e6959b9d3b24d813ffbea146", [:mix], [{:earmark_parser, "~> 1.4.19", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "f096adb8bbca677d35d278223361c7792d496b3fc0d0224c9d4bc2f651af5db1"},
-  "jason": {:hex, :jason, "1.4.0", "e855647bc964a44e2f67df589ccf49105ae039d4179db7f6271dfd3843dc27e6", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "79a3791085b2a0f743ca04cec0f7be26443738779d09302e01318f97bdb82121"},
-  "json": {:hex, :json, "1.4.1", "8648f04a9439765ad449bc56a3ff7d8b11dd44ff08ffcdefc4329f7c93843dfa", [:mix], [], "hexpm", "9abf218dbe4ea4fcb875e087d5f904ef263d012ee5ed21d46e9dbca63f053d16"},
-  "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"},
-  "makeup_elixir": {:hex, :makeup_elixir, "0.16.0", "f8c570a0d33f8039513fbccaf7108c5d750f47d8defd44088371191b76492b0b", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "28b2cbdc13960a46ae9a8858c4bebdec3c9a6d7b4b9e7f4ed1502f8159f338e7"},
-  "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"},
-  "nimble_parsec": {:hex, :nimble_parsec, "1.2.3", "244836e6e3f1200c7f30cb56733fd808744eca61fd182f731eac4af635cc6d0b", [:mix], [], "hexpm", "c8d789e39b9131acf7b99291e93dae60ab48ef14a7ee9d58c6964f59efb570b0"},
-  "rustler": {:hex, :rustler, "0.26.0", "06a2773d453ee3e9109efda643cf2ae633dedea709e2455ac42b83637c9249bf", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "42961e9d2083d004d5a53e111ad1f0c347efd9a05cb2eb2ffa1d037cdc74db91"},
-  "toml": {:hex, :toml, "0.6.2", "38f445df384a17e5d382befe30e3489112a48d3ba4c459e543f748c2f25dd4d1", [:mix], [], "hexpm", "d013e45126d74c0c26a38d31f5e8e9b83ea19fc752470feb9a86071ca5a672fa"},
-}

elixir/native/extism_nif/.cargo/config

@@ -1,15 +0,0 @@
-[target.'cfg(target_os = "macos")']
-rustflags = [
-    "-C", "link-arg=-undefined",
-    "-C", "link-arg=dynamic_lookup",
-]
-
-# See https://github.com/rust-lang/rust/issues/59302
-[target.x86_64-unknown-linux-musl]
-rustflags = [
-  "-C", "target-feature=-crt-static"
-]
-
-# Provides a small build size, but takes more time to build.
-[profile.release]
-lto = true

elixir/native/extism_nif/Cargo.toml

@@ -1,15 +0,0 @@
-[package]
-name = "extism_nif"
-version = "0.0.1-rc.6"
-edition = "2021"
-authors = ["Benjamin Eckel <bhelx@simst.im>"]
-
-[lib]
-name = "extism_nif"
-path = "src/lib.rs"
-crate-type = ["cdylib"]
-
-[dependencies]
-rustler = "0.26.0"
-extism = { version = "0.0.1-rc.5" }
-log = "0.4"

elixir/native/extism_nif/src/lib.rs

@@ -1,149 +0,0 @@
-use rustler::{Atom, Env, Term, ResourceArc};
-use extism::{Plugin, Context};
-use std::str;
-use std::path::Path;
-use std::str::FromStr;
-use std::sync::RwLock;
-use std::mem;
-
-mod atoms {
-    rustler::atoms! {
-        ok,
-        error,
-        unknown // Other error
-    }
-}
-
-struct ExtismContext {
-    ctx: RwLock<Context>
-}
-
-fn load(env: Env, _: Term) -> bool {
-    rustler::resource!(ExtismContext, env);
-    true
-}
-
-fn to_rustler_error(extism_error: extism::Error) -> rustler::Error {
-    match extism_error {
-        extism::Error::UnableToLoadPlugin(msg) => rustler::Error::Term(Box::new(msg)),
-        extism::Error::Message(msg) => rustler::Error::Term(Box::new(msg)),
-        extism::Error::Json(json_err) => rustler::Error::Term(Box::new(json_err.to_string()))
-    }
-}
-
-#[rustler::nif]
-fn context_new() -> ResourceArc<ExtismContext> {
-    ResourceArc::new(
-        ExtismContext { ctx: RwLock::new(Context::new()) }
-    )
-}
-
-#[rustler::nif]
-fn context_reset(ctx: ResourceArc<ExtismContext>) {
-    let context = &mut ctx.ctx.write().unwrap();
-    context.reset()
-}
-
-#[rustler::nif]
-fn context_free(ctx: ResourceArc<ExtismContext>) {
-    let context = &ctx.ctx.read().unwrap();
-    std::mem::drop(context)
-}
-
-#[rustler::nif]
-fn plugin_new_with_manifest(ctx: ResourceArc<ExtismContext>, manifest_payload: String, wasi: bool) -> Result<i32, rustler::Error> {
-    let context = &ctx.ctx.write().unwrap();
-    let result = match Plugin::new(context, manifest_payload, wasi) {
-        Err(e) => Err(to_rustler_error(e)),
-        Ok(plugin) => {
-            let plugin_id = plugin.as_i32();
-            // this forget should be safe because the context will clean up
-            // all it's plugins when it is dropped
-            mem::forget(plugin);
-            Ok(plugin_id)
-        }
-    };
-    result
-}
-
-#[rustler::nif]
-fn plugin_call(ctx: ResourceArc<ExtismContext>, plugin_id: i32, name: String, input: String) -> Result<String, rustler::Error> {
-    let context = &ctx.ctx.read().unwrap();
-    let plugin = unsafe { Plugin::from_id(plugin_id, context) };
-    let result = match plugin.call(name, input) {
-        Err(e) => Err(to_rustler_error(e)),
-        Ok(result) => {
-            match str::from_utf8(&result) {
-                Ok(output) => Ok(output.to_string()),
-                Err(_e) => Err(rustler::Error::Term(Box::new("Could not read output from plugin")))
-            }
-        }
-    };
-    // this forget should be safe because the context will clean up
-    // all it's plugins when it is dropped
-    mem::forget(plugin);
-    result
-}
-
-#[rustler::nif]
-fn plugin_update_manifest(ctx: ResourceArc<ExtismContext>, plugin_id: i32, manifest_payload: String, wasi: bool) -> Result<(), rustler::Error> {
-    let context = &ctx.ctx.read().unwrap();
-    let mut plugin = unsafe { Plugin::from_id(plugin_id, context) };
-    let result = match plugin.update(manifest_payload, wasi) {
-        Ok(()) => {
-            Ok(())
-        },
-        Err(e) => Err(to_rustler_error(e))
-    };
-    // this forget should be safe because the context will clean up
-    // all it's plugins when it is dropped
-    mem::forget(plugin);
-    result
-}
-
-#[rustler::nif]
-fn plugin_free(ctx: ResourceArc<ExtismContext>, plugin_id: i32) -> Result<(), rustler::Error> {
-    let context = &ctx.ctx.read().unwrap();
-    let plugin = unsafe { Plugin::from_id(plugin_id, context) };
-    std::mem::drop(plugin);
-    Ok(())
-}
-
-#[rustler::nif]
-fn set_log_file(filename: String, log_level: String) -> Result<Atom, rustler::Error> {
-    let path = Path::new(&filename);
-    match log::Level::from_str(&log_level) {
-        Err(_e) => Err(rustler::Error::Term(Box::new(format!("{} not a valid log level", log_level)))),
-        Ok(level) => {
-            extism::set_log_file(path, Some(level));
-            Ok(atoms::ok())
-        }
-    }
-}
-
-#[rustler::nif]
-fn plugin_has_function(ctx: ResourceArc<ExtismContext>, plugin_id: i32, function_name: String) -> Result<bool, rustler::Error> {
-    let context = &ctx.ctx.read().unwrap();
-    let plugin = unsafe { Plugin::from_id(plugin_id, context) };
-    let has_function = plugin.has_function(function_name);
-    // this forget should be safe because the context will clean up
-    // all it's plugins when it is dropped
-    mem::forget(plugin);
-    Ok(has_function)
-}
-
-rustler::init!(
-    "Elixir.Extism.Native",
-    [
-        context_new,
-        context_reset,
-        context_free,
-        plugin_new_with_manifest,
-        plugin_call,
-        plugin_update_manifest,
-        plugin_has_function,
-        plugin_free,
-        set_log_file,
-    ],
-    load = load
-);

elixir/test/extism_test.exs

@@ -1,81 +0,0 @@
-defmodule ExtismTest do
-  use ExUnit.Case
-  doctest Extism
-
-  test "context create & reset" do
-    ctx = Extism.Context.new()
-    path = Path.join([__DIR__, "../../wasm/code.wasm"])
-    manifest = %{wasm: [%{path: path}]}
-    {:ok, plugin} = Extism.Context.new_plugin(ctx, manifest, false)
-    Extism.Context.reset(ctx)
-    # we should expect an error after resetting context
-    {:error, _err} = Extism.Plugin.call(plugin, "count_vowels", "this is a test")
-  end
-
-  defp new_plugin() do
-    ctx = Extism.Context.new()
-    path = Path.join([__DIR__, "../../wasm/code.wasm"])
-    manifest = %{wasm: [%{path: path}]}
-    {:ok, plugin} = Extism.Context.new_plugin(ctx, manifest, false)
-    {ctx, plugin}
-  end
-
-  test "counts vowels" do
-    {ctx, plugin} = new_plugin()
-    {:ok, output} = Extism.Plugin.call(plugin, "count_vowels", "this is a test")
-    assert JSON.decode(output) == {:ok, %{"count" => 4}}
-    Extism.Context.free(ctx)
-  end
-
-  test "can make multiple calls on a plugin" do
-    {ctx, plugin} = new_plugin()
-    {:ok, output} = Extism.Plugin.call(plugin, "count_vowels", "this is a test")
-    assert JSON.decode(output) == {:ok, %{"count" => 4}}
-    {:ok, output} = Extism.Plugin.call(plugin, "count_vowels", "this is a test again")
-    assert JSON.decode(output) == {:ok, %{"count" => 7}}
-    {:ok, output} = Extism.Plugin.call(plugin, "count_vowels", "this is a test thrice")
-    assert JSON.decode(output) == {:ok, %{"count" => 6}}
-    Extism.Context.free(ctx)
-  end
-
-  test "can free a plugin" do
-    {ctx, plugin} = new_plugin()
-    {:ok, output} = Extism.Plugin.call(plugin, "count_vowels", "this is a test")
-    assert JSON.decode(output) == {:ok, %{"count" => 4}}
-    Extism.Plugin.free(plugin)
-    # Expect an error when calling a plugin that was freed
-    {:error, _err} = Extism.Plugin.call(plugin, "count_vowels", "this is a test")
-    Extism.Context.free(ctx)
-  end
-
-  test "can update manifest" do
-    {ctx, plugin} = new_plugin()
-    path = Path.join([__DIR__, "../../wasm/code.wasm"])
-    manifest = %{wasm: [%{path: path}]}
-    assert Extism.Plugin.update(plugin, manifest, true) == :ok
-    Extism.Context.free(ctx)
-  end
-
-  test "errors on bad manifest" do
-    ctx = Extism.Context.new()
-    {:error, _msg} = Extism.Context.new_plugin(ctx, %{"wasm" => 123}, false)
-    Extism.Context.free(ctx)
-  end
-
-  test "errors on unknown function" do
-    {ctx, plugin} = new_plugin()
-    {:error, _msg} = Extism.Plugin.call(plugin, "unknown", "this is a test")
-    Extism.Context.free(ctx)
-  end
-
-  test "set_log_file" do
-    Extism.set_log_file("/tmp/logfile.log", "debug")
-  end
-
-  test "has_function" do
-    {ctx, plugin} = new_plugin()
-    assert Extism.Plugin.has_function(plugin, "count_vowels")
-    assert !Extism.Plugin.has_function(plugin, "unknown")
-    Extism.Context.free(ctx)
-  end
-end

elixir/test/test_helper.exs

@@ -1 +0,0 @@
-ExUnit.start()

extism-maturin/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "extism-sys"
+edition.workspace = true
+license.workspace = true
+homepage.workspace = true
+repository.workspace = true
+version.workspace = true
+# Explicitly omit authors from this package since our Cargo "authors" are
+# incompatible with PyPI's requirements.
+# authors.workspace = true
+
+[lib]
+name = "extism_sys"
+crate-type = ["cdylib"]
+
+[dependencies]
+extism = { workspace = true }
+
+[build-dependencies]
+cc = "1"

extism-maturin/build.rs

@@ -0,0 +1,57 @@
+use std::borrow::Cow;
+
+fn rewrite(line: &'_ str) -> Option<Cow<'_, str>> {
+    let line = line.trim();
+
+    if line.is_empty() {
+        return None;
+    }
+
+    if line.starts_with('#') {
+        return None;
+    }
+
+    if cfg!(target_os = "macos") {
+        if line.starts_with("typedef __builtin_va_list ") {
+            return None;
+        }
+    } else if cfg!(target_os = "windows") {
+        if line.contains("__gnuc_va_list")
+            || line.starts_with("__pragma")
+            || line.contains("__attribute__")
+            || line.contains("uintptr_t")
+            || line.contains("intptr_t")
+            || line.contains("size_t")
+            || line.contains("ptrdiff_t")
+        {
+            return None;
+        }
+
+        return Some(Cow::Owned(line.replace("__attribute__((__cdecl__))", "")));
+    };
+
+    Some(Cow::Borrowed(line))
+}
+
+fn main() {
+    println!("cargo:rerun-if-changed=src/extism.c");
+    println!("cargo:rerun-if-changed=../runtime/extism.h");
+    std::fs::copy("../runtime/extism.h", "src/extism.h").unwrap();
+
+    let data = String::from_utf8(
+        cc::Build::new()
+            .file("src/extism.c")
+            .warnings(false)
+            .extra_warnings(false)
+            .expand(),
+    )
+    .unwrap();
+    let data: Vec<&str> = data.split('\n').collect();
+    let data: String = data
+        .into_iter()
+        .filter_map(rewrite)
+        .collect::<Vec<Cow<'_, str>>>()
+        .join("\n\n");
+
+    std::fs::write("../target/header.h", data).unwrap();
+}

extism-maturin/pyproject.toml

@@ -0,0 +1,17 @@
+[build-system]
+requires = ["maturin>=1.1,<2.0"]
+build-backend = "maturin"
+
+[project]
+name = "extism-sys"
+version = "0.0.0+replaced-by-ci"
+requires-python = ">=3.7"
+classifiers = [
+    "Programming Language :: Rust",
+    "Programming Language :: Python :: Implementation :: CPython",
+    "Programming Language :: Python :: Implementation :: PyPy",
+]
+dependencies = ["cffi"]
+
+[tool.maturin]
+bindings = "cffi"

extism-maturin/src/extism.c

@@ -0,0 +1 @@
+#include "extism.h"

extism-maturin/src/lib.rs

@@ -0,0 +1 @@
+pub use extism::sdk::*;

extism.go

@@ -1,253 +0,0 @@
-package extism
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"unsafe"
-)
-
-/*
-#cgo pkg-config: libextism.pc
-#include <extism.h>
-#include <stdlib.h>
-*/
-import "C"
-
-// Context is used to manage Plugins
-type Context struct {
-	pointer *C.ExtismContext
-}
-
-// NewContext creates a new context, it should be freed using the `Free` method
-func NewContext() Context {
-	p := C.extism_context_new()
-	return Context{
-		pointer: p,
-	}
-}
-
-// Free a context
-func (ctx *Context) Free() {
-	C.extism_context_free(ctx.pointer)
-	ctx.pointer = nil
-}
-
-// Plugin is used to call WASM functions
-type Plugin struct {
-	ctx *Context
-	id  int32
-}
-
-type WasmData struct {
-	Data []byte `json:"data"`
-	Hash string `json:"hash,omitempty"`
-	Name string `json:"name,omitempty"`
-}
-
-type WasmFile struct {
-	Path string `json:"path"`
-	Hash string `json:"hash,omitempty"`
-	Name string `json:"name,omitempty"`
-}
-
-type WasmUrl struct {
-	Url    string            `json:"url"`
-	Hash   string            `json:"hash,omitempty"`
-	Header map[string]string `json:"header,omitempty"`
-	Name   string            `json:"name,omitempty"`
-	Method string            `json:"method,omitempty"`
-}
-
-type Wasm interface{}
-
-type Manifest struct {
-	Wasm   []Wasm `json:"wasm"`
-	Memory struct {
-		Max uint32 `json:"max,omitempty"`
-	} `json:"memory,omitempty"`
-	Config       map[string]string `json:"config,omitempty"`
-	AllowedHosts []string          `json:"allowed_hosts,omitempty"`
-}
-
-func makePointer(data []byte) unsafe.Pointer {
-	var ptr unsafe.Pointer = nil
-	if len(data) > 0 {
-		ptr = unsafe.Pointer(&data[0])
-	}
-	return ptr
-}
-
-// SetLogFile sets the log file and level, this is a global setting
-func SetLogFile(filename string, level string) bool {
-	name := C.CString(filename)
-	l := C.CString(level)
-	r := C.extism_log_file(name, l)
-	C.free(unsafe.Pointer(name))
-	C.free(unsafe.Pointer(l))
-	return bool(r)
-}
-
-// ExtismVersion gets the Extism version string
-func ExtismVersion() string {
-	return C.GoString(C.extism_version())
-}
-
-func register(ctx *Context, data []byte, wasi bool) (Plugin, error) {
-	ptr := makePointer(data)
-	plugin := C.extism_plugin_new(
-		ctx.pointer,
-		(*C.uchar)(ptr),
-		C.uint64_t(len(data)),
-		C._Bool(wasi),
-	)
-
-	if plugin < 0 {
-		err := C.extism_error(ctx.pointer, C.int32_t(-1))
-		msg := "Unknown"
-		if err != nil {
-			msg = C.GoString(err)
-		}
-
-		return Plugin{id: -1}, errors.New(
-			fmt.Sprintf("Unable to load plugin: %s", msg),
-		)
-	}
-
-	return Plugin{id: int32(plugin), ctx: ctx}, nil
-}
-
-func update(ctx *Context, plugin int32, data []byte, wasi bool) error {
-	ptr := makePointer(data)
-	b := bool(C.extism_plugin_update(
-		ctx.pointer,
-		C.int32_t(plugin),
-		(*C.uchar)(ptr),
-		C.uint64_t(len(data)),
-		C._Bool(wasi),
-	))
-
-	if b {
-		return nil
-	}
-
-	err := C.extism_error(ctx.pointer, C.int32_t(-1))
-	msg := "Unknown"
-	if err != nil {
-		msg = C.GoString(err)
-	}
-
-	return errors.New(
-		fmt.Sprintf("Unable to load plugin: %s", msg),
-	)
-}
-
-// PluginFromManifest creates a plugin from a `Manifest`
-func (ctx *Context) PluginFromManifest(manifest Manifest, wasi bool) (Plugin, error) {
-	data, err := json.Marshal(manifest)
-	if err != nil {
-		return Plugin{id: -1}, err
-	}
-
-	return register(ctx, data, wasi)
-}
-
-// Plugin creates a plugin from a WASM module
-func (ctx *Context) Plugin(module io.Reader, wasi bool) (Plugin, error) {
-	wasm, err := io.ReadAll(module)
-	if err != nil {
-		return Plugin{id: -1}, err
-	}
-
-	return register(ctx, wasm, wasi)
-}
-
-// Update a plugin with a new WASM module
-func (p *Plugin) Update(module io.Reader, wasi bool) error {
-	wasm, err := io.ReadAll(module)
-	if err != nil {
-		return err
-	}
-
-	return update(p.ctx, p.id, wasm, wasi)
-}
-
-// Update a plugin with a new Manifest
-func (p *Plugin) UpdateManifest(manifest Manifest, wasi bool) error {
-	data, err := json.Marshal(manifest)
-	if err != nil {
-		return err
-	}
-
-	return update(p.ctx, p.id, data, wasi)
-}
-
-// Set configuration values
-func (plugin Plugin) SetConfig(data map[string][]byte) error {
-	s, err := json.Marshal(data)
-	if err != nil {
-		return err
-	}
-	ptr := makePointer(s)
-	C.extism_plugin_config(plugin.ctx.pointer, C.int(plugin.id), (*C.uchar)(ptr), C.uint64_t(len(s)))
-	return nil
-}
-
-// FunctionExists returns true when the named function is present in the plugin
-func (plugin Plugin) FunctionExists(functionName string) bool {
-	name := C.CString(functionName)
-	b := C.extism_plugin_function_exists(plugin.ctx.pointer, C.int(plugin.id), name)
-	C.free(unsafe.Pointer(name))
-	return bool(b)
-}
-
-// Call a function by name with the given input, returning the output
-func (plugin Plugin) Call(functionName string, input []byte) ([]byte, error) {
-	ptr := makePointer(input)
-	name := C.CString(functionName)
-	rc := C.extism_plugin_call(
-		plugin.ctx.pointer,
-		C.int32_t(plugin.id),
-		name,
-		(*C.uchar)(ptr),
-		C.uint64_t(len(input)),
-	)
-	C.free(unsafe.Pointer(name))
-
-	if rc != 0 {
-		err := C.extism_error(plugin.ctx.pointer, C.int32_t(plugin.id))
-		msg := "<unset by plugin>"
-		if err != nil {
-			msg = C.GoString(err)
-		}
-
-		return nil, errors.New(
-			fmt.Sprintf("Plugin error: %s, code: %d", msg, rc),
-		)
-	}
-
-	length := C.extism_plugin_output_length(plugin.ctx.pointer, C.int32_t(plugin.id))
-
-	if length > 0 {
-		x := C.extism_plugin_output_data(plugin.ctx.pointer, C.int32_t(plugin.id))
-		y := (*[]byte)(unsafe.Pointer(&x))
-		return []byte((*y)[0:length]), nil
-	}
-
-	return []byte{}, nil
-}
-
-// Free a plugin
-func (plugin *Plugin) Free() {
-	if plugin.ctx.pointer == nil {
-		return
-	}
-	C.extism_plugin_free(plugin.ctx.pointer, C.int32_t(plugin.id))
-	plugin.id = -1
-}
-
-// Reset removes all registered plugins in a Context
-func (ctx Context) Reset() {
-	C.extism_context_reset(ctx.pointer)
-}

extism.opam

@@ -1,36 +0,0 @@
-# This file is generated by dune, edit dune-project instead
-opam-version: "2.0"
-synopsis: "Extism bindings"
-description: "Bindings to Extism, the universal plugin system"
-maintainer: ["Extism Authors <oss@extism.org>"]
-authors: ["Extism Authors <oss@extism.org>"]
-license: "BSD-3-Clause"
-tags: ["topics" "wasm" "plugin"]
-homepage: "https://github.com/extism/extism"
-doc: "https://github.com/extism/extism"
-bug-reports: "https://github.com/extism/extism/issues"
-depends: [
-  "ocaml"
-  "dune" {>= "3.2"}
-  "ctypes-foreign"
-  "bigstringaf"
-  "ppx_yojson_conv"
-  "base64"
-  "ppx_inline_test"
-  "odoc" {with-doc}
-]
-build: [
-  ["dune" "subst"] {dev}
-  [
-    "dune"
-    "build"
-    "-p"
-    name
-    "-j"
-    jobs
-    "@install"
-    "@runtest" {with-test}
-    "@doc" {with-doc}
-  ]
-]
-dev-repo: "git+https://github.com/extism/extism.git"

extism_test.go

@@ -1,148 +0,0 @@
-package extism
-
-import (
-	"encoding/json"
-	"fmt"
-	"testing"
-)
-
-func manifest() Manifest {
-	return Manifest{
-		Wasm: []Wasm{
-			WasmFile{
-				Path: "./wasm/code.wasm",
-			},
-		},
-	}
-}
-
-func expectVowelCount(plugin Plugin, input string, count int) error {
-	out, err := plugin.Call("count_vowels", []byte(input))
-	if err != nil {
-		return err
-	}
-	var result map[string]int
-	json.Unmarshal(out, &result)
-	if result["count"] != count {
-		return fmt.Errorf("Got count %d but expected %d", result["count"], count)
-	}
-	return nil
-}
-
-func TestCreateAndFreeContext(t *testing.T) {
-	ctx := NewContext()
-	ctx.Free()
-}
-
-func TestCallPlugin(t *testing.T) {
-	ctx := NewContext()
-	defer ctx.Free()
-
-	plugin, err := ctx.PluginFromManifest(manifest(), false)
-	if err != nil {
-		t.Error(err)
-	}
-
-	if err := expectVowelCount(plugin, "this is a test", 4); err != nil {
-		t.Error(err)
-	}
-	if err := expectVowelCount(plugin, "this is a test again", 7); err != nil {
-		t.Error(err)
-	}
-	if err := expectVowelCount(plugin, "this is a test thrice", 6); err != nil {
-		t.Error(err)
-	}
-}
-
-func TestFreePlugin(t *testing.T) {
-	ctx := NewContext()
-	defer ctx.Free()
-
-	plugin, err := ctx.PluginFromManifest(manifest(), false)
-	if err != nil {
-		t.Error(err)
-	}
-	if err := expectVowelCount(plugin, "this is a test", 4); err != nil {
-		t.Error(err)
-	}
-
-	// free this specific plugin
-	plugin.Free()
-
-	if err := expectVowelCount(plugin, "this is a test", 4); err == nil {
-		t.Fatal("Expected an error after plugin was freed")
-	}
-}
-
-func TestContextReset(t *testing.T) {
-	ctx := NewContext()
-	defer ctx.Free()
-
-	plugin, err := ctx.PluginFromManifest(manifest(), false)
-	if err != nil {
-		t.Error(err)
-	}
-	if err := expectVowelCount(plugin, "this is a test", 4); err != nil {
-		t.Error(err)
-	}
-
-	// reset the context dropping all plugins
-	ctx.Reset()
-
-	if err := expectVowelCount(plugin, "this is a test", 4); err == nil {
-		t.Fatal("Expected an error after plugin was freed")
-	}
-}
-
-func TestCanUpdateAManifest(t *testing.T) {
-	ctx := NewContext()
-	defer ctx.Free()
-
-	plugin, err := ctx.PluginFromManifest(manifest(), false)
-	if err != nil {
-		t.Error(err)
-	}
-
-	if err := expectVowelCount(plugin, "this is a test", 4); err != nil {
-		t.Error(err)
-	}
-
-	plugin.UpdateManifest(manifest(), false)
-
-	// can still call the plugin
-	if err := expectVowelCount(plugin, "this is a test", 4); err != nil {
-		t.Error(err)
-	}
-}
-
-func TestFunctionExists(t *testing.T) {
-	ctx := NewContext()
-	defer ctx.Free()
-
-	plugin, err := ctx.PluginFromManifest(manifest(), false)
-	if err != nil {
-		t.Error(err)
-	}
-
-	if !plugin.FunctionExists("count_vowels") {
-		t.Fatal("Was expecting to find the function count_vowels")
-	}
-	if plugin.FunctionExists("i_dont_exist") {
-		t.Fatal("Was not expecting to find the function i_dont_exist")
-	}
-}
-
-func TestErrorsOnUnknownFunction(t *testing.T) {
-	ctx := NewContext()
-	defer ctx.Free()
-
-	plugin, err := ctx.PluginFromManifest(manifest(), false)
-	if err != nil {
-		t.Error(err)
-	}
-
-	_, err = plugin.Call("i_dont_exist", []byte("someinput"))
-	if err == nil {
-		t.Fatal("Was expecting call to unknown function to fail")
-	}
-}

go.mod

@@ -1,3 +0,0 @@
-module github.com/extism/extism
-
-go 1.18

go/go.mod

@@ -1,7 +0,0 @@
-module github.com/extism/extism-go-example
-
-go 1.18
-
-replace github.com/extism/extism => ../
-
-require github.com/extism/extism v0.0.0-00010101000000-000000000000

go/main.go

@@ -1,47 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"os"
-
-	"github.com/extism/extism"
-)
-
-func main() {
-	version := extism.ExtismVersion()
-	fmt.Println("Extism Version: ", version)
-
-	ctx := extism.NewContext()
-	defer ctx.Free() // this will free the context and all associated plugins
-
-	// set some input data to provide to the plugin module
-	var data []byte
-	if len(os.Args) > 1 {
-		data = []byte(os.Args[1])
-	} else {
-		data = []byte("testing from go -> wasm shared memory...")
-	}
-
-	manifest := extism.Manifest{Wasm: []extism.Wasm{extism.WasmFile{Path: "../wasm/code.wasm"}}}
-	plugin, err := ctx.PluginFromManifest(manifest, false)
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-
-	// use the extism Go library to provide the input data to the plugin, execute it, and then
-	// collect the plugin state and error if present
-	out, err := plugin.Call("count_vowels", data)
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-
-	// "out" is []byte type, and the plugin sends back json, so deserialize it into a map.
-	// expect this object: `{"count": n}`
-	var dest map[string]int
-	json.Unmarshal(out, &dest)
-
-	fmt.Println("Count:", dest["count"])
-}

haskell/CHANGELOG.md

@@ -1,5 +0,0 @@
-# Revision history for extism
-
-## 0.1.0.0 -- YYYY-mm-dd
-
-* First version. Released on an unsuspecting world.

haskell/Example.hs

@@ -1,23 +0,0 @@
-module Main where
-
-import System.Exit (exitFailure, exitSuccess)
-import qualified Data.ByteString as B
-import Extism
-import Extism.Manifest
-
-try f (Right x) = f x
-try f (Left (ErrorMessage msg)) = do
-  _ <- putStrLn msg
-  exitFailure
-
-handlePlugin plugin = do
-  res <- Extism.call plugin "count_vowels" (Extism.toByteString "this is a test")
-  try (\bs -> do
-    _ <- putStrLn (Extism.fromByteString bs)
-    _ <- Extism.free plugin
-    exitSuccess) res
-
-main = do
-  context <- Extism.newContext ()
-  plugin <- Extism.pluginFromManifest context (manifest [wasmFile "../wasm/code.wasm"]) False
-  try handlePlugin plugin

haskell/extism.cabal

@@ -1,54 +0,0 @@
-cabal-version:      2.4
-name:               extism
-version:            0.0.1.0
-
--- A short (one-line) description of the package.
-synopsis: Extism bindings
-
--- A longer description of the package.
-description: Bindings to Extism, the universal plugin system
-
--- A URL where users can report bugs.
-bug-reports: https://github.com/extism/extism
-
--- The license under which the package is released.
-license: BSD-3-Clause
-
-author:             Extism authors
-maintainer:         oss@extism.org
-
--- A copyright notice.
--- copyright:
-category: Plugins, WebAssembly
-extra-source-files: CHANGELOG.md
-
-library
-    exposed-modules: Extism Extism.Manifest
-
-    -- Modules included in this library but not exported.
-    other-modules:
-
-    -- LANGUAGE extensions used by modules in this package.
-    -- other-extensions:
-    build-depends:    
-          base ^>=4.16.1.0
-        , bytestring
-        , base64-bytestring
-        , json
-    hs-source-dirs:   src
-    default-language: Haskell2010
-    extra-libraries: extism
-    extra-lib-dirs: /usr/local/lib
-    
-Test-Suite extism-example
-    type:       exitcode-stdio-1.0
-    main-is:    Example.hs
-    build-depends: base, extism, bytestring
-    default-language: Haskell2010
-        
-Test-Suite extism-test
-    type:       exitcode-stdio-1.0
-    main-is:    Test.hs
-    hs-source-dirs:   test
-    build-depends: base, extism, bytestring, HUnit
-    default-language: Haskell2010

haskell/src/Extism.hs

@@ -1,192 +0,0 @@
-{-# LANGUAGE ForeignFunctionInterface #-}
-
-module Extism (module Extism, module Extism.Manifest) where
-import GHC.Int
-import GHC.Word
-import Foreign.C.Types
-import Foreign.Ptr
-import Foreign.ForeignPtr
-import Foreign.C.String
-import Control.Monad (void)
-import Data.ByteString as B
-import Data.ByteString.Internal (c2w, w2c)
-import Data.ByteString.Unsafe (unsafeUseAsCString)
-import Data.Bifunctor (second)
-import Text.JSON (JSON, toJSObject, toJSString, encode, JSValue(JSNull, JSString))
-import Extism.Manifest (Manifest, toString)
-
-newtype ExtismContext = ExtismContext () deriving Show
-
-foreign import ccall unsafe "extism.h extism_context_new" extism_context_new :: IO (Ptr ExtismContext)
-foreign import ccall unsafe "extism.h &extism_context_free" extism_context_free :: FunPtr (Ptr ExtismContext -> IO ())
-foreign import ccall unsafe "extism.h extism_plugin_new" extism_plugin_new :: Ptr ExtismContext -> Ptr Word8 -> Word64 -> CBool -> IO Int32
-foreign import ccall unsafe "extism.h extism_plugin_update" extism_plugin_update :: Ptr ExtismContext -> Int32 -> Ptr Word8 -> Word64 -> CBool -> IO CBool
-foreign import ccall unsafe "extism.h extism_plugin_call" extism_plugin_call :: Ptr ExtismContext -> Int32 -> CString -> Ptr Word8 -> Word64 -> IO Int32
-foreign import ccall unsafe "extism.h extism_plugin_function_exists" extism_plugin_function_exists :: Ptr ExtismContext -> Int32 -> CString -> IO CBool
-foreign import ccall unsafe "extism.h extism_error" extism_error :: Ptr ExtismContext -> Int32 -> IO CString
-foreign import ccall unsafe "extism.h extism_plugin_output_length" extism_plugin_output_length :: Ptr ExtismContext -> Int32 -> IO Word64
-foreign import ccall unsafe "extism.h extism_plugin_output_data" extism_plugin_output_data :: Ptr ExtismContext -> Int32 -> IO (Ptr Word8)
-foreign import ccall unsafe "extism.h extism_log_file" extism_log_file :: CString -> CString -> IO CBool
-foreign import ccall unsafe "extism.h extism_plugin_config" extism_plugin_config :: Ptr ExtismContext -> Int32 -> Ptr Word8 -> Int64 -> IO CBool
-foreign import ccall unsafe "extism.h extism_plugin_free" extism_plugin_free :: Ptr ExtismContext -> Int32 -> IO ()
-foreign import ccall unsafe "extism.h extism_context_reset" extism_context_reset :: Ptr ExtismContext -> IO ()
-foreign import ccall unsafe "extism.h extism_version" extism_version :: IO CString
-
--- Context manages plugins
-newtype Context = Context (ForeignPtr ExtismContext)
-
--- Plugins can be used to call WASM function
-data Plugin = Plugin Context Int32
-
--- Log level
-data LogLevel = Error | Warn | Info | Debug | Trace deriving (Show)
-
--- Extism error
-newtype Error = ErrorMessage String deriving Show
-
--- Helper function to convert a string to a bytestring
-toByteString :: String -> ByteString
-toByteString x = B.pack (Prelude.map c2w x)
-
--- Helper function to convert a bytestring to a string
-fromByteString :: ByteString -> String
-fromByteString bs = Prelude.map w2c $ B.unpack bs
-
--- Get the Extism version string
-extismVersion :: () -> IO String
-extismVersion () = do
-  v <- extism_version
-  peekCString v
-
--- Remove all registered plugins in a Context
-reset :: Context -> IO ()
-reset (Context ctx) =
-  withForeignPtr ctx extism_context_reset
-
--- Create a new context
-newContext :: () -> IO Context
-newContext () = do
-  ptr <- extism_context_new
-  fptr <- newForeignPtr extism_context_free ptr
-  return (Context fptr)
- 
--- Execute a function with a new context that is destroyed when it returns
-withContext :: (Context -> IO a) -> IO a
-withContext f = do
-  ctx <- newContext ()
-  f ctx
-
--- Create a plugin from a WASM module, `useWasi` determines if WASI should
--- be linked
-plugin :: Context -> B.ByteString -> Bool -> IO (Either Error Plugin)
-plugin c wasm useWasi =
-  let length = fromIntegral (B.length wasm) in
-  let wasi = fromInteger (if useWasi then 1 else 0) in
-  let Context ctx = c in
-  do
-    withForeignPtr ctx (\ctx -> do
-      p <- unsafeUseAsCString wasm (\s ->
-        extism_plugin_new ctx (castPtr s) length wasi)
-      if p < 0 then do
-        err <- extism_error ctx (-1)
-        e <- peekCString err
-        return $ Left (ErrorMessage e)
-      else
-        return $ Right (Plugin c p))
-
--- Create a plugin from a Manifest
-pluginFromManifest :: Context -> Manifest -> Bool -> IO (Either Error Plugin)
-pluginFromManifest ctx manifest useWasi =
-  let wasm = toByteString $ toString manifest in
-  plugin ctx wasm useWasi
-
--- Update a plugin with a new WASM module
-update :: Plugin -> B.ByteString -> Bool -> IO (Either Error ())
-update (Plugin (Context ctx) id) wasm useWasi =
-  let length = fromIntegral (B.length wasm) in
-  let wasi = fromInteger (if useWasi then 1 else 0) in
-  do
-    withForeignPtr ctx (\ctx -> do
-      b <- unsafeUseAsCString wasm (\s ->
-        extism_plugin_update ctx id (castPtr s) length wasi)
-      if b <= 0 then do
-        err <- extism_error ctx (-1)
-        e <- peekCString err
-        return $ Left (ErrorMessage e)
-      else
-        return (Right ()))
-
--- Update a plugin with a new Manifest
-updateManifest :: Plugin -> Manifest -> Bool -> IO (Either Error ())
-updateManifest plugin manifest useWasi =
-  let wasm = toByteString $ toString manifest in
-  update plugin wasm useWasi
-
--- Check if a plugin is value
-isValid :: Plugin -> Bool
-isValid (Plugin _ p) = p >= 0
-
-convertMaybeString Nothing = JSNull
-convertMaybeString (Just s) = JSString (toJSString s)
-
--- Set configuration values for a plugin
-setConfig :: Plugin -> [(String, Maybe String)] -> IO Bool
-setConfig (Plugin (Context ctx) plugin) x =
-  if plugin < 0
-    then return False
-  else
-    let obj = toJSObject [(k, convertMaybeString v) | (k, v) <- x] in
-    let bs = toByteString (encode obj) in
-    let length = fromIntegral (B.length bs) in
-    unsafeUseAsCString bs (\s -> do
-      withForeignPtr ctx (\ctx -> do
-        b <- extism_plugin_config ctx plugin (castPtr s) length
-        return $ b /= 0))
-
-levelStr Error = "error"
-levelStr Debug = "debug"
-levelStr Warn = "warn"
-levelStr Trace = "trace"
-levelStr Info = "info"
-
--- Set the log file and level, this is a global configuration
-setLogFile :: String -> LogLevel -> IO Bool
-setLogFile filename level =
-  let s = levelStr level in
-  withCString filename (\f ->
-    withCString s (\l -> do
-      b <- extism_log_file f l
-      return $ b /= 0))
-
--- Check if a function exists in the given plugin
-functionExists :: Plugin -> String -> IO Bool
-functionExists (Plugin (Context ctx) plugin) name = do
-  withForeignPtr ctx (\ctx -> do
-    b <- withCString name (extism_plugin_function_exists ctx plugin)
-    if b == 1 then return True else return False)
-
---- Call a function provided by the given plugin
-call :: Plugin -> String -> B.ByteString -> IO (Either Error B.ByteString)
-call (Plugin (Context ctx) plugin) name input =
-  let length = fromIntegral (B.length input) in
-  do
-    withForeignPtr ctx (\ctx -> do
-      rc <- withCString name (\name ->
-        unsafeUseAsCString input (\input ->
-          extism_plugin_call ctx plugin name (castPtr input) length))
-      err <- extism_error ctx plugin
-      if err /= nullPtr
-        then do e <- peekCString err
-                return $ Left (ErrorMessage e)
-      else if rc == 0
-        then do
-          length <- extism_plugin_output_length ctx plugin
-          ptr <- extism_plugin_output_data ctx plugin
-          buf <- packCStringLen (castPtr ptr, fromIntegral length)
-          return $ Right buf
-      else return $ Left (ErrorMessage "Call failed"))
-
--- Free a plugin
-free :: Plugin -> IO ()
-free (Plugin (Context ctx) plugin) =
-  withForeignPtr ctx (`extism_plugin_free` plugin)

haskell/src/Extism/Manifest.hs

@@ -1,183 +0,0 @@
-module Extism.Manifest where
-
-import Text.JSON
-  (
-    JSValue(JSNull, JSString, JSArray),
-    toJSString, showJSON, makeObj, encode
-  )
-import qualified Data.ByteString as B
-import qualified Data.ByteString.Base64 as B64
-import qualified Data.ByteString.Char8 as BS (unpack)
-
-valueOrNull f Nothing = JSNull
-valueOrNull f (Just x) = f x
-makeString s = JSString (toJSString s)
-stringOrNull = valueOrNull makeString
-makeArray f [] = JSNull
-makeArray f x = JSArray [f a | a <- x]
-filterNulls obj = [(a, b) | (a, b) <- obj, not (isNull b)]
-mapObj f x = makeObj (filterNulls [(a, f b) | (a, b) <- x])
-isNull JSNull = True
-isNull _ = False
-
-newtype Memory = Memory
-  {
-    memoryMax :: Maybe Int
-  }
-
-class JSONValue a where
-  toJSONValue :: a -> JSValue
-
-instance JSONValue Memory where
-  toJSONValue x =
-    case memoryMax x of
-      Nothing -> makeObj []
-      Just max -> makeObj [("max", showJSON max)]
-
-data HttpRequest = HttpRequest
-  {
-    url :: String
-  , header :: [(String, String)]
-  , method :: Maybe String
-  }
-
-requestObj x =
-  let meth = stringOrNull $ method x in
-  let h = mapObj makeString $ header x in
-  filterNulls [
-    ("url", makeString $ url x),
-    ("header", h),
-    ("method", meth)
-  ]
-
-instance JSONValue HttpRequest where
-  toJSONValue x =
-    makeObj $ requestObj x
-
-data WasmFile = WasmFile
-  {
-    filePath :: String
-  , fileName :: Maybe String
-  , fileHash :: Maybe String
-  }
-
-instance JSONValue WasmFile where
-  toJSONValue x =
-    let path = makeString $ filePath x in
-    let name = stringOrNull $ fileName x in
-    let hash = stringOrNull $ fileHash x in
-    makeObj $ filterNulls [
-      ("path", path),
-      ("name", name),
-      ("hash", hash)
-    ]
-
-data WasmCode = WasmCode
-  {
-    codeBytes :: B.ByteString
-  , codeName :: Maybe String
-  , codeHash :: Maybe String
-  }
-
-
-instance JSONValue WasmCode where
-  toJSONValue x =
-    let bytes = makeString $ BS.unpack $ B64.encode $ codeBytes x in
-    let name = stringOrNull $ codeName x in
-    let hash = stringOrNull $ codeHash x in
-    makeObj $ filterNulls [
-      ("data", bytes),
-      ("name", name),
-      ("hash", hash)
-    ]
-
-data WasmURL = WasmURL
-  {
-    req :: HttpRequest
-  , urlName :: Maybe String
-  , urlHash :: Maybe String
-  }
-
-
-instance JSONValue WasmURL where
-  toJSONValue x =
-    let request = requestObj $ req x in
-    let name = stringOrNull $ urlName x in
-    let hash = stringOrNull $ urlHash x in
-    makeObj $ filterNulls $ ("name", name) : ("hash", hash) : request
-
-data Wasm = File WasmFile | Code WasmCode | URL WasmURL
-
-instance JSONValue Wasm where
-  toJSONValue x =
-    case x of
-      File f -> toJSONValue f
-      Code d -> toJSONValue d
-      URL u -> toJSONValue u
-
-wasmFile :: String -> Wasm
-wasmFile path =
-  File WasmFile { filePath = path, fileName = Nothing, fileHash = Nothing}
-
-wasmURL :: String -> String -> Wasm
-wasmURL method url =
-  let r = HttpRequest { url = url, header = [], method = Just method } in
-  URL WasmURL { req = r, urlName = Nothing, urlHash = Nothing }
-
-wasmCode :: B.ByteString -> Wasm
-wasmCode code =
-  Code WasmCode { codeBytes = code, codeName = Nothing, codeHash = Nothing }
-
-withName :: Wasm -> String -> Wasm
-withName (Code code) name = Code code { codeName = Just name }
-withName (URL url) name =  URL url { urlName = Just name }
-withName (File f) name = File  f { fileName = Just name }
-
-
-withHash :: Wasm -> String -> Wasm
-withHash (Code code) hash = Code code { codeHash = Just hash }
-withHash (URL url) hash =  URL url { urlHash = Just hash }
-withHash (File f) hash = File  f { fileHash = Just hash }
-
-data Manifest = Manifest
-  {
-    wasm :: [Wasm]
-  , memory :: Maybe Memory
-  , config :: [(String, String)]
-  , allowed_hosts :: [String]
-  }
-
-manifest :: [Wasm] -> Manifest
-manifest wasm =
-  Manifest {
-    wasm = wasm,
-    memory = Nothing,
-    config = [],
-    allowed_hosts = []
-  }
-
-withConfig :: Manifest -> [(String, String)] -> Manifest
-withConfig m config =
-  m { config = config }
-
-
-withHosts :: Manifest -> [String] -> Manifest
-withHosts m hosts =
-  m { allowed_hosts = hosts }
-
-instance JSONValue Manifest where
-  toJSONValue x =
-    let w = makeArray toJSONValue $ wasm x in
-    let mem = valueOrNull toJSONValue $ memory x in
-    let c = mapObj makeString $ config x in
-    let hosts = makeArray makeString $ allowed_hosts x in
-    makeObj $ filterNulls [
-      ("wasm", w),
-      ("memory", mem),
-      ("config", c),
-      ("allowed_hosts", hosts)
-    ]
-
-toString :: Manifest -> String
-toString manifest =
-  encode (toJSONValue manifest)

haskell/stack.yaml

@@ -1,67 +0,0 @@
-# This file was automatically generated by 'stack init'
-#
-# Some commonly used options have been documented as comments in this file.
-# For advanced use and comprehensive documentation of the format, please see:
-# https://docs.haskellstack.org/en/stable/yaml_configuration/
-
-# Resolver to choose a 'specific' stackage snapshot or a compiler version.
-# A snapshot resolver dictates the compiler version and the set of packages
-# to be used for project dependencies. For example:
-#
-# resolver: lts-3.5
-# resolver: nightly-2015-09-21
-# resolver: ghc-7.10.2
-#
-# The location of a snapshot can be provided as a file or url. Stack assumes
-# a snapshot provided as a file might change, whereas a url resource does not.
-#
-# resolver: ./custom-snapshot.yaml
-# resolver: https://example.com/snapshots/2018-01-01.yaml
-resolver:
-  url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/nightly/2022/8/30.yaml
-
-# User packages to be built.
-# Various formats can be used as shown in the example below.
-#
-# packages:
-# - some-directory
-# - https://example.com/foo/bar/baz-0.0.2.tar.gz
-#   subdirs:
-#   - auto-update
-#   - wai
-packages:
-- .
-# Dependency packages to be pulled from upstream that are not in the resolver.
-# These entries can reference officially published versions as well as
-# forks / in-progress versions pinned to a git hash. For example:
-#
-# extra-deps:
-# - acme-missiles-0.3
-# - git: https://github.com/commercialhaskell/stack.git
-#   commit: e7b331f14bcffb8367cd58fbfc8b40ec7642100a
-#
-# extra-deps: []
-
-# Override default flag values for local packages and extra-deps
-# flags: {}
-
-# Extra package databases containing global packages
-# extra-package-dbs: []
-
-# Control whether we use the GHC we find on the path
-# system-ghc: true
-#
-# Require a specific version of stack, using version ranges
-# require-stack-version: -any # Default
-# require-stack-version: ">=2.7"
-#
-# Override the architecture used by stack, especially useful on Windows
-# arch: i386
-# arch: x86_64
-#
-# Extra directories used by stack for building
-# extra-include-dirs: [/path/to/dir]
-# extra-lib-dirs: [/path/to/dir]
-#
-# Allow a newer minor version of GHC than the snapshot specifies
-# compiler-check: newer-minor

haskell/stack.yaml.lock

@@ -1,13 +0,0 @@
-# This file was autogenerated by Stack.
-# You should not edit this file by hand.
-# For more information, please see the documentation at:
-#   https://docs.haskellstack.org/en/stable/lock_files
-
-packages: []
-snapshots:
-- completed:
-    size: 632828
-    url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/nightly/2022/8/30.yaml
-    sha256: 5b02c2ce430ac62843fb884126765628da2ca2280bb9de0c6635c723e32a9f6b
-  original:
-    url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/nightly/2022/8/30.yaml

haskell/test/Test.hs

@@ -1,73 +0,0 @@
-import Test.HUnit
-import Extism
-import Extism.Manifest
-
-
-unwrap' (Right x) = return x
-unwrap' (Left (ErrorMessage msg)) =
-  assertFailure msg
-
-unwrap io = do
-  x <- io
-  unwrap' x
-
-defaultManifest = manifest [wasmFile "test/code.wasm"]
-
-initPlugin context =
-  unwrap (Extism.pluginFromManifest context defaultManifest False)
-
-pluginFunctionExists = do
-  withContext (\ctx -> do
-    p <- initPlugin ctx
-    exists <- functionExists p "count_vowels"
-    assertBool "function exists" exists
-    exists' <- functionExists p "function_doesnt_exist"
-    assertBool "function doesn't exist" (not exists'))
-
-checkCallResult p = do
-    res <- unwrap (call p "count_vowels" (toByteString "this is a test"))
-    assertEqual "count vowels output" "{\"count\": 4}" (fromByteString res)
-
-pluginCall = do
-  withContext (\ctx -> do
-    p <- initPlugin ctx
-    checkCallResult p)
-
-pluginMultiple = do
-  withContext (\ctx -> do
-    p <- initPlugin ctx
-    checkCallResult p
-    q <- initPlugin ctx
-    r <- initPlugin ctx
-    checkCallResult q
-    checkCallResult r)
-
-pluginUpdate = do
-  withContext (\ctx -> do
-    p <- initPlugin ctx
-    unwrap (updateManifest p defaultManifest True)
-    checkCallResult p)
-
-pluginConfig = do
-  withContext (\ctx -> do
-    p <- initPlugin ctx
-    b <- setConfig p [("a", Just "1"), ("b", Just "2"), ("c", Just "3"), ("d", Nothing)]
-    assertBool "set config" b)
-
-testSetLogFile = do
-  b <- setLogFile "stderr" Error
-  assertBool "set log file" b
-
-t name f = TestLabel name (TestCase f)
-
-main = do
-  runTestTT (TestList
-    [
-      t "Plugin.FunctionExists" pluginFunctionExists
-      , t "Plugin.Call" pluginCall
-      , t "Plugin.Multiple" pluginMultiple
-      , t "Plugin.Update" pluginUpdate
-      , t "Plugin.Config" pluginConfig
-      , t "SetLogFile" testSetLogFile
-    ])
-

kernel/.cargo/config

@@ -0,0 +1,2 @@
+[build]
+target = "wasm32-unknown-unknown"

kernel/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "extism-runtime-kernel"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+
+[dev-dependencies]
+wasm-bindgen-test = "0.3.39"
+
+[features]
+default = ["bounds-checking"]
+bounds-checking = []
+
+[workspace]
+members = [
+  "."
+]

kernel/README.md

@@ -0,0 +1,20 @@
+# Extism kernel
+
+The Extism kernel implements core parts of the Extism runtime in Rust compiled to WebAssembly. This code is a conceptual
+re-write of [memory.rs][] with the goal of making core parts of the Extism implementation more portable across WebAssembly
+runtimes.
+
+See [lib.rs][] for more details about the implementation itself.
+
+## Building
+
+Because this crate is built using the `wasm32-unknown-unknown` target, it is a separate build process from the `extism-runtime` crate. 
+
+To build `extism-runtime.wasm`, strip it and copy it to the proper location in the `extism-runtime` tree you can run:
+
+```shell
+$ sh build.sh
+```
+
+[memory.rs]: https://github.com/extism/extism/blob/f4aa139eced4a74eb4a103f78222ba503e146109/runtime/src/memory.rs
+[lib.rs]: ./src/lib.rs

kernel/build.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+export CARGO_FLAGS=""
+
+while getopts d flag
+do
+    case "${flag}" in
+        d)
+            echo "Disabled bounds-checking";
+            export CARGO_FLAGS="--no-default-features";;
+        *)
+            echo "usage $0 [-d]"
+            echo "\t-d: build with bounds checking disabled"
+            exit 1
+    esac
+done
+
+cargo build --package extism-runtime-kernel --bin extism-runtime --release --target wasm32-unknown-unknown $CARGO_FLAGS
+cp target/wasm32-unknown-unknown/release/extism-runtime.wasm .
+wasm-strip extism-runtime.wasm
+mv extism-runtime.wasm ../runtime/src/extism-runtime.wasm
+

kernel/rust-toolchain.toml

@@ -0,0 +1,4 @@
+[toolchain]
+channel = "stable"
+components = [ "rustfmt", "rust-std" ]
+targets = [ "wasm32-unknown-unknown" ]

kernel/src/bin/extism-runtime.rs

@@ -0,0 +1,10 @@
+#![no_main]
+#![no_std]
+
+pub use extism_runtime_kernel::*;
+
+#[cfg(all(target_arch = "wasm32", not(test)))]
+#[panic_handler]
+fn panic(_info: &core::panic::PanicInfo) -> ! {
+    core::arch::wasm32::unreachable()
+}

kernel/src/lib.rs

@@ -0,0 +1,641 @@
+//! # Extism kernel
+//!
+//! - Isolated memory from both host and plugin
+//! - An allocator for managing that memory
+//! - Input/output handling
+//! - Error message handling
+//!
+//! ## Allocator
+//!
+//! The Extism allocator is a bump allocator that tracks the `length` of the total number of bytes
+//! available to the allocator and `position` to track how much of the data has been used. Things like memory
+//! have not really been optimized at all. When a new allocation that is larger than the remaning size is made,
+//! the allocator attempts to call `memory.grow` if that fails a `0` offset is returned, which should be interpreted
+//! as a failed allocation.
+//!
+//! ## Input/Output
+//!
+//! Input and output are just allocated blocks of memory that are marked as either input or output using
+//! the `input_set` or `output_set` functions. The MemoryRoot field `input_offset` contains
+//! the offset in memory to the input data and `input_length` contains the size of the input data. `output_offset`
+//! and `output_length` are used for the output data.
+//!
+//! ## Error handling
+//!
+//! The `error` field is used to track the current error message. If it is set to `0` then there is no error.
+//! The length of the error message can be retreived using `length`.
+//!
+//! ## Memory offsets
+//! An offset of `0` is similar to a `NULL` pointer in C - it implies an allocation failure or memory error
+//! of some kind
+//!
+//! ## Extism functions
+//!
+//! These functions are backward compatible with the pre-kernel runtime, but a few new functions are added to
+//! give runtimes more access to the internals necesarry to load data in and out of a plugin.
+#![no_std]
+#![allow(clippy::missing_safety_doc)]
+
+use core::sync::atomic::*;
+
+pub type Pointer = u64;
+pub type Handle = u64;
+
+/// WebAssembly page size
+const PAGE_SIZE: usize = 65536;
+
+/// Provides information about the usage status of a `MemoryBlock`
+#[repr(u8)]
+#[derive(PartialEq)]
+pub enum MemoryStatus {
+    /// Unused memory that is available b
+    Unused = 0,
+    /// In-use memory
+    Active = 1,
+    /// Free memory that is available for re-use
+    Free = 2,
+}
+
+/// A single `MemoryRoot` exists at the start of the memory to track information about the total
+/// size of the allocated memory and the position of the bump allocator.
+///
+/// The overall layout of the Extism-manged memory is organized like this:
+
+/// |------|-------+---------|-------+--------------|
+/// | Root | Block +  Data   | Block +     Data     | ...
+/// |------|-------+---------|-------+--------------|
+///
+/// Where `Root` and `Block` are fixed to the size of the `MemoryRoot` and `MemoryBlock` structs. But
+/// the size of `Data` is dependent on the allocation size.
+///
+/// This means that the offset of a `Block` is the size of `Root` plus the size of all existing `Blocks`
+/// including their data.
+#[repr(C)]
+pub struct MemoryRoot {
+    /// Set to true after initialization
+    pub initialized: AtomicBool,
+    /// Position of the bump allocator, relative to `blocks` field
+    pub position: AtomicU64,
+    /// The total size of all data allocated using this allocator
+    pub length: AtomicU64,
+    /// Offset of error block
+    pub error: AtomicU64,
+    /// Input position in memory
+    pub input_offset: Handle,
+    /// Input length
+    pub input_length: u64,
+    /// Output position in memory
+    pub output_offset: Pointer,
+    /// Output length
+    pub output_length: u64,
+    /// A pointer to the start of the first block
+    pub blocks: [MemoryBlock; 0],
+}
+
+/// A `MemoryBlock` contains some metadata about a single allocation
+#[repr(C)]
+pub struct MemoryBlock {
+    /// The usage status of the block, `Unused` or `Free` blocks can be re-used.
+    pub status: AtomicU8,
+    /// The total size of the allocation
+    pub size: usize,
+    /// The number of bytes currently being used. If this block is a fresh allocation then `size` and `used` will
+    /// always be the same. If a block is re-used then these numbers may differ.
+    pub used: usize,
+    /// A pointer to the block data
+    pub data: [u8; 0],
+}
+
+/// Returns the number of pages needed for the given number of bytes
+pub fn num_pages(nbytes: u64) -> usize {
+    let npages = nbytes / PAGE_SIZE as u64;
+    let remainder = nbytes % PAGE_SIZE as u64;
+    if remainder != 0 {
+        (npages + 1) as usize
+    } else {
+        npages as usize
+    }
+}
+
+// Get the `MemoryRoot`, this is always stored at offset 1 in memory
+#[inline]
+unsafe fn memory_root() -> &'static mut MemoryRoot {
+    &mut *(1 as *mut MemoryRoot)
+}
+
+impl MemoryRoot {
+    /// Initialize or load the `MemoryRoot` from the correct position in memory
+    pub unsafe fn new() -> &'static mut MemoryRoot {
+        let root = memory_root();
+
+        // If this fails then `INITIALIZED` is already `true` and we can just return the
+        // already initialized `MemoryRoot`
+        if root
+            .initialized
+            .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
+            .is_err()
+        {
+            return root;
+        }
+
+        // Ensure that at least one page is allocated to store the `MemoryRoot` data
+        if core::arch::wasm32::memory_size(0) == 0
+            && core::arch::wasm32::memory_grow(0, 1) == usize::MAX
+        {
+            core::arch::wasm32::unreachable()
+        }
+
+        root.input_offset = 0;
+        root.input_length = 0;
+        root.output_offset = 0;
+        root.output_length = 0;
+        root.error.store(0, Ordering::Release);
+
+        // Initialize the `MemoryRoot` length, position and data
+        root.length.store(
+            PAGE_SIZE as u64 - core::mem::size_of::<MemoryRoot>() as u64,
+            Ordering::Release,
+        );
+        root.position.store(0, Ordering::Release);
+
+        // Ensure the first block is marked as `Unused`
+        #[allow(clippy::size_of_in_element_count)]
+        core::ptr::write_bytes(
+            root.blocks.as_mut_ptr() as *mut _,
+            MemoryStatus::Unused as u8,
+            core::mem::size_of::<MemoryBlock>(),
+        );
+        root
+    }
+
+    /// Resets the position of the allocator and zeroes out all allocations
+    pub unsafe fn reset(&mut self) {
+        // Clear allocated data
+        let self_position = self.position.fetch_and(0, Ordering::SeqCst);
+        core::ptr::write_bytes(
+            self.blocks.as_mut_ptr() as *mut u8,
+            MemoryStatus::Unused as u8,
+            self_position as usize,
+        );
+
+        // Clear extism runtime metadata
+        self.error.store(0, Ordering::Release);
+        self.input_offset = 0;
+        self.input_length = 0;
+        self.output_offset = 0;
+        self.output_length = 0;
+    }
+
+    #[inline(always)]
+    #[allow(unused)]
+    fn pointer_in_bounds(&self, p: Pointer) -> bool {
+        let start_ptr = self.blocks.as_ptr() as Pointer;
+        p >= start_ptr && p < start_ptr + self.length.load(Ordering::Acquire) as Pointer
+    }
+
+    #[inline(always)]
+    #[allow(unused)]
+    fn pointer_in_bounds_fast(p: Pointer) -> bool {
+        // Similar to `pointer_in_bounds` but less accurate on the upper bound. This uses the total memory size,
+        // instead of checking `MemoryRoot::length`
+        let end = (core::arch::wasm32::memory_size(0) as u64) << 16;
+        p >= core::mem::size_of::<Self>() as Pointer && p <= end as u64
+    }
+
+    // Find a block that is free to use, this can be a new block or an existing freed block. The `self_position` argument
+    // is used to avoid loading the allocators position more than once when performing an allocation.
+    unsafe fn find_free_block(
+        &mut self,
+        length: u64,
+        self_position: u64,
+    ) -> Option<&'static mut MemoryBlock> {
+        // Get the first block
+        let mut block = self.blocks.as_mut_ptr();
+
+        // Only loop while the block pointer is less then the current position
+        while (block as u64) < self.blocks.as_ptr() as u64 + self_position {
+            let b = &mut *block;
+
+            // Get the block status, this lets us know if we are able to re-use it
+            let status = b.status.load(Ordering::Acquire);
+
+            // An unused block is safe to use
+            if status == MemoryStatus::Unused as u8 {
+                return Some(b);
+            }
+
+            // Re-use freed blocks when they're large enough
+            if status == MemoryStatus::Free as u8 && b.size >= length as usize {
+                // Split block if there is too much excess
+                if b.size - length as usize >= 128 {
+                    b.size -= length as usize + core::mem::size_of::<MemoryBlock>();
+                    b.used = 0;
+
+                    let block1 = b.data.as_mut_ptr().add(b.size) as *mut MemoryBlock;
+                    let b1 = &mut *block1;
+                    b1.size = length as usize;
+                    b1.used = 0;
+                    b1.status.store(MemoryStatus::Free as u8, Ordering::Release);
+                    return Some(b1);
+                }
+
+                // Otherwise return the whole block
+                return Some(b);
+            }
+
+            // Get the next block
+            block = b.next_ptr();
+        }
+
+        None
+    }
+
+    /// Create a new `MemoryBlock`, when `Some(block)` is returned, `block` will contain at least enough room for `length` bytes
+    /// but may be as large as `length` + `BLOCK_SPLIT_SIZE` bytes. When `None` is returned the allocation has failed.
+    pub unsafe fn alloc(&mut self, length: u64) -> Option<&'static mut MemoryBlock> {
+        let self_position = self.position.load(Ordering::Acquire);
+        let self_length = self.length.load(Ordering::Acquire);
+        let b = self.find_free_block(length, self_position);
+
+        // If there's a free block then re-use it
+        if let Some(b) = b {
+            b.used = length as usize;
+            b.status
+                .store(MemoryStatus::Active as u8, Ordering::Release);
+            return Some(b);
+        }
+
+        // Get the current index for a new block
+        let curr = self.blocks.as_ptr() as u64 + self_position;
+
+        // Get the number of bytes available
+        let mem_left = self_length - self_position - core::mem::size_of::<MemoryRoot>() as u64;
+        let length_with_block = length + core::mem::size_of::<MemoryBlock>() as u64;
+
+        // When the allocation is larger than the number of bytes available
+        // we will need to try to grow the memory
+        if length_with_block >= mem_left {
+            // Calculate the number of pages needed to cover the remaining bytes
+            let npages = num_pages(length_with_block - mem_left);
+            let x = core::arch::wasm32::memory_grow(0, npages);
+            if x == usize::MAX {
+                return None;
+            }
+            self.length
+                .fetch_add(npages as u64 * PAGE_SIZE as u64, Ordering::SeqCst);
+        }
+
+        // Bump the position by the size of the actual data + the size of the MemoryBlock structure
+        self.position.fetch_add(
+            length + core::mem::size_of::<MemoryBlock>() as u64,
+            Ordering::SeqCst,
+        );
+
+        // Initialize a new block at the current position
+        let ptr = curr as *mut MemoryBlock;
+        let block = &mut *ptr;
+        block
+            .status
+            .store(MemoryStatus::Active as u8, Ordering::Release);
+        block.size = length as usize;
+        block.used = length as usize;
+        Some(block)
+    }
+
+    /// Finds the block at an offset in memory
+    pub unsafe fn find_block(&mut self, offs: Pointer) -> Option<&mut MemoryBlock> {
+        if !Self::pointer_in_bounds_fast(offs) {
+            return None;
+        }
+
+        // Get the first block
+        let mut block = self.blocks.as_mut_ptr();
+
+        // Only loop while the block pointer is less then the current position
+        while (block as u64) < self.blocks.as_ptr() as u64 + offs {
+            let b = &mut *block;
+
+            // Get the block status, this lets us know if we are able to re-use it
+            let status = b.status.load(Ordering::Acquire);
+
+            if status == MemoryStatus::Active as u8 && b.data.as_ptr() as Pointer == offs {
+                return Some(b);
+            }
+
+            // Get the next block
+            block = b.next_ptr();
+        }
+
+        None
+    }
+}
+
+impl MemoryBlock {
+    /// Get a pointer to the next block
+    ///
+    /// NOTE: This does no checking to ensure the resulting pointer is valid, the offset
+    /// is calculated based on metadata provided by the current block
+    #[inline]
+    pub unsafe fn next_ptr(&mut self) -> *mut MemoryBlock {
+        self.data.as_mut_ptr().add(self.size) as *mut MemoryBlock
+    }
+
+    /// Mark a block as free
+    pub fn free(&mut self) {
+        self.status
+            .store(MemoryStatus::Free as u8, Ordering::Release);
+    }
+}
+
+// Extism functions
+
+/// Allocate a block of memory and return the offset
+#[no_mangle]
+pub unsafe fn alloc(n: u64) -> Handle {
+    if n == 0 {
+        return 0;
+    }
+    let region = MemoryRoot::new();
+    let block = region.alloc(n);
+    match block {
+        Some(block) => block.data.as_mut_ptr() as Handle,
+        None => 0,
+    }
+}
+
+/// Free allocated memory
+#[no_mangle]
+pub unsafe fn free(p: Handle) {
+    if p == 0 {
+        return;
+    }
+    let root = MemoryRoot::new();
+    let block = root.find_block(p);
+    if let Some(block) = block {
+        block.free();
+
+        // If the input pointer is freed for some reason, make sure the input length to 0
+        // since the original data is gone
+        if p == root.input_offset {
+            root.input_length = 0;
+        }
+    }
+}
+
+/// Get the length of an allocated memory block
+///
+/// Note: this should only be called on memory handles returned
+/// by a call to `alloc` - it will return garbage on invalid offsets
+#[no_mangle]
+pub unsafe fn length_unsafe(p: Handle) -> u64 {
+    if p == 0 {
+        return 0;
+    }
+
+    if !MemoryRoot::pointer_in_bounds_fast(p) {
+        return 0;
+    }
+
+    let ptr = p - core::mem::size_of::<MemoryBlock>() as u64;
+    let block = &mut *(ptr as *mut MemoryBlock);
+
+    // Simplest sanity check to verify the pointer is a block
+    if block.status.load(Ordering::Acquire) != MemoryStatus::Active as u8 {
+        return 0;
+    }
+
+    block.used as u64
+}
+
+/// Get the length but returns 0 if the offset is not a valid handle.
+///
+/// Note: this function walks each node in the allocations list, which ensures correctness, but is also
+/// slow
+#[no_mangle]
+pub unsafe fn length(p: Pointer) -> u64 {
+    if p == 0 {
+        return 0;
+    }
+
+    if let Some(block) = MemoryRoot::new().find_block(p) {
+        block.used as u64
+    } else {
+        0
+    }
+}
+
+/// Load a byte from Extism-managed memory
+#[no_mangle]
+pub unsafe fn load_u8(p: Pointer) -> u8 {
+    #[cfg(feature = "bounds-checking")]
+    if !MemoryRoot::pointer_in_bounds_fast(p) {
+        return 0;
+    }
+    *(p as *mut u8)
+}
+
+/// Load a u64 from Extism-managed memory
+#[no_mangle]
+pub unsafe fn load_u64(p: Pointer) -> u64 {
+    #[cfg(feature = "bounds-checking")]
+    if !MemoryRoot::pointer_in_bounds_fast(p + core::mem::size_of::<u64>() as u64 - 1) {
+        return 0;
+    }
+    *(p as *mut u64)
+}
+
+/// Load a byte from the input data
+#[no_mangle]
+pub unsafe fn input_load_u8(offset: u64) -> u8 {
+    let root = MemoryRoot::new();
+    #[cfg(feature = "bounds-checking")]
+    if offset >= root.input_length {
+        return 0;
+    }
+    *((root.input_offset + offset) as *mut u8)
+}
+
+/// Load a u64 from the input data
+#[no_mangle]
+pub unsafe fn input_load_u64(offset: u64) -> u64 {
+    let root = MemoryRoot::new();
+    #[cfg(feature = "bounds-checking")]
+    if offset + core::mem::size_of::<u64>() as u64 > root.input_length {
+        return 0;
+    }
+    *((root.input_offset + offset) as *mut u64)
+}
+
+/// Write a byte in Extism-managed memory
+#[no_mangle]
+pub unsafe fn store_u8(p: Pointer, x: u8) {
+    #[cfg(feature = "bounds-checking")]
+    if !MemoryRoot::pointer_in_bounds_fast(p) {
+        return;
+    }
+    *(p as *mut u8) = x;
+}
+
+/// Write a u64 in Extism-managed memory
+#[no_mangle]
+pub unsafe fn store_u64(p: Pointer, x: u64) {
+    #[cfg(feature = "bounds-checking")]
+    if !MemoryRoot::pointer_in_bounds_fast(p + core::mem::size_of::<u64>() as u64 - 1) {
+        return;
+    }
+    *(p as *mut u64) = x;
+}
+
+/// Set the range of the input data in memory
+/// h must always be a handle so that length works on it
+/// len must match length(handle)
+#[no_mangle]
+pub unsafe fn input_set(h: Handle, len: u64) {
+    let root = MemoryRoot::new();
+    #[cfg(feature = "bounds-checking")]
+    {
+        if !root.pointer_in_bounds(h) || !root.pointer_in_bounds(h + len - 1) {
+            return;
+        }
+    }
+    root.input_offset = h;
+    root.input_length = len;
+}
+
+/// Set the range of the output data in memory
+#[no_mangle]
+pub unsafe fn output_set(p: Pointer, len: u64) {
+    let root = MemoryRoot::new();
+    #[cfg(feature = "bounds-checking")]
+    {
+        if !root.pointer_in_bounds(p) || !root.pointer_in_bounds(p + len - 1) {
+            return;
+        }
+    }
+    root.output_offset = p;
+    root.output_length = len;
+}
+
+/// Get the input length
+#[no_mangle]
+pub fn input_length() -> u64 {
+    unsafe { MemoryRoot::new().input_length }
+}
+
+/// Get the input offset in Exitsm-managed memory
+#[no_mangle]
+pub fn input_offset() -> Handle {
+    unsafe { MemoryRoot::new().input_offset }
+}
+
+/// Get the output length
+#[no_mangle]
+pub fn output_length() -> u64 {
+    unsafe { MemoryRoot::new().output_length }
+}
+
+/// Get the output offset in Extism-managed memory
+#[no_mangle]
+pub unsafe fn output_offset() -> Pointer {
+    MemoryRoot::new().output_offset
+}
+
+/// Reset the allocator
+#[no_mangle]
+pub unsafe fn reset() {
+    MemoryRoot::new().reset()
+}
+
+/// Set the error message offset
+#[no_mangle]
+pub unsafe fn error_set(h: Handle) {
+    let root = MemoryRoot::new();
+
+    // Allow ERROR to be set to 0
+    if h == 0 {
+        root.error.store(h, Ordering::SeqCst);
+        return;
+    }
+
+    #[cfg(feature = "bounds-checking")]
+    if !root.pointer_in_bounds(h) {
+        return;
+    }
+    root.error.store(h, Ordering::SeqCst);
+}
+
+/// Get the error message offset, if it's `0` then no error has been set
+#[no_mangle]
+pub unsafe fn error_get() -> Handle {
+    MemoryRoot::new().error.load(Ordering::SeqCst)
+}
+
+/// Get the position of the allocator, this can be used as an indication of how many bytes are currently in-use
+#[no_mangle]
+pub unsafe fn memory_bytes() -> u64 {
+    MemoryRoot::new().length.load(Ordering::Acquire)
+}
+
+#[cfg(test)]
+mod test {
+    use crate::*;
+    use wasm_bindgen_test::*;
+
+    // See https://github.com/extism/extism/pull/659
+    #[wasm_bindgen_test]
+    fn test_659() {
+        unsafe {
+            // Warning: These offsets will need to change if we adjust the kernel memory layout at all
+            reset();
+            assert_eq!(alloc(1065), 77);
+            assert_eq!(alloc(288), 1154);
+            assert_eq!(alloc(128), 1454);
+            assert_eq!(length(1154), 288);
+            assert_eq!(length(1454), 128);
+            free(1454);
+            assert_eq!(alloc(213), 1594);
+            length_unsafe(1594);
+            assert_eq!(alloc(511), 1819);
+            assert_eq!(alloc(4), 1454);
+            assert_eq!(length(1454), 4);
+            assert_eq!(length(1819), 511);
+            assert_eq!(alloc(13), 2342);
+            assert_eq!(length(2342), 13);
+            assert_eq!(alloc(336), 2367);
+            assert_eq!(alloc(1077), 2715);
+            assert_eq!(length(2367), 336);
+            assert_eq!(length(2715), 1077);
+            free(2715);
+            assert_eq!(alloc(1094), 3804);
+            length_unsafe(3804);
+
+            // Allocate 4 bytes, expect to receive address 3788
+            assert_eq!(alloc(4), 3788);
+
+            assert_eq!(alloc(4), 3772);
+            assert_eq!(length(3772), 4);
+
+            // Address 3788 has not been freed yet, so expect it to have 4 bytes allocated
+            assert_eq!(length(3788), 4);
+        }
+    }
+
+    #[wasm_bindgen_test]
+    fn test_oom() {
+        let size = 1024 * 1024 * 5;
+
+        let mut last = 0;
+        for _ in 0..1024 {
+            unsafe {
+                let ptr = alloc(size);
+                last = ptr;
+                if ptr == 0 {
+                    break;
+                }
+                assert_eq!(length(ptr), size);
+            }
+        }
+
+        assert_eq!(last, 0);
+    }
+}

kernel/test.sh

@@ -0,0 +1,5 @@
+# install wasm-bindgen-cli to get wasm-bindgen-runner if it is not installed yet
+which wasm-bindgen-test-runner 1>/dev/null || cargo install -f wasm-bindgen-cli
+
+# run tests with the wasm-bindgen-runner
+CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER=wasm-bindgen-test-runner cargo test --release --target=wasm32-unknown-unknown

libextism.pc

@@ -1,10 +0,0 @@
-prefix=/usr/local
-exec_prefix=${prefix}
-includedir=${prefix}/include
-libdir=${exec_prefix}/lib
-
-Name: extism
-Description: The Extism universal plug-in system.
-Version: 0.0.1
-Cflags: -I${includedir}
-Libs: -L${libdir} -lextism

libextism/API.md

@@ -0,0 +1,261 @@
+# libextism API
+
+We [generate C headers](https://github.com/extism/extism/blob/main/runtime/extism.h) so that any language with a C-compatible FFI can bind functions to the runtime itself and embed Extism. This is how most of the [official SDKs](/docs/concepts/host-sdk) are created.
+
+If you would like to embed Extism into a language that we currently do not support, you should take a look at the header file linked above.
+
+The general set of functions that is necessary to satisfy the runtime requirements is:
+
+### `extism_plugin_new`
+
+Create a new plugin.
+- `wasm`: is a WASM module (wat or wasm) or a JSON encoded manifest
+- `wasm_size`: the length of the `wasm` parameter
+- `functions`: is an array of `ExtismFunction*`
+- `n_functions`: is the number of functions
+- `with_wasi`: enables/disables WASI
+- `errmsg`: error message during plugin creation
+
+```c
+ExtismPlugin extism_plugin_new(const uint8_t *wasm,
+                               ExtismSize wasm_size,
+                               const ExtismFunction **functions,
+                               ExtismSize n_functions,
+                               bool with_wasi,
+                               char **errmsg);
+```
+
+### `extism_plugin_free`
+
+Remove a plugin from the registry and free associated memory.
+
+```c
+void extism_plugin_free(ExtismPlugin *plugin);
+```
+
+### `extism_plugin_config`
+
+Update plugin config values, this will merge with the existing values.
+
+```c
+bool extism_plugin_config(ExtismPlugin *plugin,
+                          const uint8_t *json,
+                          ExtismSize json_size);
+```
+
+### `extism_plugin_function_exists`
+
+Returns true if `func_name` exists.
+
+```c
+bool extism_plugin_function_exists(ExtismPlugin *plugin,
+                                   const char *func_name);
+```
+
+### `extism_plugin_call`
+
+Call a function.
+- `func_name`: is the function to call
+- `data`: is the input data
+- `data_len`: is the length of `data`
+
+```c
+int32_t extism_plugin_call(ExtismPlugin *plugin,
+                           const char *func_name,
+                           const uint8_t *data,
+                           ExtismSize data_len);
+```
+
+### `extism_plugin_error`
+
+Get the error associated with a `Plugin`
+
+```c
+const char *extism_plugin_error(ExtismPlugin *plugin);
+```
+
+### `extism_plugin_output_length`
+
+Get the length of a plugin's output data.
+
+```c
+ExtismSize extism_plugin_output_length(ExtismPlugin *plugin);
+```
+
+### `extism_plugin_output_data`
+
+Get the plugin's output data.
+
+```c
+const uint8_t *extism_plugin_output_data(ExtismPlugin *plugin);
+```
+
+### `extism_plugin_reset`
+
+Reset the Extism runtime, this will invalidate all allocated memory.
+
+```c
+bool extism_plugin_reset(ExtismPlugin *plugin);
+```
+
+### `extism_log_file`
+
+Set log file and level.
+
+```c
+bool extism_log_file(const char *filename, const char *log_level);
+```
+
+### `extism_log_custom`
+
+Enable a custom log handler, this will buffer logs until `extism_log_drain`
+is called Log level should be one of: info, error, trace, debug, warn
+
+```c
+bool extism_log_custom(const char *log_level);
+```
+
+### `extism_log_drain`
+
+Calls the provided callback function for each buffered log line.
+This is only needed when `extism_log_custom` is used.
+
+```c
+void extism_log_drain(void (*handler)(const char *, uintptr_t));
+```
+
+### `extism_version`
+
+Get the Extism version string.
+
+```c
+const char *extism_version(void);
+```
+
+### `extism_current_plugin_memory`
+
+Returns a pointer to the memory of the currently running plugin
+
+```c
+uint8_t *extism_current_plugin_memory(ExtismCurrentPlugin *plugin);
+```
+
+### `extism_current_plugin_memory_alloc`
+
+Allocate a memory block in the currently running plugin
+
+```c
+uint64_t extism_current_plugin_memory_alloc(ExtismCurrentPlugin *plugin, ExtismSize n);
+```
+
+### `extism_current_plugin_memory_length`
+
+Get the length of an allocated block
+
+```c
+ExtismSize extism_current_plugin_memory_length(ExtismCurrentPlugin *plugin, ExtismSize n);
+```
+
+### `extism_current_plugin_memory_free`
+
+Free an allocated memory block
+
+```c
+void extism_current_plugin_memory_free(ExtismCurrentPlugin *plugin, uint64_t ptr);
+```
+
+### `extism_function_new`
+Create a new host function
+- `name`: function name, this should be valid UTF-8
+- `inputs`: argument types
+- `n_inputs`: number of argument types
+- `outputs`: return types
+- `n_outputs`: number of return types
+- `func`: the function to call
+- `user_data`: a pointer that will be passed to the function when it's called
+   this value should live as long as the function exists
+- `free_user_data`: a callback to release the `user_data` value when the resulting
+  `ExtismFunction` is freed.
+
+Returns a new `ExtismFunction` or `null` if the `name` argument is invalid.
+
+```c
+ExtismFunction *extism_function_new(const char *name,
+                                    const ExtismValType *inputs,
+                                    ExtismSize n_inputs,
+                                    const ExtismValType *outputs,
+                                    ExtismSize n_outputs,
+                                    ExtismFunctionType func,
+                                    void *user_data,
+                                    void (*free_user_data)(void *_));
+```
+
+### `extism_function_set_namespace`
+
+Set the namespace of an `ExtismFunction`
+
+```c
+void extism_function_set_namespace(ExtismFunction *ptr, const char *namespace_);
+```
+
+### `extism_function_free`
+
+Free an `ExtismFunction`
+
+```c
+void extism_function_free(ExtismFunction *ptr);
+```
+
+### `extism_plugin_cancel_handle`
+
+Get handle for plugin cancellation
+
+```c
+const ExtismCancelHandle *extism_plugin_cancel_handle(const ExtismPlugin *plugin);
+```
+
+### `extism_plugin_cancel`
+
+Cancel a running plugin from another thread
+
+```c
+bool extism_plugin_cancel(const ExtismCancelHandle *handle);
+```
+
+## Type definitions:
+
+### `ExtismPlugin`
+
+```c
+typedef int32_t ExtismPlugin;
+```
+
+### `ExtismSize`
+
+```c
+typedef uint64_t ExtismSize;
+```
+
+### `ExtismFunction`
+
+`ExtismFunction` is used to register host functions with plugins
+
+```c
+typedef struct ExtismFunction ExtismFunction;
+```
+
+### `ExtismCurrentPlugin`
+
+`ExtismCurrentPlugin` provides access to the currently executing plugin from within a host function
+
+```c
+typedef struct ExtismCurrentPlugin ExtismCurrentPlugin;
+```
+
+### `ExtismCancelHandle`
+
+`ExtismCancelHandle` can be used to cancel a running plugin from another thread
+
+```c
+typedef struct ExtismCancelHandle ExtismCancelHandle;
+```

libextism/CMakeLists.txt

@@ -0,0 +1,34 @@
+project(extism)
+cmake_minimum_required(VERSION 3.22)
+include(FetchContent)
+
+FetchContent_Declare(
+    Corrosion
+    GIT_REPOSITORY https://github.com/corrosion-rs/corrosion.git
+    GIT_TAG v0.4.4
+)
+FetchContent_MakeAvailable(Corrosion)
+
+corrosion_import_crate(MANIFEST_PATH ./Cargo.toml PROFILE release CRATES libextism FEATURES default)
+target_include_directories(extism INTERFACE ../runtime)
+target_include_directories(extism-static INTERFACE ../runtime)
+target_include_directories(extism-shared INTERFACE ../runtime)
+
+configure_file(extism.pc.in extism.pc @ONLY)
+configure_file(extism-static.pc.in extism-static.pc @ONLY)
+
+# corrosion doesn't supporting installing libraries yet
+# https://github.com/corrosion-rs/corrosion/issues/415
+# so we'll do it ourselves
+include(GNUInstallDirs)
+install( FILES ${CMAKE_CURRENT_BINARY_DIR}/libextism.a
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}
+)
+install( FILES ${CMAKE_CURRENT_BINARY_DIR}/libextism.so
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    OPTIONAL
+)
+install( FILES ../runtime/extism.h
+    DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+)
+install(FILES ${CMAKE_CURRENT_BINARY_DIR}/extism.pc ${CMAKE_CURRENT_BINARY_DIR}/extism-static.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)

libextism/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "libextism"
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+homepage.workspace = true
+repository.workspace = true
+version.workspace = true
+description = "libextism"
+
+[lib]
+name = "extism"
+crate-type = ["cdylib", "staticlib"]
+doc = false
+
+[dependencies]
+extism = {workspace = true, path = "../runtime"}
+
+[features]
+default = ["http", "register-http", "register-filesystem"]
+register-http = ["extism/register-http"] # enables wasm to be downloaded using http
+register-filesystem = ["extism/register-filesystem"] # enables wasm to be loaded from disk
+http = ["extism/http"] # enables extism_http_request

libextism/Makefile

@@ -0,0 +1,34 @@
+.PHONY: build
+build:
+	$(CC) -g -o example example.c -lextism
+
+.PHONY: static
+static:
+	$(CC) -g -o example example.c -l:libextism.a -lm
+
+# if needed, set PKG_CONFIG_PATH= to the directory with extism*.pc installed
+LDFLAGS=`pkg-config --libs extism`
+.PHONY: pkg-config
+pkg-config:
+	$(CC) -g -o example example.c $(LDFLAGS)
+
+LDFLAGS_STATIC=`pkg-config --static --libs extism-static`
+.PHONY: pkg-config-static
+pkg-config-static:
+	$(CC) -g -o example example.c $(LDFLAGS_STATIC)
+
+# This produces an entirely static binary
+#
+# MUSL libc is highly recommended over glibc for this purpose as some glibc
+# functionality such as getaddrinfo, iconv depends on dynamically loading glibc.
+#
+# To build and install libextism with musl for x86_64 in the parent directory:
+# make RUST_TARGET=x86_64-unknown-linux-musl && sudo make RUST_TARGET=x86_64-unknown-linux-musl install
+# Then, from this directory you can build with CC=musl-gcc make fully-static
+.PHONY: fully-static
+fully-static:
+	$(CC) -static -g -o example example.c $(LDFLAGS_STATIC)
+
+.PHONY: clean
+clean:
+	rm -f example

libextism/README.md

@@ -0,0 +1,292 @@
+# Extism C SDK
+
+This crate contains no actual code, but is used to generated `libextism` from the [extism](../runtime) crate.
+
+The C SDK is a little different from the other languages because it is generated from the Rust source using cbindgen. It operates at a lower level than the other SDKs because they build higher level abstractions on top of it.
+
+## Building from source
+
+`libextism` can be built using the `Makefile` in the root of the repository:
+
+```shell
+make
+```
+
+`libextism` will be built in `target/release/libextism.*` and the header file can be found in `runtime/extism.h`
+
+## Installation 
+
+The [Extism CLI](https://github.com/extism/cli) can be used to install releases from Github:
+
+```shell
+sudo PATH="$PATH" env extism lib install
+```
+
+Or from source:
+
+```shell
+sudo make install DEST=/usr/local
+```
+
+This will install the shared object into `/usr/local/lib` and `extism.h` into `/usr/local/include`.
+
+
+## Getting Started
+
+To use libextism you should include the header file:
+
+```c
+#include <extism.h>
+```
+
+and link the library:
+
+```
+-lextism
+```
+
+### Creating A Plug-in
+
+The primary concept in Extism is the [plug-in](https://extism.org/docs/concepts/plug-in). You can think of a plug-in as a code module stored in a `.wasm` file.
+
+Since you may not have an Extism plug-in on hand to test, let's load a demo plug-in from the web:
+
+```c
+#include <extism.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+void print_plugin_output(ExtismPlugin *plugin, int32_t rc){
+  if (rc != EXTISM_SUCCESS) {
+    fprintf(stderr, "ERROR: %s\n", extism_plugin_error(plugin));
+    return;
+  }
+
+  size_t outlen = extism_plugin_output_length(plugin);
+  const uint8_t *out = extism_plugin_output_data(plugin);
+  write(STDOUT_FILENO, out, outlen);
+}
+
+int main(void) {
+  const char *manifest = "{\"wasm\": [{\"url\": "
+                         "\"https://github.com/extism/plugins/releases/latest/"
+                         "download/count_vowels.wasm\"}]}";
+
+  char *errmsg = NULL;
+  ExtismPlugin *plugin = extism_plugin_new(
+      (const uint8_t *)manifest, strlen(manifest), NULL, 0, true, &errmsg);
+  if (plugin == NULL) {
+    fprintf(stderr, "ERROR: %s\n", errmsg);
+    extism_plugin_new_error_free(errmsg);
+    exit(1);
+  }
+
+  const char *input = "Hello, world!";
+  print_plugin_output(plugin, extism_plugin_call(plugin, "count_vowels",
+                                  (const uint8_t *)input, strlen(input)));
+  extism_plugin_free(plugin);
+  return 0;
+}
+```
+
+> **Note**: In this case the manifest is a string constant, however it has a rich schema and a lot of options, see the [extism-manifest docs](https://docs.rs/extism-manifest/latest/extism_manifest/) for more details.
+
+### Calling A Plug-in's Exports
+
+This plug-in was written in Rust and it does one thing, it counts vowels in a string. As such, it exposes one "export" function: `count_vowels`. We can call exports using `extism_plugin_call`, then will use `extism_plugin_output_length`
+and `extism_plugin_output_data` to get the result:
+
+```c
+int32_t rc = extism_plugin_call(plugin, "count_vowels",
+                                (const uint8_t *)input, strlen(input));
+if (rc != EXTISM_SUCCESS) {
+  fprintf(stderr, "ERROR: %s\n", extism_plugin_error(plugin));
+  exit(2);
+}
+
+size_t outlen = extism_plugin_output_length(plugin);
+const uint8_t *out = extism_plugin_output_data(plugin);
+write(STDOUT_FILENO, out, outlen);
+```
+
+Will print
+
+```
+{"count": 3, "total": 3, "vowels": "aeiouAEIOU"}
+```
+
+All exports have a simple interface of bytes-in and bytes-out. This plug-in happens to take a string and return a JSON encoded string with a report of results.
+
+### Plug-in State
+
+Plug-ins may be stateful or stateless. Plug-ins can maintain state b/w calls by the use of variables. Our count vowels plug-in remembers the total number of vowels it's ever counted in the "total" key in the result. You can see this by making subsequent calls to the export:
+
+```c
+print_plugin_output(plugin, extism_plugin_call(plugin, "count_vowels",
+                                (const uint8_t *)input, strlen(input)));
+# => {"count": 3, "total": 6, "vowels": "aeiouAEIOU"}
+print_plugin_output(plugin, extism_plugin_call(plugin, "count_vowels",
+                                (const uint8_t *)input, strlen(input)));
+# => {"count": 3, "total": 9, "vowels": "aeiouAEIOU"}
+```
+
+These variables will persist until this plug-in is freed or you initialize a new one.
+
+### Configuration
+
+Plug-ins may optionally take a configuration object. This is a static way to configure the plug-in. Our count-vowels plugin takes an optional configuration to change out which characters are considered vowels. Example:
+
+```c
+const char *input = "Yellow, world!";
+print_plugin_output(plugin, extism_plugin_call(plugin, "count_vowels",
+                                (const uint8_t *)input, strlen(input)));
+# => {"count": 3, "total": 3, "vowels": "aeiouAEIOU"}
+const char * config = "{\"vowels\": \"aeiouyAEIOUY\"}";
+extism_plugin_config(plugin, config, strlen(config));
+print_plugin_output(plugin, extism_plugin_call(plugin, "count_vowels",
+                                (const uint8_t *)input, strlen(input)));
+# => {"count": 4, "total": 4, "vowels": "aeiouyAEIOUY"}
+```
+
+### Host Functions
+
+Let's extend our count-vowels example a little bit: Instead of storing the `total` in an ephemeral plug-in var, let's store it in a persistent key-value store!
+
+Wasm can't use our KV store on it's own. This is where [Host Functions](https://extism.org/docs/concepts/host-functions) come in.
+
+[Host functions](https://extism.org/docs/concepts/host-functions) allow us to grant new capabilities to our plug-ins from our application. They are simply some C functions you write which can be passed down and invoked from any language inside the plug-in.
+
+Let's load the manifest like usual but load up this `count_vowels_kvstore` plug-in from `https://github.com/extism/plugins/releases/latest/download/count_vowels.wasm`
+
+> *Note*: The source code for this is [here](https://github.com/extism/plugins/blob/main/count_vowels_kvstore/src/lib.rs) and is written in rust, but it could be written in any of our PDK languages.
+
+Unlike our previous plug-in, this plug-in expects you to provide host functions that satisfy our its import interface for a KV store.
+
+We want to expose two functions to our plugin, `kv_write(key: String, value: Bytes)` which writes a bytes value to a key and `kv_read(key: String) -> Bytes` which reads the bytes at the given `key`.
+
+```c
+#include <extism.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+// A stubbed out KV store
+typedef struct KVStore KVStore;
+extern KVStore *fake_kv_store_new();
+extern void fake_kv_store_free(KVStore *kv);
+extern void fake_kv_store_set(KVStore *kv, const char *key, size_t keylen,
+                              uint32_t);
+extern const uint32_t fake_kv_store_get(KVStore *kv, const char *key,
+                                        size_t keylen);
+
+// Our host functions to access the fake KV store
+void kv_get(ExtismCurrentPlugin *plugin, const ExtismVal *inputs,
+            size_t ninputs, ExtismVal *outputs, size_t noutputs,
+            void *userdata) {
+  // Cast the userdata pointer
+  KVStore *kv = (KVStore *)userdata;
+
+  // Get the offset to the key in the plugin memory
+  uint64_t offs = inputs[0].v.i64;
+  size_t keylen = extism_current_plugin_memory_length(plugin, offs);
+
+  // Allocate a new block to return
+  uint64_t outoffs =
+      extism_current_plugin_memory_alloc(plugin, sizeof(uint32_t));
+
+  // Load the value from our k/v store
+  uint64_t value = fake_kv_store_get(
+      kv, (const char *)extism_current_plugin_memory(plugin) + offs, keylen);
+
+  // Update the plugin memory
+  *(uint64_t *)(extism_current_plugin_memory(plugin) + outoffs) = value;
+
+  // Return the offset to our allocated block
+  outputs[0].t = PTR;
+  outputs[0].v.i64 = outoffs;
+}
+
+void kv_set(ExtismCurrentPlugin *plugin, const ExtismVal *inputs,
+            size_t ninputs, ExtismVal *outputs, size_t noutputs,
+            void *userdata) {
+  // Cast the userdata pointer
+  KVStore *kv = (KVStore *)userdata;
+
+  // Get the offset to the key in the plugin memory
+  uint64_t keyoffs = inputs[0].v.i64;
+  size_t keylen = extism_current_plugin_memory_length(plugin, keyoffs);
+
+  // Get the offset to the value in the plugin memory
+  uint64_t valueoffs = inputs[1].v.i64;
+  size_t valuelen = extism_current_plugin_memory_length(plugin, valueoffs);
+
+  // Set key => value
+  fake_kv_store_set(
+      kv, (const char *)extism_current_plugin_memory(plugin) + keyoffs, keylen,
+      *(uint32_t *)(extism_current_plugin_memory(plugin) + keyoffs));
+}
+
+int main(void) {
+  KVStore *kv = fake_kv_store_new();
+  const char *manifest = "{\"wasm\": [{\"url\": "
+                         "\"https://github.com/extism/plugins/releases/latest/"
+                         "download/count_vowels_kvstore.wasm\"}]}";
+  const ExtismValType kv_get_inputs[] = {PTR};
+  const ExtismValType kv_get_outputs[] = {PTR};
+  ExtismFunction *kv_get_fn = extism_function_new(
+      "kv_get", kv_get_inputs, 1, kv_get_outputs, 1, kv_get, kv, NULL);
+
+  const ExtismValType kv_set_inputs[] = {PTR};
+  const ExtismValType kv_set_outputs[] = {PTR};
+  ExtismFunction *kv_set_fn = extism_function_new(
+      "kv_set", kv_set_inputs, 1, kv_set_outputs, 1, kv_set, kv, NULL);
+  const ExtismFunction *functions[] = {kv_get_fn};
+  char *errmsg = NULL;
+  ExtismPlugin *plugin = extism_plugin_new(
+      (const uint8_t *)manifest, strlen(manifest), functions, 1, true, &errmsg);
+  if (plugin == NULL) {
+    fprintf(stderr, "ERROR: %s\n", errmsg);
+    extism_plugin_new_error_free(errmsg);
+    exit(1);
+  }
+
+  const char *input = "Hello, world!";
+  print_plugin_output(plugin, extism_plugin_call(plugin, "count_vowels",
+                                                 (const uint8_t *)input,
+                                                 strlen(input)));
+  print_plugin_output(plugin, extism_plugin_call(plugin, "count_vowels",
+                                                 (const uint8_t *)input,
+                                                 strlen(input)));
+
+  extism_plugin_free(plugin);
+  extism_function_free(kv_get_fn);
+  extism_function_free(kv_set_fn);
+  fake_kv_store_free(kv);
+  return 0;
+}
+```
+
+> *Note*: In order to write host functions you should get familiar with the `extism_current_plugin_*` functions.
+
+Now when we invoke the event:
+
+```c
+print_plugin_output(plugin, extism_plugin_call(plugin, "count_vowels",
+                                               (const uint8_t *)input,
+                                               strlen(input)));
+# => Read from key=count-vowels"
+# => Writing value=3 from key=count-vowels"
+# => {"count": 3, "total": 3, "vowels": "aeiouAEIOU"}
+
+print_plugin_output(plugin, extism_plugin_call(plugin, "count_vowels",
+                                               (const uint8_t *)input,
+                                               strlen(input)));
+# => Read from key=count-vowels"
+# => Writing value=6 from key=count-vowels"
+# => {"count": 3, "total": 6, "vowels": "aeiouAEIOU"}
+```
+
+

libextism/example.c

@@ -0,0 +1,92 @@
+#include "../runtime/extism.h"
+
+#include <assert.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+void log_handler(const char *line, uintptr_t length) {
+  fwrite(line, length, 1, stderr);
+}
+
+void hello_world(ExtismCurrentPlugin *plugin, const ExtismVal *inputs,
+                 uint64_t n_inputs, ExtismVal *outputs, uint64_t n_outputs,
+                 void *data) {
+  puts("Hello from C!");
+  puts(data);
+
+  ExtismSize ptr_offs = inputs[0].v.i64;
+
+  uint8_t *buf = extism_current_plugin_memory(plugin) + ptr_offs;
+  uint64_t length = extism_current_plugin_memory_length(plugin, ptr_offs);
+  fwrite(buf, length, 1, stdout);
+  fputc('\n', stdout);
+  outputs[0].v.i64 = inputs[0].v.i64;
+}
+
+void free_data(void *x) { puts("Freeing userdata"); }
+
+uint8_t *read_file(const char *filename, size_t *len) {
+
+  FILE *fp = fopen(filename, "rb");
+  if (fp == NULL) {
+    return NULL;
+  }
+  fseek(fp, 0, SEEK_END);
+  size_t length = ftell(fp);
+  fseek(fp, 0, SEEK_SET);
+
+  uint8_t *data = malloc(length);
+  if (data == NULL) {
+    fclose(fp);
+    return NULL;
+  }
+
+  assert(fread(data, 1, length, fp) == length);
+  fclose(fp);
+
+  *len = length;
+  return data;
+}
+
+int main(int argc, char *argv[]) {
+  if (argc < 2) {
+    fputs("Not enough arguments\n", stderr);
+    exit(1);
+  }
+
+  extism_log_custom("extism=trace,cranelift=trace");
+
+  size_t len = 0;
+  uint8_t *data = read_file("../wasm/code-functions.wasm", &len);
+  ExtismValType inputs[] = {PTR};
+  ExtismValType outputs[] = {PTR};
+  ExtismFunction *f =
+      extism_function_new("hello_world", inputs, 1, outputs, 1, hello_world,
+                          "Hello, again!", free_data);
+
+  char *errmsg = NULL;
+  ExtismPlugin *plugin = extism_plugin_new(
+      data, len, (const ExtismFunction **)&f, 1, true, &errmsg);
+  free(data);
+  if (plugin == NULL) {
+    puts(errmsg);
+    extism_plugin_new_error_free(errmsg);
+    exit(1);
+  }
+
+  assert(extism_plugin_call(plugin, "count_vowels", (uint8_t *)argv[1],
+                            strlen(argv[1])) == 0);
+  ExtismSize out_len = extism_plugin_output_length(plugin);
+  const uint8_t *output = extism_plugin_output_data(plugin);
+  write(STDOUT_FILENO, output, out_len);
+  write(STDOUT_FILENO, "\n", 1);
+  extism_plugin_free(plugin);
+  extism_function_free(f);
+  extism_log_drain(log_handler);
+  return 0;
+}

libextism/extism-static.pc.in

@@ -0,0 +1,10 @@
+prefix=@CMAKE_INSTALL_PREFIX@
+exec_prefix=${prefix}
+libdir=${prefix}/lib
+includedir=${prefix}/include
+Version: 1.0.0
+Name: Extism
+Description: The framework for building with WebAssembly (wasm).
+Libs: -L${libdir} -l:libextism.a
+Libs.private: -lm
+Cflags: -I${includedir}

libextism/extism.pc.in

@@ -0,0 +1,10 @@
+prefix=@CMAKE_INSTALL_PREFIX@
+exec_prefix=${prefix}
+libdir=${prefix}/lib
+includedir=${prefix}/include
+Version: 1.0.0
+Name: Extism
+Description: The framework for building with WebAssembly (wasm).
+Libs: -L${libdir} -lextism
+Libs.private: -lm
+Cflags: -I${includedir}

libextism/src/lib.rs

@@ -0,0 +1,10 @@
+//! This crate is used to generate `libextism` using `extism-runtime`
+
+pub use extism::sdk::*;
+
+#[cfg(test)]
+#[test]
+fn test_version() {
+    let s = unsafe { std::ffi::CStr::from_ptr(extism_version()) };
+    assert!(s.to_bytes() != b"0.0.0");
+}

manifest/Cargo.toml

@@ -1,25 +1,22 @@
 [package]
 name = "extism-manifest"
-version = "0.0.1-rc.6"
-edition = "2021"
-authors = ["The Extism Authors", "oss@extism.org"]
-license = "BSD-3-Clause"
-homepage = "https://extism.org"
-repository = "https://github.com/extism/extism"
 description = "Extism plug-in manifest crate"
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+homepage.workspace = true
+repository.workspace = true
+version.workspace = true
 
 [dependencies]
-serde = {version = "1", features=["derive"]}
-base64 = "0.20.0-alpha"
-schemars = {version = "0.8", optional=true}
+serde = { version = "1", features = ["derive"] }
+base64 = "~0.22"
+schemars = { version = "0.8", optional = true }
+serde_json = "1"
 
 [features]
 json_schema = ["schemars"]
 
-[dev-dependencies]
-serde_json = "1"
-
 [[example]]
 name = "json_schema"
 required-features = ["json_schema"]
-

manifest/schema.json

@@ -1,9 +1,11 @@
 {
   "$schema": "http://json-schema.org/draft-07/schema#",
   "title": "Manifest",
+  "description": "The `Manifest` type is used to configure the runtime and specify how to load modules.",
   "type": "object",
   "properties": {
     "allowed_hosts": {
+      "description": "Specifies which hosts may be accessed via HTTP, if this is empty then no hosts may be accessed. Wildcards may be used.",
       "default": null,
       "type": [
         "array",
@@ -13,7 +15,19 @@
         "type": "string"
       }
     },
+    "allowed_paths": {
+      "description": "Specifies which paths should be made available on disk when using WASI. This is a mapping from the path on disk to the path it should be available inside the plugin. For example, `\".\": \"/tmp\"` would mount the current directory as `/tmp` inside the module",
+      "default": null,
+      "type": [
+        "object",
+        "null"
+      ],
+      "additionalProperties": {
+        "type": "string"
+      }
+    },
     "config": {
+      "description": "Config values are made accessible using the PDK `extism_config_get` function",
       "default": {},
       "type": "object",
       "additionalProperties": {
@@ -21,52 +35,94 @@
       }
     },
     "memory": {
+      "description": "Memory options",
       "default": {
-        "max": null
+        "max_http_response_bytes": null,
+        "max_pages": null,
+        "max_var_bytes": null
       },
       "allOf": [
         {
-          "$ref": "#/definitions/ManifestMemory"
+          "$ref": "#/definitions/MemoryOptions"
         }
       ]
     },
+    "timeout_ms": {
+      "description": "The plugin timeout in milliseconds",
+      "default": null,
+      "type": [
+        "integer",
+        "null"
+      ],
+      "format": "uint64",
+      "minimum": 0.0
+    },
     "wasm": {
+      "description": "WebAssembly modules, the `main` module should be named `main` or listed last",
       "default": [],
       "type": "array",
       "items": {
-        "$ref": "#/definitions/ManifestWasm"
+        "$ref": "#/definitions/Wasm"
       }
     }
   },
+  "additionalProperties": false,
   "definitions": {
-    "ManifestMemory": {
+    "MemoryOptions": {
+      "description": "Configure memory settings",
       "type": "object",
       "properties": {
-        "max": {
+        "max_http_response_bytes": {
+          "description": "The maximum number of bytes allowed in an HTTP response",
+          "default": null,
+          "type": [
+            "integer",
+            "null"
+          ],
+          "format": "uint64",
+          "minimum": 0.0
+        },
+        "max_pages": {
+          "description": "The max number of WebAssembly pages that should be allocated",
           "type": [
             "integer",
             "null"
           ],
           "format": "uint32",
           "minimum": 0.0
+        },
+        "max_var_bytes": {
+          "description": "The maximum number of bytes allowed to be used by plugin vars. Setting this to 0 will disable Extism vars. The default value is 1mb.",
+          "default": 1048576,
+          "type": [
+            "integer",
+            "null"
+          ],
+          "format": "uint64",
+          "minimum": 0.0
         }
-      }
+      },
+      "additionalProperties": false
     },
-    "ManifestWasm": {
+    "Wasm": {
+      "description": "The `Wasm` type specifies how to access a WebAssembly module",
       "anyOf": [
         {
+          "description": "From disk",
           "type": "object",
           "required": [
             "path"
           ],
           "properties": {
             "hash": {
+              "description": "Module hash, if the data loaded from disk or via HTTP doesn't match an error will be raised",
               "type": [
                 "string",
                 "null"
               ]
             },
             "name": {
+              "description": "Module name, this is used by Extism to determine which is the `main` module",
               "type": [
                 "string",
                 "null"
@@ -75,45 +131,72 @@
             "path": {
               "type": "string"
             }
-          }
+          },
+          "additionalProperties": false
         },
         {
+          "description": "From memory",
           "type": "object",
           "required": [
             "data"
           ],
           "properties": {
             "data": {
-              "type": "string",
-              "format": "string"
+              "type": [
+                "string",
+                "object"
+              ],
+              "required": [
+                "len",
+                "ptr"
+              ],
+              "properties": {
+                "len": {
+                  "type": "integer",
+                  "format": "uint64",
+                  "minimum": 0.0
+                },
+                "ptr": {
+                  "type": "integer",
+                  "format": "uint64",
+                  "minimum": 0.0
+                }
+              },
+              "additionalProperties": false
             },
             "hash": {
+              "description": "Module hash, if the data loaded from disk or via HTTP doesn't match an error will be raised",
               "type": [
                 "string",
                 "null"
               ]
             },
             "name": {
+              "description": "Module name, this is used by Extism to determine which is the `main` module",
               "type": [
                 "string",
                 "null"
               ]
             }
-          }
+          },
+          "additionalProperties": false
         },
         {
+          "description": "Via HTTP",
           "type": "object",
           "required": [
             "url"
           ],
           "properties": {
             "hash": {
+              "description": "Module hash, if the data loaded from disk or via HTTP doesn't match an error will be raised",
               "type": [
                 "string",
                 "null"
               ]
             },
-            "header": {
+            "headers": {
+              "description": "Request headers",
               "default": {},
               "type": "object",
               "additionalProperties": {
@@ -121,21 +204,25 @@
               }
             },
             "method": {
+              "description": "Request method",
               "type": [
                 "string",
                 "null"
               ]
             },
             "name": {
+              "description": "Module name, this is used by Extism to determine which is the `main` module",
               "type": [
                 "string",
                 "null"
               ]
             },
             "url": {
+              "description": "The request URL",
               "type": "string"
             }
-          }
+          },
+          "additionalProperties": false
         }
       ]
     }

manifest/src/lib.rs

@@ -1,96 +1,435 @@
 use std::collections::BTreeMap;
+use std::path::{Path, PathBuf};
 
-#[derive(Default, serde::Serialize, serde::Deserialize)]
+#[deprecated]
+pub type ManifestMemory = MemoryOptions;
+
+/// Configure memory settings
+#[derive(Default, Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
 #[cfg_attr(feature = "json_schema", derive(schemars::JsonSchema))]
-pub struct ManifestMemory {
-    pub max: Option<u32>,
+#[serde(deny_unknown_fields)]
+pub struct MemoryOptions {
+    /// The max number of WebAssembly pages that should be allocated
+    #[serde(alias = "max")]
+    pub max_pages: Option<u32>,
+
+    /// The maximum number of bytes allowed in an HTTP response
+    #[serde(default)]
+    pub max_http_response_bytes: Option<u64>,
+
+    /// The maximum number of bytes allowed to be used by plugin vars. Setting this to 0
+    /// will disable Extism vars. The default value is 1mb.
+    #[serde(default = "default_var_bytes")]
+    pub max_var_bytes: Option<u64>,
 }
 
-#[derive(serde::Serialize, serde::Deserialize)]
+impl MemoryOptions {
+    /// Create an empty `MemoryOptions` value
+    pub fn new() -> Self {
+        Default::default()
+    }
+
+    /// Set max pages
+    pub fn with_max_pages(mut self, pages: u32) -> Self {
+        self.max_pages = Some(pages);
+        self
+    }
+
+    /// Set max HTTP response size
+    pub fn with_max_http_response_bytes(mut self, bytes: u64) -> Self {
+        self.max_http_response_bytes = Some(bytes);
+        self
+    }
+
+    /// Set max size of Extism vars
+    pub fn with_max_var_bytes(mut self, bytes: u64) -> Self {
+        self.max_var_bytes = Some(bytes);
+        self
+    }
+}
+
+fn default_var_bytes() -> Option<u64> {
+    Some(1024 * 1024)
+}
+
+/// Generic HTTP request structure
+#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
 #[cfg_attr(feature = "json_schema", derive(schemars::JsonSchema))]
+#[serde(deny_unknown_fields)]
 pub struct HttpRequest {
+    /// The request URL
     pub url: String,
+
+    /// Request headers
     #[serde(default)]
-    pub header: std::collections::BTreeMap<String, String>,
+    #[serde(alias = "header")]
+    pub headers: std::collections::BTreeMap<String, String>,
+
+    /// Request method
     pub method: Option<String>,
 }
 
 impl HttpRequest {
+    /// Create a new `HttpRequest` to the given URL
     pub fn new(url: impl Into<String>) -> HttpRequest {
         HttpRequest {
             url: url.into(),
-            header: Default::default(),
+            headers: Default::default(),
             method: None,
         }
     }
 
+    /// Update the method
     pub fn with_method(mut self, method: impl Into<String>) -> HttpRequest {
         self.method = Some(method.into());
         self
     }
 
+    /// Add a header
     pub fn with_header(mut self, key: impl Into<String>, value: impl Into<String>) -> HttpRequest {
-        self.header.insert(key.into(), value.into());
+        self.headers.insert(key.into(), value.into());
         self
     }
 }
 
-#[derive(serde::Serialize, serde::Deserialize)]
+/// Provides additional metadata about a Webassembly module
+#[derive(Default, Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
+#[cfg_attr(feature = "json_schema", derive(schemars::JsonSchema))]
+#[serde(deny_unknown_fields)]
+pub struct WasmMetadata {
+    /// Module name, this is used by Extism to determine which is the `main` module
+    pub name: Option<String>,
+
+    /// Module hash, if the data loaded from disk or via HTTP doesn't match an error will be raised
+    pub hash: Option<String>,
+}
+
+impl From<HttpRequest> for Wasm {
+    fn from(req: HttpRequest) -> Self {
+        Wasm::Url {
+            req,
+            meta: WasmMetadata::default(),
+        }
+    }
+}
+
+impl From<std::path::PathBuf> for Wasm {
+    fn from(path: std::path::PathBuf) -> Self {
+        Wasm::File {
+            path,
+            meta: WasmMetadata::default(),
+        }
+    }
+}
+
+impl From<Vec<u8>> for Wasm {
+    fn from(data: Vec<u8>) -> Self {
+        Wasm::Data {
+            data,
+            meta: WasmMetadata::default(),
+        }
+    }
+}
+
+#[deprecated]
+pub type ManifestWasm = Wasm;
+
+/// The `Wasm` type specifies how to access a WebAssembly module
+#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
 #[cfg_attr(feature = "json_schema", derive(schemars::JsonSchema))]
 #[serde(untagged)]
-pub enum ManifestWasm {
+#[serde(deny_unknown_fields)]
+pub enum Wasm {
+    /// From disk
     File {
-        path: std::path::PathBuf,
-        name: Option<String>,
-        hash: Option<String>,
+        path: PathBuf,
+        #[serde(flatten)]
+        meta: WasmMetadata,
     },
+
+    /// From memory
     Data {
-        #[serde(with = "base64")]
-        #[cfg_attr(feature = "json_schema", schemars(schema_with = "base64_schema"))]
+        #[serde(with = "wasmdata")]
+        #[cfg_attr(feature = "json_schema", schemars(schema_with = "wasmdata_schema"))]
         data: Vec<u8>,
-        name: Option<String>,
-        hash: Option<String>,
+        #[serde(flatten)]
+        meta: WasmMetadata,
     },
+
+    /// Via HTTP
     Url {
         #[serde(flatten)]
         req: HttpRequest,
-        name: Option<String>,
-        hash: Option<String>,
+        #[serde(flatten)]
+        meta: WasmMetadata,
     },
 }
 
+impl Wasm {
+    /// Load Wasm from a path
+    pub fn file(path: impl AsRef<std::path::Path>) -> Self {
+        Wasm::File {
+            path: path.as_ref().to_path_buf(),
+            meta: Default::default(),
+        }
+    }
+
+    /// Load Wasm directly from a buffer
+    pub fn data(data: impl Into<Vec<u8>>) -> Self {
+        Wasm::Data {
+            data: data.into(),
+            meta: Default::default(),
+        }
+    }
+
+    /// Load Wasm from a URL
+    pub fn url(url: impl Into<String>) -> Self {
+        Wasm::Url {
+            req: HttpRequest {
+                url: url.into(),
+                headers: Default::default(),
+                method: None,
+            },
+            meta: Default::default(),
+        }
+    }
+
+    /// Load Wasm from an HTTP request
+    pub fn http(req: impl Into<HttpRequest>) -> Self {
+        Wasm::Url {
+            req: req.into(),
+            meta: Default::default(),
+        }
+    }
+
+    /// Get the metadata
+    pub fn meta(&self) -> &WasmMetadata {
+        match self {
+            Wasm::File { path: _, meta } => meta,
+            Wasm::Data { data: _, meta } => meta,
+            Wasm::Url { req: _, meta } => meta,
+        }
+    }
+
+    /// Get mutable access to the metadata
+    pub fn meta_mut(&mut self) -> &mut WasmMetadata {
+        match self {
+            Wasm::File { path: _, meta } => meta,
+            Wasm::Data { data: _, meta } => meta,
+            Wasm::Url { req: _, meta } => meta,
+        }
+    }
+
+    /// Update Wasm module name
+    pub fn with_name(mut self, name: impl Into<String>) -> Self {
+        self.meta_mut().name = Some(name.into());
+        self
+    }
+
+    /// Update Wasm module hash
+    pub fn with_hash(mut self, hash: impl Into<String>) -> Self {
+        self.meta_mut().hash = Some(hash.into());
+        self
+    }
+}
+
+#[derive(Default, Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
+#[cfg_attr(feature = "json_schema", derive(schemars::JsonSchema))]
+#[serde(deny_unknown_fields)]
+struct DataPtrLength {
+    ptr: u64,
+    len: u64,
+}
+
 #[cfg(feature = "json_schema")]
-fn base64_schema(gen: &mut schemars::gen::SchemaGenerator) -> schemars::schema::Schema {
+fn wasmdata_schema(gen: &mut schemars::gen::SchemaGenerator) -> schemars::schema::Schema {
     use schemars::{schema::SchemaObject, JsonSchema};
     let mut schema: SchemaObject = <String>::json_schema(gen).into();
-    schema.format = Some("string".to_owned());
+    let objschema: SchemaObject = <DataPtrLength>::json_schema(gen).into();
+    let types = schemars::schema::SingleOrVec::<schemars::schema::InstanceType>::Vec(vec![
+        schemars::schema::InstanceType::String,
+        schemars::schema::InstanceType::Object,
+    ]);
+    schema.instance_type = Some(types);
+    schema.object = objschema.object;
     schema.into()
 }
 
-#[derive(Default, serde::Serialize, serde::Deserialize)]
+/// The `Manifest` type is used to configure the runtime and specify how to load modules.
+#[derive(Default, Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
 #[cfg_attr(feature = "json_schema", derive(schemars::JsonSchema))]
+#[serde(deny_unknown_fields)]
 pub struct Manifest {
+    /// WebAssembly modules, the `main` module should be named `main` or listed last
     #[serde(default)]
-    pub wasm: Vec<ManifestWasm>,
+    pub wasm: Vec<Wasm>,
+
+    /// Memory options
     #[serde(default)]
-    pub memory: ManifestMemory,
+    pub memory: MemoryOptions,
+
+    /// Config values are made accessible using the PDK `extism_config_get` function
     #[serde(default)]
     pub config: BTreeMap<String, String>,
     #[serde(default)]
+
+    /// Specifies which hosts may be accessed via HTTP, if this is empty then
+    /// no hosts may be accessed. Wildcards may be used.
     pub allowed_hosts: Option<Vec<String>>,
+
+    /// Specifies which paths should be made available on disk when using WASI. This is a mapping from
+    /// the path on disk to the path it should be available inside the plugin.
+    /// For example, `".": "/tmp"` would mount the current directory as `/tmp` inside the module
+    #[serde(default)]
+    pub allowed_paths: Option<BTreeMap<PathBuf, PathBuf>>,
+
+    /// The plugin timeout in milliseconds
+    #[serde(default)]
+    pub timeout_ms: Option<u64>,
 }
 
-mod base64 {
+impl Manifest {
+    /// Create a new manifest
+    pub fn new(wasm: impl IntoIterator<Item = impl Into<Wasm>>) -> Manifest {
+        Manifest {
+            wasm: wasm.into_iter().map(|x| x.into()).collect(),
+            ..Default::default()
+        }
+    }
+
+    pub fn with_wasm(mut self, wasm: impl Into<Wasm>) -> Self {
+        self.wasm.push(wasm.into());
+        self
+    }
+
+    /// Disallow HTTP requests to all hosts
+    pub fn disallow_all_hosts(mut self) -> Self {
+        self.allowed_hosts = Some(vec![]);
+        self
+    }
+
+    /// Set memory options
+    pub fn with_memory_options(mut self, memory: MemoryOptions) -> Self {
+        self.memory = memory;
+        self
+    }
+
+    /// Set MemoryOptions::memory_max
+    pub fn with_memory_max(mut self, max: u32) -> Self {
+        self.memory.max_pages = Some(max);
+        self
+    }
+
+    /// Add a hostname to `allowed_hosts`
+    pub fn with_allowed_host(mut self, host: impl Into<String>) -> Self {
+        match &mut self.allowed_hosts {
+            Some(h) => {
+                h.push(host.into());
+            }
+            None => self.allowed_hosts = Some(vec![host.into()]),
+        }
+
+        self
+    }
+
+    /// Set `allowed_hosts`
+    pub fn with_allowed_hosts(mut self, hosts: impl Iterator<Item = String>) -> Self {
+        self.allowed_hosts = Some(hosts.collect());
+        self
+    }
+
+    /// Add a path to `allowed_paths`
+    pub fn with_allowed_path(mut self, src: impl AsRef<Path>, dest: impl AsRef<Path>) -> Self {
+        let src = src.as_ref().to_path_buf();
+        let dest = dest.as_ref().to_path_buf();
+        match &mut self.allowed_paths {
+            Some(p) => {
+                p.insert(src, dest);
+            }
+            None => {
+                let mut p = BTreeMap::new();
+                p.insert(src, dest);
+                self.allowed_paths = Some(p);
+            }
+        }
+
+        self
+    }
+
+    /// Set `allowed_paths`
+    pub fn with_allowed_paths(mut self, paths: impl Iterator<Item = (PathBuf, PathBuf)>) -> Self {
+        self.allowed_paths = Some(paths.collect());
+        self
+    }
+
+    /// Set `config`
+    pub fn with_config(
+        mut self,
+        c: impl Iterator<Item = (impl Into<String>, impl Into<String>)>,
+    ) -> Self {
+        for (k, v) in c {
+            self.config.insert(k.into(), v.into());
+        }
+        self
+    }
+
+    /// Set a single `config` key
+    pub fn with_config_key(mut self, k: impl Into<String>, v: impl Into<String>) -> Self {
+        self.config.insert(k.into(), v.into());
+        self
+    }
+
+    /// Set `timeout_ms`, which will interrupt a plugin function's execution if it meets or
+    /// exceeds this value. When an interrupt is made, the plugin will not be able to recover and
+    /// continue execution.
+    pub fn with_timeout(mut self, timeout: std::time::Duration) -> Self {
+        self.timeout_ms = Some(timeout.as_millis() as u64);
+        self
+    }
+}
+
+mod wasmdata {
+    use crate::DataPtrLength;
+    use base64::{engine::general_purpose, Engine as _};
     use serde::{Deserialize, Serialize};
     use serde::{Deserializer, Serializer};
+    use std::slice;
 
     pub fn serialize<S: Serializer>(v: &Vec<u8>, s: S) -> Result<S::Ok, S::Error> {
-        let base64 = base64::encode(v);
+        let base64 = general_purpose::STANDARD.encode(v.as_slice());
         String::serialize(&base64, s)
     }
 
     pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Vec<u8>, D::Error> {
-        let base64 = String::deserialize(d)?;
-        base64::decode(base64.as_bytes()).map_err(serde::de::Error::custom)
+        #[derive(Deserialize)]
+        #[serde(untagged)]
+        enum WasmDataTypes {
+            String(String),
+            DataPtrLength(DataPtrLength),
+        }
+        Ok(match WasmDataTypes::deserialize(d)? {
+            WasmDataTypes::String(string) => general_purpose::STANDARD
+                .decode(string.as_bytes())
+                .map_err(serde::de::Error::custom)?,
+            WasmDataTypes::DataPtrLength(ptrlen) => {
+                let slice =
+                    unsafe { slice::from_raw_parts(ptrlen.ptr as *const u8, ptrlen.len as usize) };
+                slice.to_vec()
+            }
+        })
+    }
+}
+
+impl<'a> From<Manifest> for std::borrow::Cow<'a, [u8]> {
+    fn from(m: Manifest) -> Self {
+        let s = serde_json::to_vec(&m).unwrap();
+        std::borrow::Cow::Owned(s)
+    }
+}
+
+impl<'a> From<&Manifest> for std::borrow::Cow<'a, [u8]> {
+    fn from(m: &Manifest) -> Self {
+        let s = serde_json::to_vec(&m).unwrap();
+        std::borrow::Cow::Owned(s)
     }
 }

node/.gitignore

@@ -1,3 +0,0 @@
-dist/
-coverage/
-doc/

node/.prettierignore

@@ -1,2 +0,0 @@
-dist
-coverage