Compare commits

..

3 Commits

Author SHA1 Message Date
zach
22e8a1bfed wip: copy header file 2023-12-08 11:18:40 -08:00
zach
a872ac42f3 wip: remove postInstall 2023-12-08 11:00:53 -08:00
zach
03b8b34a16 wip: add nix flake 2023-12-07 14:07:59 -08:00
78 changed files with 3899 additions and 3911 deletions

1
.github/CODEOWNERS vendored
View File

@@ -1 +0,0 @@
* @zshipko

Binary file not shown.

Before

Width:  |  Height:  |  Size: 465 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 465 KiB

View File

@@ -9,8 +9,33 @@ updates:
directory: "/" # Location of package manifests
schedule:
interval: "weekly"
ignore:
- dependency-name: "wasmtime"
- dependency-name: "wasi-common"
- dependency-name: "wiggle"
- package-ecosystem: "pip"
directory: "python"
schedule:
interval: "weekly"
- package-ecosystem: "mix"
directory: "elixir"
schedule:
interval: "weekly"
- package-ecosystem: "npm"
directory: "node"
schedule:
interval: "weekly"
- package-ecosystem: "composer"
directory: "/"
schedule:
interval: "weekly"
- package-ecosystem: "bundler"
directory: "ruby"
schedule:
interval: "weekly"
- package-ecosystem: "gomod"
directory: "/"
schedule:
interval: "weekly"

View File

@@ -1,4 +1,4 @@
on:
on:
pull_request:
paths:
- .github/actions/extism/**
@@ -52,7 +52,7 @@ jobs:
shell: bash
run: cargo build --release -p ${{ env.LIBEXTISM_CRATE }}
- name: Upload artifact
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: libextism-${{ matrix.os }}
path: |
@@ -116,9 +116,4 @@ jobs:
path: target/**
key: ${{ runner.os }}-target-${{ github.sha }}
- run: cargo install cargo-criterion
- run:
cargo criterion
- run: |
git fetch
git checkout main
cargo criterion
- run: cargo criterion

View File

@@ -21,9 +21,6 @@ jobs:
target: wasm32-unknown-unknown
- uses: Swatinem/rust-cache@v2
- name: install wasm-tools
uses: bytecodealliance/actions/wasm-tools/setup@v1
- name: Install deps
run: |
sudo apt install wabt --yes
@@ -42,7 +39,6 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
if: ${{ env.GIT_EXIT_CODE }} != 0
with:
author: "zshipko <zshipko@users.noreply.github.com>"
title: "update(kernel): extism-runtime.wasm in ${{ github.event.pull_request.head.ref }}"
body: "Automated PR to update `runtime/src/extism-runtime.wasm` in PR #${{ github.event.number }}"
base: "${{ github.event.pull_request.head.ref }}"

View File

@@ -1,6 +1,4 @@
on:
release:
types: [published, edited]
workflow_dispatch:
name: Release .NET Native NuGet Packages
@@ -20,13 +18,10 @@ jobs:
uses: actions/setup-dotnet@v3.0.3
with:
dotnet-version: 7.x
- name: download release
run: |
tag='${{ github.ref }}'
tag="${tag/refs\/tags\//}"
gh release download "$tag" -p 'libextism-*.tar.gz'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- uses: dawidd6/action-download-artifact@v2
with:
workflow: release.yml
name: release-artifacts
- name: Extract Archive
run: |
extract_archive() {

View File

@@ -36,18 +36,6 @@ jobs:
override: true
target: ${{ matrix.target }}
- name: Release Rust convert-macros Crate
env:
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_API_TOKEN }}
run: |
version=$(cargo metadata --format-version=1 | jq -r '.packages[] | select(.name == "extism") | .version')
if ! &>/dev/null curl -sLIf https://crates.io/api/v1/crates/extism-convert-macros/${version}/download; then
cargo publish --manifest-path convert-macros/Cargo.toml --allow-dirty
else
echo "already published ${version}"
fi
- name: Release Rust Convert Crate
env:
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_API_TOKEN }}

View File

@@ -7,6 +7,11 @@ on:
name: Release
env:
RUNTIME_MANIFEST: runtime/Cargo.toml
RUNTIME_CRATE: libextism
RUSTFLAGS: -C target-feature=-crt-static
ARTIFACT_DIR: release-artifacts
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@@ -16,11 +21,6 @@ jobs:
release:
name: ${{ matrix.os }} ${{ matrix.target }}
runs-on: ${{ matrix.os }}-latest
env:
RUNTIME_MANIFEST: runtime/Cargo.toml
RUNTIME_CRATE: libextism
RUSTFLAGS: -C target-feature=-crt-static
ARTIFACT_DIR: release-artifacts-${{ matrix.os }}-${{ matrix.target }}
strategy:
matrix:
include:
@@ -28,56 +28,48 @@ jobs:
target: 'x86_64-apple-darwin'
artifact: 'libextism.dylib'
static-artifact: 'libextism.a'
static-dll-artifact: ''
pc-in: 'extism.pc.in'
static-pc-in: 'extism-static.pc.in'
- os: 'macos'
target: 'aarch64-apple-darwin'
artifact: 'libextism.dylib'
static-artifact: 'libextism.a'
static-dll-artifact: ''
pc-in: 'extism.pc.in'
static-pc-in: 'extism-static.pc.in'
- os: 'ubuntu'
target: 'aarch64-unknown-linux-gnu'
artifact: 'libextism.so'
static-artifact: 'libextism.a'
static-dll-artifact: ''
pc-in: 'extism.pc.in'
static-pc-in: 'extism-static.pc.in'
- os: 'ubuntu'
target: 'aarch64-unknown-linux-musl'
artifact: 'libextism.so'
static-artifact: 'libextism.a'
static-dll-artifact: ''
pc-in: 'extism.pc.in'
static-pc-in: 'extism-static.pc.in'
- os: 'ubuntu'
target: 'x86_64-unknown-linux-gnu'
artifact: 'libextism.so'
static-artifact: 'libextism.a'
static-dll-artifact: ''
pc-in: 'extism.pc.in'
static-pc-in: 'extism-static.pc.in'
- os: 'ubuntu'
target: 'x86_64-unknown-linux-musl'
artifact: 'libextism.so'
artifact: ''
static-artifact: 'libextism.a'
static-dll-artifact: ''
pc-in: 'extism.pc.in'
pc-in: ''
static-pc-in: 'extism-static.pc.in'
- os: 'windows'
target: 'x86_64-pc-windows-gnu'
artifact: 'extism.dll'
static-artifact: 'libextism.a'
static-dll-artifact: 'libextism.dll.a'
pc-in: 'extism.pc.in'
static-pc-in: 'extism-static.pc.in'
- os: 'windows'
target: 'x86_64-pc-windows-msvc'
artifact: 'extism.dll'
static-artifact: 'extism.lib'
static-dll-artifact: 'extism.dll.lib'
pc-in: ''
static-pc-in: ''
@@ -98,11 +90,13 @@ jobs:
pyproject="$(cat extism-maturin/pyproject.toml)"
<<<"$pyproject" >extism-maturin/pyproject.toml sed -e 's/^version = "0.0.0.replaced-by-ci"/version = "'"$version"'"/g'
- uses: actions-rust-lang/setup-rust-toolchain@v1
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
target: ${{ matrix.target }}
override: true
toolchain: stable
profile: minimal
override: true
target: ${{ matrix.target }}
- uses: Swatinem/rust-cache@v2
with:
@@ -111,15 +105,11 @@ jobs:
cache-on-failure: "true"
- name: Build Target (${{ matrix.os }} ${{ matrix.target }})
if: ${{ matrix.os != 'windows' }}
run: |
cargo install cross
cross build --release --target ${{ matrix.target }} -p ${{ env.RUNTIME_CRATE }}
- name: Build Target (${{ matrix.os }} ${{ matrix.target }})
if: ${{ matrix.os == 'windows' }}
run: |
cargo build --release --target ${{ matrix.target }} -p ${{ env.RUNTIME_CRATE }}
uses: actions-rs/cargo@v1
with:
use-cross: ${{ matrix.os != 'windows' }}
command: build
args: --release --target ${{ matrix.target }} -p ${{ env.RUNTIME_CRATE }}
- uses: actions/setup-python@v4
with:
@@ -168,8 +158,7 @@ jobs:
cp LICENSE ${SRC_DIR}
tar -C ${SRC_DIR} -czvf ${ARCHIVE} extism.h \
${{ matrix.artifact }} ${{ matrix.static-artifact }} \
${{ matrix.pc-in }} ${{ matrix.static-pc-in }} \
${{ matrix.static-dll-artifact }}
${{ matrix.pc-in }} ${{ matrix.static-pc-in }}
ls -ll ${ARCHIVE}
if &>/dev/null which shasum; then
@@ -192,7 +181,7 @@ jobs:
ls -ll ${DEST_DIR}
- name: Upload Artifact to Summary
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: ${{ env.ARTIFACT_DIR }}
path: ${{ env.ARTIFACT_DIR }}
@@ -211,10 +200,9 @@ jobs:
needs: [release]
if: github.ref == 'refs/heads/main'
steps:
- uses: actions/download-artifact@v4
- uses: actions/download-artifact@v3
with:
pattern: release-artifacts-*
merge-multiple: true
name: ${{ env.ARTIFACT_DIR }}
- uses: "marvinpinto/action-automatic-releases@latest"
with:

1
.gitignore vendored
View File

@@ -1,7 +1,6 @@
target
runtime/ast.json
runtime/target
Cargo.lock
.DS_Store
.vscode
**/libextism.dylib

2910
Cargo.lock generated Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
[workspace]
resolver = "2"
members = ["extism-maturin", "manifest", "runtime", "libextism", "convert", "convert-macros"]
members = ["extism-maturin", "manifest", "runtime", "libextism", "convert"]
exclude = ["kernel"]
[workspace.package]
@@ -14,5 +14,4 @@ version = "0.0.0+replaced-by-ci"
[workspace.dependencies]
extism = { path = "./runtime", version = "0.0.0+replaced-by-ci" }
extism-convert = { path = "./convert", version = "0.0.0+replaced-by-ci" }
extism-convert-macros = { path = "./convert-macros", version = "0.0.0+replaced-by-ci" }
extism-manifest = { path = "./manifest", version = "0.0.0+replaced-by-ci" }

View File

@@ -4,12 +4,10 @@ AEXT=a
FEATURES?=default
DEFAULT_FEATURES?=yes
RUST_TARGET?=
EXTRA_LIBS=
UNAME := $(shell uname -s)
ifeq ($(UNAME),Darwin)
SOEXT=dylib
EXTRA_LIBS=-framework Security
endif
ifeq ($(DEFAULT_FEATURES),no)
@@ -31,8 +29,7 @@ endif
build:
cargo build --release $(FEATURE_FLAGS) --manifest-path libextism/Cargo.toml $(TARGET_FLAGS)
sed -e "s%@CMAKE_INSTALL_PREFIX@%$(DEST)%" libextism/extism.pc.in > libextism/extism.pc
sed -e "s%@CMAKE_INSTALL_PREFIX@%$(DEST)%" \
-e "s%Libs: %Libs: $(EXTRA_LIBS) %" libextism/extism-static.pc.in > libextism/extism-static.pc
sed -e "s%@CMAKE_INSTALL_PREFIX@%$(DEST)%" libextism/extism-static.pc.in > libextism/extism-static.pc
bench:
@(cargo criterion $(TARGET_FLAGS) || echo 'For nicer output use cargo-criterion: `cargo install cargo-criterion` - using `cargo bench`') && cargo bench $(TARGET_FLAGS)

205
README.md
View File

@@ -1,188 +1,65 @@
<div align="center">
<a href="https://extism.org">
<picture>
<source media="(prefers-color-scheme: dark)" srcset=".github/assets/logo-horizontal-darkmode.png">
<img alt="Extism - the WebAssembly framework" width="75%" style="max-width: 600px" src=".github/assets/logo-horizontal.png">
</picture>
</a>
# [Extism](https://extism.org)
[![Discord](https://img.shields.io/discord/1011124058408112148?color=%23404eed&label=Community%20Chat&logo=Discord&logoColor=%23404eed)](https://extism.org/discord)
[![Discord](https://img.shields.io/discord/1011124058408112148?color=%23404eed&label=Community%20Chat&logo=Discord&logoColor=%23404eed)](https://discord.gg/cx3usBCWnc)
![GitHub Org's stars](https://img.shields.io/github/stars/extism)
![Downloads](https://img.shields.io/crates/d/extism-manifest)
![GitHub all releases](https://img.shields.io/github/downloads/extism/extism/total)
![GitHub License](https://img.shields.io/github/license/extism/extism)
![GitHub release (with filter)](https://img.shields.io/github/v/release/extism/extism)
</div>
The universal plug-in system. Run WebAssembly extensions inside your app. Use idiomatic Host SDKs for [Go](https://github.com/extism/go-sdk#readme),
[Ruby](https://github.com/extism/ruby-sdk#readme),
[Python](https://github.com/extism/python-sdk#readme),
[JavaScript](https://github.com/extism/js-sdk#readme),
[Rust](/runtime/#readme),
[C](libextism/#readme),
[C++](https://github.com/extism/cpp-sdk/#readme),
[OCaml](https://github.com/extism/ocaml-sdk#readme),
[Haskell](https://github.com/extism/haskell-sdk#readme),
[PHP](https://github.com/extism/php-sdk#readme),
[Elixir](https://github.com/extism/elixir-sdk#readme),
[.NET](https://github.com/extism/dotnet-sdk#readme),
[Java](https://github.com/extism/java-sdk#readme),
[Zig](https://github.com/extism/zig-sdk#readme),
[D](https://github.com/extism/d-sdk#readme),
&amp; more (others coming soon).
# Overview
Plug-in development kits (PDK) for plug-in authors supported in [Rust](https://github.com/extism/rust-pdk#readme), [AssemblyScript](https://github.com/extism/assemblyscript-pdk#readme), [Go](https://github.com/extism/go-pdk#readme), [C/C++](https://github.com/extism/c-pdk#readme), [Haskell](https://github.com/extism/haskell-pdk#readme), [JavaScript](https://github.com/extism/js-pdk#readme), [C#](https://github.com/extism/dotnet-pdk#readme), [F#](https://github.com/extism/dotnet-pdk#readme) and [Zig](https://github.com/extism/zig-pdk#readme).
Extism is a lightweight framework for building with WebAssembly (Wasm). It
supports running Wasm code on servers, the edge, CLIs, IoT, browsers and
everything in between. Extism is designed to be "universal" in that it supports
a common interface, no matter where it runs.
<p align="center">
<img style="width: 70%;" src="https://user-images.githubusercontent.com/7517515/210286900-39b144fd-1b26-4dd0-b7a9-2b5755bc174d.png" alt="Extism embedded SDK language support"/>
</p>
> **Note:** One of the primary use cases for Extism is **building extensible
> software & plugins**. You want to be able to execute arbitrary, untrusted code
> from your users? Extism makes this safe and practical to do.
Add a flexible, secure, and _bLaZiNg FaSt_ plug-in system to your project. Server, desktop, mobile, web, database -- you name it. Enable users to write and execute safe extensions to your software in **3 easy steps:**
Additionally, Extism adds some extra utilities on top of standard Wasm runtimes.
For example, we support persistent memory/module-scope variables, secure &
host-controlled HTTP without WASI, runtime limiters & timers, simpler host
function linking, and more. Extism users build:
### 1. Import
- plug-in systems
- FaaS platforms
- code generators
- web applications
- & much more...
Import an Extism Host SDK into your code as a library dependency.
# Supported Targets
### 2. Integrate
We currently provide releases for the following targets:
Identify the place(s) in your code where some arbitrary logic should run (the plug-in!), returning your code some results.
- aarch64-apple-darwin
- aarch64-unknown-linux-gnu
- aarch64-unknown-linux-musl
- x86_64-apple-darwin
- x86_64-pc-windows-gnu
- x86_64-pc-windows-msvc
- x86_64-unknown-linux-gnu
- x86_64-unknown-linux-musl
### 3. Execute
For Android we suggest taking a look at the [Chicory SDK](https://github.com/extism/chicory-sdk) for a pure Java
Extism runtime.
Load WebAssembly modules at any time in your app's lifetime and Extism will execute them in a secure sandbox, fully isolated from your program's memory.
# Run WebAssembly In Your App
# API Status
Pick a SDK to import into your program, and refer to the documentation to get
started:
**Please note:** This project still under active development and APIs are still changing. We are aiming for a stable 1.0 release in January, 2024.
The main branch may have breaking changes until that point, but if you starting today, a 1.0.0-rcx release is the best place to start.
| Type | Language | Source Code | Package |
| ----------- | ---------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------- | ----------------------------------------------------------------------- |
| Rust SDK | <img alt="Rust SDK" src="https://extism.org/img/sdk-languages/rust.svg" width="50px"/> | https://github.com/extism/extism/tree/main/runtime | [Crates.io](https://crates.io/crates/extism) |
| JS SDK | <img alt="JS SDK" src="https://extism.org/img/sdk-languages/js.svg" width="50px"/> | https://github.com/extism/js-sdk <br/>(supports Web, Node, Deno & Bun!) | [NPM](https://www.npmjs.com/package/@extism/extism) |
| Elixir SDK | <img alt="Elixir SDK" src="https://extism.org/img/sdk-languages/elixir.svg" width="50px"/> | https://github.com/extism/elixir-sdk | [Hex](https://hex.pm/packages/extism) |
| Go SDK | <img alt="Go SDK" src="https://extism.org/img/sdk-languages/go.svg" width="50px"/> | https://github.com/extism/go-sdk | [Go mod](https://pkg.go.dev/github.com/extism/go-sdk) |
| Haskell SDK | <img alt="Haskell SDK" src="https://extism.org/img/sdk-languages/haskell.svg" width="50px"/> | https://github.com/extism/haskell-sdk | [Hackage](https://hackage.haskell.org/package/extism) |
| Java SDK | <img alt="Java SDK" src="https://extism.org/img/sdk-languages/java-android.svg" width="50px"/> | https://github.com/extism/java-sdk | [Sonatype](https://central.sonatype.com/artifact/org.extism.sdk/extism) |
| .NET SDK | <img alt=".NET SDK" src="https://extism.org/img/sdk-languages/dotnet.svg" width="50px"/> | https://github.com/extism/dotnet-sdk <br/>(supports C# & F#!) | [Nuget](https://www.nuget.org/packages/Extism.Sdk) |
| OCaml SDK | <img alt="OCaml SDK" src="https://extism.org/img/sdk-languages/ocaml.svg" width="50px"/> | https://github.com/extism/ocaml-sdk | [opam](https://opam.ocaml.org/packages/extism/) |
| Perl SDK | <img alt="Perl SDK" src="https://extism.org/img/sdk-languages/perl.svg" width="50px"/> | https://github.com/extism/perl-sdk | [CPAN](https://metacpan.org/pod/Extism) |
| PHP SDK | <img alt="PHP SDK" src="https://extism.org/img/sdk-languages/php.svg" width="50px"/> | https://github.com/extism/php-sdk | [Packagist](https://packagist.org/packages/extism/extism) |
| Python SDK | <img alt="Python SDK" src="https://extism.org/img/sdk-languages/python.svg" width="50px"/> | https://github.com/extism/python-sdk | [PyPi](https://pypi.org/project/extism/) |
| Ruby SDK | <img alt="Ruby SDK" src="https://extism.org/img/sdk-languages/ruby.svg" width="50px"/> | https://github.com/extism/ruby-sdk | [RubyGems](https://rubygems.org/gems/extism) |
| Zig SDK | <img alt="Zig SDK" src="https://extism.org/img/sdk-languages/zig.svg" width="50px"/> | https://github.com/extism/zig-sdk | N/A |
| C SDK | <img alt="C SDK" src="https://extism.org/img/sdk-languages/c.svg" width="50px"/> | https://github.com/extism/extism/tree/main/libextism | N/A |
| C++ SDK | <img alt="C++ SDK" src="https://extism.org/img/sdk-languages/cpp.svg" width="50px"/> | https://github.com/extism/cpp-sdk | N/A |
# Compile WebAssembly to run in Extism Hosts
Extism Hosts (running the SDK) must execute WebAssembly code that has a
[PDK, or Plug-in Development Kit](https://extism.org/docs/concepts/pdk), library
compiled in to the `.wasm` binary. PDKs make it easy for plug-in / extension
code authors to read input from the host and return data back, read provided
configuration, set/get variables, make outbound HTTP calls if allowed, and more.
Pick a PDK to import into your Wasm program, and refer to the documentation to
get started:
| Type | Language | Source Code | Package |
| ------------------ | ---------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------- | --------------------------------------------------------- |
| Rust PDK | <img alt="Rust PDK" src="https://extism.org/img/sdk-languages/rust.svg" width="50px"/> | https://github.com/extism/rust-pdk | [Crates.io](https://crates.io/crates/extism-pdk) |
| JS PDK | <img alt="JS PDK" src="https://extism.org/img/sdk-languages/js.svg" width="50px"/> | https://github.com/extism/js-pdk | N/A |
| Python PDK | <img alt="Python PDK" src="https://extism.org/img/sdk-languages/python.svg" width="50px"/> | https://github.com/extism/python-pdk | N/A |
| Go PDK | <img alt="Go PDK" src="https://extism.org/img/sdk-languages/go.svg" width="50px"/> | https://github.com/extism/go-pdk | [Go mod](https://pkg.go.dev/github.com/extism/go-pdk) |
| Haskell PDK | <img alt="Haskell PDK" src="https://extism.org/img/sdk-languages/haskell.svg" width="50px"/> | https://github.com/extism/haskell-pdk | [Hackage](https://hackage.haskell.org/package/extism-pdk) |
| AssemblyScript PDK | <img alt="AssemblyScript PDK" src="https://extism.org/img/sdk-languages/assemblyscript.svg" width="50px"/> | https://github.com/extism/assemblyscript-pdk | [NPM](https://www.npmjs.com/package/@extism/as-pdk) |
| .NET PDK | <img alt=".NET PDK" src="https://extism.org/img/sdk-languages/dotnet.svg" width="50px"/> | https://github.com/extism/dotnet-pdk <br/>(supports C# & F#!) | [Nuget](https://www.nuget.org/packages/Extism.Pdk) |
| C PDK | <img alt="C PDK" src="https://extism.org/img/sdk-languages/c.svg" width="50px"/> | https://github.com/extism/c-pdk | N/A |
| C++ PDK | <img alt="C++ PDK" src="https://extism.org/img/sdk-languages/cpp.svg" width="50px"/> | https://github.com/extism/cpp-pdk | N/A |
| Zig PDK | <img alt="Zig PDK" src="https://extism.org/img/sdk-languages/zig.svg" width="50px"/> | https://github.com/extism/zig-pdk | N/A |
# Generating Bindings
It's often very useful to define a schema to describe the function signatures
and types you want to use between Extism SDK and PDK languages.
[XTP Bindgen](https://github.com/dylibso/xtp-bindgen) is an open source
framework to generate PDK bindings for Extism plug-ins. It's used by the
[XTP Platform](https://www.getxtp.com/), but can be used outside of the platform
to define any Extism compatible plug-in system.
## 1. Install the `xtp` CLI.
See installation instructions
[here](https://docs.xtp.dylibso.com/docs/cli#installation).
## 2. Create a schema using our OpenAPI-inspired IDL:
```yaml
version: v1-draft
exports:
CountVowels:
input:
type: string
contentType: text/plain; charset=utf-8
output:
$ref: "#/components/schemas/VowelReport"
contentType: application/json
# components.schemas defined in example-schema.yaml...
```
> See an example in [example-schema.yaml](./example-schema.yaml), or a full
> "kitchen sink" example on
> [the docs page](https://docs.xtp.dylibso.com/docs/concepts/xtp-schema/).
## 3. Generate bindings to use from your plugins:
```
xtp plugin init --schema-file ./example-schema.yaml
> 1. TypeScript
2. Go
3. Rust
4. Python
5. C#
6. Zig
7. C++
8. GitHub Template
9. Local Template
```
This will create an entire boilerplate plugin project for you to get started
with. Implement the empty function(s), and run `xtp plugin build` to compile
your plugin.
> For more information about XTP Bindgen, see the
> [dylibso/xtp-bindgen](https://github.com/dylibso/xtp-bindgen) repository and
> the official
> [XTP Schema documentation](https://docs.xtp.dylibso.com/docs/concepts/xtp-schema).
# Support
## Discord
If you experience any problems or have any questions, please join our
[Discord](https://extism.org/discord) and let us know. Our community is very
responsive and happy to help get you started.
If you experience any problems or have any questions, please join our [Discord](https://discord.gg/cx3usBCWnc) and let us know.
Our community is very responsive and happy to help get you started.
## Usage
Head to the [project website](https://extism.org) for more information and docs.
Also, consider reading an [overview](https://extism.org/docs/overview) of Extism
and its goals & approach.
Head to the [project website](https://extism.org) for more information and docs. Also, consider reading an [overview](https://extism.org/docs/overview) of Extism and its goals & approach.
## Contribution
Thank you for considering a contribution to Extism, we are happy to help you
make a PR or find something to work on!
Thank you for considering a contribution to Extism, we are happy to help you make a PR or find something to work on!
The easiest way to start would be to join the
[Discord](https://extism.org/discord) or open an issue on the
[`extism/proposals`](https://github.com/extism/proposals) issue tracker, which
can eventually become an Extism Improvement Proposal (EIP).
For more information, please read the
[Contributing](https://extism.org/docs/concepts/contributing) guide.
The easiest way to start would be to join the [Discord](https://discord.gg/cx3usBCWnc) or open an issue on the [`extism/proposals`](https://github.com/extism/proposals) issue tracker, which can eventually become an Extism Improvement Proposal (EIP).
---
@@ -191,8 +68,8 @@ For more information, please read the
Extism is an open-source product from the team at:
<p align="left">
<a href="https://dylibso.com" _target="blanks"><img width="200px" src="https://user-images.githubusercontent.com/7517515/198204119-5afdebb9-a5d8-4322-bd2a-46179c8d7b24.svg"/></a>
<a href="https://dylib.so" _target="blanks"><img width="200px" src="https://user-images.githubusercontent.com/7517515/198204119-5afdebb9-a5d8-4322-bd2a-46179c8d7b24.svg"/></a>
</p>
_Reach out and tell us what you're building! We'd love to help:_
<a href="mailto:hello@dylibso.com">hello@dylibso.com</a>
_Reach out and tell us what you're building! We'd love to help._

View File

@@ -1,26 +0,0 @@
[package]
name = "extism-convert-macros"
edition.workspace = true
authors.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
version.workspace = true
description = "Macros to remove boilerplate with Extism"
[lib]
proc-macro = true
[features]
extism-path = []
extism-pdk-path = []
[dependencies]
manyhow.version = "0.11.0"
proc-macro-crate = "3.1.0"
proc-macro2 = "1.0.78"
quote = "1.0.35"
syn = { version = "2.0.48", features = ["derive"] }
[dev-dependencies]
trybuild = "1.0.89"

View File

@@ -1,108 +0,0 @@
use std::iter;
use manyhow::{ensure, error_message, manyhow, Result};
use proc_macro_crate::{crate_name, FoundCrate};
use quote::{format_ident, quote, ToTokens};
use syn::{parse_quote, Attribute, DeriveInput, Path};
/// Tries to resolve the path to `extism_convert` dynamically, falling back to feature flags when unsuccessful.
fn convert_path() -> Path {
match (
crate_name("extism"),
crate_name("extism-convert"),
crate_name("extism-pdk"),
) {
(Ok(FoundCrate::Name(name)), ..) => {
let ident = format_ident!("{name}");
parse_quote!(::#ident::convert)
}
(_, Ok(FoundCrate::Name(name)), ..) | (.., Ok(FoundCrate::Name(name))) => {
let ident = format_ident!("{name}");
parse_quote!(::#ident)
}
(Ok(FoundCrate::Itself), ..) => parse_quote!(::extism::convert),
(_, Ok(FoundCrate::Itself), ..) => parse_quote!(::extism_convert),
(.., Ok(FoundCrate::Itself)) => parse_quote!(::extism_pdk),
_ if cfg!(feature = "extism-path") => parse_quote!(::extism::convert),
_ if cfg!(feature = "extism-pdk-path") => parse_quote!(::extism_pdk),
_ => parse_quote!(::extism_convert),
}
}
fn extract_encoding(attrs: &[Attribute]) -> Result<Path> {
let encodings: Vec<_> = attrs
.iter()
.filter(|attr| attr.path().is_ident("encoding"))
.collect();
ensure!(!encodings.is_empty(), "encoding needs to be specified"; try = "`#[encoding(Json)]`");
ensure!(encodings.len() < 2, encodings[1], "only one encoding can be specified"; try = "remove `{}`", encodings[1].to_token_stream());
Ok(encodings[0].parse_args().map_err(
|e| error_message!(e.span(), "{e}"; note= "expects a path"; try = "`#[encoding(Json)]`"),
)?)
}
#[manyhow]
#[proc_macro_derive(ToBytes, attributes(encoding))]
pub fn to_bytes(
DeriveInput {
attrs,
ident,
generics,
..
}: DeriveInput,
) -> Result {
let encoding = extract_encoding(&attrs)?;
let convert = convert_path();
let (_, type_generics, _) = generics.split_for_impl();
let mut generics = generics.clone();
generics.make_where_clause().predicates.push(
parse_quote!(for<'__to_bytes_b> #encoding<&'__to_bytes_b Self>: #convert::ToBytes<'__to_bytes_b>)
);
generics.params = iter::once(parse_quote!('__to_bytes_a))
.chain(generics.params)
.collect();
let (impl_generics, _, where_clause) = generics.split_for_impl();
Ok(quote! {
impl #impl_generics #convert::ToBytes<'__to_bytes_a> for #ident #type_generics #where_clause
{
type Bytes = ::std::vec::Vec<u8>;
fn to_bytes(&self) -> Result<Self::Bytes, #convert::Error> {
#convert::ToBytes::to_bytes(&#encoding(self)).map(|__bytes| __bytes.as_ref().to_vec())
}
}
})
}
#[manyhow]
#[proc_macro_derive(FromBytes, attributes(encoding))]
pub fn from_bytes(
DeriveInput {
attrs,
ident,
mut generics,
..
}: DeriveInput,
) -> Result {
let encoding = extract_encoding(&attrs)?;
let convert = convert_path();
generics
.make_where_clause()
.predicates
.push(parse_quote!(#encoding<Self>: #convert::FromBytesOwned));
let (impl_generics, type_generics, where_clause) = generics.split_for_impl();
Ok(quote! {
impl #impl_generics #convert::FromBytesOwned for #ident #type_generics #where_clause
{
fn from_bytes_owned(__data: &[u8]) -> Result<Self, #convert::Error> {
<#encoding<Self> as #convert::FromBytesOwned>::from_bytes_owned(__data).map(|__encoding| __encoding.0)
}
}
})
}

View File

@@ -1,5 +0,0 @@
#[test]
fn ui() {
let t = trybuild::TestCases::new();
t.compile_fail("tests/ui/*.rs");
}

View File

@@ -1,23 +0,0 @@
use extism_convert_macros::ToBytes;
#[derive(ToBytes)]
struct MissingEncoding;
#[derive(ToBytes)]
#[encoding]
struct EmptyAttr;
#[derive(ToBytes)]
#[encoding = "string"]
struct EqNoParen;
#[derive(ToBytes)]
#[encoding(something, else)]
struct NotAPath;
#[derive(ToBytes)]
#[encoding(Multiple)]
#[encoding(Encodings)]
struct MultipleEncodings;
fn main() {}

View File

@@ -1,44 +0,0 @@
error: encoding needs to be specified
= try: `#[encoding(Json)]`
--> tests/ui/invalid-encoding.rs:3:10
|
3 | #[derive(ToBytes)]
| ^^^^^^^
|
= note: this error originates in the derive macro `ToBytes` (in Nightly builds, run with -Z macro-backtrace for more info)
error: expected attribute arguments in parentheses: #[encoding(...)]
= note: expects a path
= try: `#[encoding(Json)]`
--> tests/ui/invalid-encoding.rs:7:3
|
7 | #[encoding]
| ^^^^^^^^
error: expected parentheses: #[encoding(...)]
= note: expects a path
= try: `#[encoding(Json)]`
--> tests/ui/invalid-encoding.rs:11:12
|
11 | #[encoding = "string"]
| ^
error: unexpected token
= note: expects a path
= try: `#[encoding(Json)]`
--> tests/ui/invalid-encoding.rs:15:21
|
15 | #[encoding(something, else)]
| ^
error: only one encoding can be specified
= try: remove `#[encoding(Encodings)]`
--> tests/ui/invalid-encoding.rs:20:1
|
20 | #[encoding(Encodings)]
| ^^^^^^^^^^^^^^^^^^^^^^

View File

@@ -11,21 +11,16 @@ description = "Traits to make Rust types usable with Extism"
[dependencies]
anyhow = "1.0.75"
base64 = "~0.22"
bytemuck = {version = "1.14.0", optional = true }
prost = { version = "0.13.1", optional = true }
protobuf = { version = "3.2.0", optional = true }
base64 = "~0.21"
prost = { version = "0.12.0", optional = true }
rmp-serde = { version = "1.1.2", optional = true }
serde = "1.0.186"
serde_json = "1.0.105"
extism-convert-macros.workspace = true
[dev-dependencies]
serde = { version = "1.0.186", features = ["derive"] }
[features]
default = ["msgpack", "prost", "raw"]
default = ["msgpack", "protobuf"]
msgpack = ["rmp-serde"]
raw = ["bytemuck"]
extism-path = ["extism-convert-macros/extism-path"]
extism-pdk-path = ["extism-convert-macros/extism-pdk-path"]
protobuf = ["prost"]

View File

@@ -11,8 +11,8 @@ use base64::Engine;
/// extism_convert::encoding!(MyJson, serde_json::to_vec, serde_json::from_slice);
/// ```
///
/// This will create a struct `struct MyJson<T>(pub T)` and implement [`ToBytes`] using [`serde_json::to_vec`]
/// and [`FromBytesOwned`] using [`serde_json::from_slice`]
/// This will create a struct `struct MyJson<T>(pub T)` and implement `ToBytes` using `serde_json::to_vec`
/// and `FromBytesOwned` using `serde_json::from_vec`
#[macro_export]
macro_rules! encoding {
($pub:vis $name:ident, $to_vec:expr, $from_slice:expr) => {
@@ -55,7 +55,7 @@ encoding!(pub Json, serde_json::to_vec, serde_json::from_slice);
#[cfg(feature = "msgpack")]
encoding!(pub Msgpack, rmp_serde::to_vec, rmp_serde::from_slice);
impl ToBytes<'_> for serde_json::Value {
impl<'a> ToBytes<'a> for serde_json::Value {
type Bytes = Vec<u8>;
fn to_bytes(&self) -> Result<Self::Bytes, Error> {
@@ -85,7 +85,7 @@ impl<T: AsRef<[u8]>> From<T> for Base64<T> {
}
}
impl<T: AsRef<[u8]>> ToBytes<'_> for Base64<T> {
impl<'a, T: AsRef<[u8]>> ToBytes<'a> for Base64<T> {
type Bytes = String;
fn to_bytes(&self) -> Result<Self::Bytes, Error> {
@@ -112,19 +112,19 @@ impl FromBytesOwned for Base64<String> {
/// Protobuf encoding
///
/// Allows for `prost` Protobuf messages to be used as arguments to Extism plugin calls
#[cfg(feature = "prost")]
#[cfg(feature = "protobuf")]
#[derive(Debug)]
pub struct Prost<T: prost::Message>(pub T);
pub struct Protobuf<T: prost::Message>(pub T);
#[cfg(feature = "prost")]
impl<T: prost::Message> From<T> for Prost<T> {
#[cfg(feature = "protobuf")]
impl<T: prost::Message> From<T> for Protobuf<T> {
fn from(data: T) -> Self {
Self(data)
}
}
#[cfg(feature = "prost")]
impl<T: prost::Message> ToBytes<'_> for Prost<T> {
#[cfg(feature = "protobuf")]
impl<'a, T: prost::Message> ToBytes<'a> for Protobuf<T> {
type Bytes = Vec<u8>;
fn to_bytes(&self) -> Result<Self::Bytes, Error> {
@@ -132,56 +132,9 @@ impl<T: prost::Message> ToBytes<'_> for Prost<T> {
}
}
#[cfg(feature = "prost")]
impl<T: Default + prost::Message> FromBytesOwned for Prost<T> {
#[cfg(feature = "protobuf")]
impl<T: Default + prost::Message> FromBytesOwned for Protobuf<T> {
fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
Ok(Prost(T::decode(data)?))
Ok(Protobuf(T::decode(data)?))
}
}
/// Protobuf encoding
///
/// Allows for `rust-protobuf` Protobuf messages to be used as arguments to Extism plugin calls
#[cfg(feature = "protobuf")]
pub struct Protobuf<T: protobuf::Message>(pub T);
#[cfg(feature = "protobuf")]
impl<T: protobuf::Message> ToBytes<'_> for Protobuf<T> {
type Bytes = Vec<u8>;
fn to_bytes(&self) -> Result<Self::Bytes, Error> {
Ok(self.0.write_to_bytes()?)
}
}
#[cfg(feature = "protobuf")]
impl<T: Default + protobuf::Message> FromBytesOwned for Protobuf<T> {
fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
Ok(Protobuf(T::parse_from_bytes(data)?))
}
}
/// Raw does no conversion, it just copies the memory directly.
/// Note: This will only work for types that implement [bytemuck::Pod](https://docs.rs/bytemuck/latest/bytemuck/trait.Pod.html)
#[cfg(all(feature = "raw", target_endian = "little"))]
pub struct Raw<'a, T: bytemuck::Pod>(pub &'a T);
#[cfg(all(feature = "raw", target_endian = "little"))]
impl<'a, T: bytemuck::Pod> ToBytes<'a> for Raw<'a, T> {
type Bytes = &'a [u8];
fn to_bytes(&self) -> Result<Self::Bytes, Error> {
Ok(bytemuck::bytes_of(self.0))
}
}
#[cfg(all(feature = "raw", target_endian = "little"))]
impl<'a, T: bytemuck::Pod> FromBytes<'a> for Raw<'a, T> {
fn from_bytes(data: &'a [u8]) -> Result<Self, Error> {
let x = bytemuck::try_from_bytes(data).map_err(|x| Error::msg(x.to_string()))?;
Ok(Raw(x))
}
}
#[cfg(all(feature = "raw", target_endian = "big"))]
compile_error!("The raw feature is only supported on little endian targets");

View File

@@ -1,68 +1,14 @@
use crate::*;
pub use extism_convert_macros::FromBytes;
/// `FromBytes` is used to define how a type should be decoded when working with
/// Extism memory. It is used for plugin output and host function input.
///
/// `FromBytes` can be derived by delegating encoding to generic type implementing
/// `FromBytes`, e.g., [`Json`], [`Msgpack`].
///
/// ```
/// use extism_convert::{Json, FromBytes};
/// use serde::Deserialize;
///
/// #[derive(FromBytes, Deserialize, PartialEq, Debug)]
/// #[encoding(Json)]
/// struct Struct {
/// hello: String,
/// }
///
/// assert_eq!(Struct::from_bytes(br#"{"hello":"hi"}"#)?, Struct { hello: "hi".into() });
/// # Ok::<(), extism_convert::Error>(())
/// ```
///
/// Custom encodings can also be used, through new-types with a single generic
/// argument, i.e., `Type<T>(T)`, that implement `FromBytesOwned` for the struct.
///
/// ```
/// use std::str::{self, FromStr};
/// use std::convert::Infallible;
/// use extism_convert::{Error, FromBytes, FromBytesOwned};
///
/// // Custom serialization using `FromStr`
/// struct StringEnc<T>(T);
/// impl<T: FromStr> FromBytesOwned for StringEnc<T> where Error: From<<T as FromStr>::Err> {
/// fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
/// Ok(Self(str::from_utf8(data)?.parse()?))
/// }
/// }
///
/// #[derive(FromBytes, PartialEq, Debug)]
/// #[encoding(StringEnc)]
/// struct Struct {
/// hello: String,
/// }
///
/// impl FromStr for Struct {
/// type Err = Infallible;
/// fn from_str(s: &str) -> Result<Self, Infallible> {
/// Ok(Self { hello: s.to_owned() })
/// }
/// }
///
/// assert_eq!(Struct::from_bytes(b"hi")?, Struct { hello: "hi".into() });
/// # Ok::<(), extism_convert::Error>(())
/// ```
pub trait FromBytes<'a>: Sized {
/// Decode a value from a slice of bytes
fn from_bytes(data: &'a [u8]) -> Result<Self, Error>;
}
/// `FromBytesOwned` is similar to [`FromBytes`] but it doesn't borrow from the input slice.
/// [`FromBytes`] is automatically implemented for all types that implement `FromBytesOwned`.
///
/// `FromBytesOwned` can be derived through [`#[derive(FromBytes)]`](FromBytes).
/// `FromBytesOwned` is similar to `FromBytes` but it doesn't borrow from the input slice.
/// `FromBytes` is automatically implemented for all types that implement `FromBytesOwned`
pub trait FromBytesOwned: Sized {
/// Decode a value from a slice of bytes, the resulting value should not borrow the input
/// data.
@@ -141,16 +87,6 @@ impl FromBytesOwned for u32 {
}
}
impl FromBytesOwned for bool {
fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
if let Some(x) = data.first() {
Ok(*x != 0)
} else {
Err(Error::msg("Expected one byte to read boolean value"))
}
}
}
impl FromBytesOwned for () {
fn from_bytes_owned(_: &[u8]) -> Result<Self, Error> {
Ok(())
@@ -162,13 +98,3 @@ impl<'a, T: FromBytes<'a>> FromBytes<'a> for std::io::Cursor<T> {
Ok(std::io::Cursor::new(T::from_bytes(data)?))
}
}
impl<'a, T: FromBytes<'a>> FromBytes<'a> for Option<T> {
fn from_bytes(data: &'a [u8]) -> Result<Self, Error> {
if data.is_empty() {
return Ok(None);
}
T::from_bytes(data).map(Some)
}
}

View File

@@ -5,9 +5,6 @@
//! similar to [axum extractors](https://docs.rs/axum/latest/axum/extract/index.html#intro) - they are
//! implemented as a tuple struct with a single field that is meant to be extracted using pattern matching.
// Makes proc-macros able to resolve `::extism_convert` correctly
extern crate self as extism_convert;
pub use anyhow::Error;
mod encoding;
@@ -21,15 +18,9 @@ pub use encoding::{Base64, Json};
#[cfg(feature = "msgpack")]
pub use encoding::Msgpack;
#[cfg(feature = "prost")]
pub use encoding::Prost;
#[cfg(feature = "protobuf")]
pub use encoding::Protobuf;
#[cfg(all(feature = "raw", target_endian = "little"))]
pub use encoding::Raw;
pub use from_bytes::{FromBytes, FromBytesOwned};
pub use memory_handle::MemoryHandle;
pub use to_bytes::ToBytes;

View File

@@ -20,7 +20,6 @@ fn roundtrip_json() {
}
#[test]
#[cfg(feature = "msgpack")]
fn roundtrip_msgpack() {
let x = Testing {
a: "foobar".to_string(),
@@ -38,64 +37,3 @@ fn roundtrip_base64() {
let Base64(s): Base64<String> = FromBytes::from_bytes(bytes.as_bytes()).unwrap();
assert_eq!(s, "this is a test");
}
#[test]
fn rountrip_option() {
// `None` case
let e0: Option<Json<Testing>> = FromBytes::from_bytes(&[]).unwrap();
let b = e0.to_bytes().unwrap();
let e1: Option<Json<Testing>> = FromBytes::from_bytes(&b).unwrap();
assert!(e0.is_none());
assert_eq!(e0.is_none(), e1.is_none());
// `Some` case
let x = Testing {
a: "foobar".to_string(),
b: 123,
c: 456.7,
};
let bytes = Json(&x).to_bytes().unwrap();
let y: Option<Json<Testing>> = FromBytes::from_bytes(&bytes).unwrap();
let b = ToBytes::to_bytes(&y).unwrap();
let z: Option<Json<Testing>> = FromBytes::from_bytes(&b).unwrap();
assert_eq!(y.unwrap().0, z.unwrap().0);
}
#[test]
fn check_bool() {
// `None` case
let a = true.to_bytes().unwrap();
let b = false.to_bytes().unwrap();
assert_ne!(a, b);
assert_eq!(a, [1]);
assert_eq!(b, [0]);
}
#[cfg(all(feature = "raw", target_endian = "little"))]
mod raw_tests {
use crate::*;
#[test]
fn test_raw() {
#[derive(Debug, Clone, Copy, PartialEq)]
struct TestRaw {
a: i32,
b: f64,
c: bool,
}
unsafe impl bytemuck::Pod for TestRaw {}
unsafe impl bytemuck::Zeroable for TestRaw {}
let x = TestRaw {
a: 123,
b: 45678.91011,
c: true,
};
let raw = Raw(&x).to_bytes().unwrap();
let y = Raw::from_bytes(raw).unwrap();
assert_eq!(&x, y.0);
let y: Result<Raw<[u8; std::mem::size_of::<TestRaw>()]>, Error> = Raw::from_bytes(raw);
assert!(y.is_ok());
}
}

View File

@@ -1,58 +1,7 @@
use crate::*;
pub use extism_convert_macros::ToBytes;
/// `ToBytes` is used to define how a type should be encoded when working with
/// Extism memory. It is used for plugin input and host function output.
///
/// `ToBytes` can be derived by delegating encoding to generic type implementing
/// `ToBytes`, e.g., [`Json`], [`Msgpack`].
///
/// ```
/// use extism_convert::{Json, ToBytes};
/// use serde::Serialize;
///
/// #[derive(ToBytes, Serialize)]
/// #[encoding(Json)]
/// struct Struct {
/// hello: String,
/// }
///
/// assert_eq!(Struct { hello: "hi".into() }.to_bytes()?, br#"{"hello":"hi"}"#);
/// # Ok::<(), extism_convert::Error>(())
/// ```
///
/// But custom types can also be used, as long as they are new-types with a single
/// generic argument, i.e., `Type<T>(T)`, that implement `ToBytes` for the struct.
///
/// ```
/// use extism_convert::{Error, ToBytes};
///
/// // Custom serialization using `ToString`
/// struct StringEnc<T>(T);
/// impl<T: ToString> ToBytes<'_> for StringEnc<&T> {
/// type Bytes = String;
///
/// fn to_bytes(&self) -> Result<String, Error> {
/// Ok(self.0.to_string())
/// }
/// }
///
/// #[derive(ToBytes)]
/// #[encoding(StringEnc)]
/// struct Struct {
/// hello: String,
/// }
///
/// impl ToString for Struct {
/// fn to_string(&self) -> String {
/// self.hello.clone()
/// }
/// }
///
/// assert_eq!(Struct { hello: "hi".into() }.to_bytes()?, b"hi");
/// # Ok::<(), Error>(())
/// ```
pub trait ToBytes<'a> {
/// A configurable byte slice representation, allows any type that implements `AsRef<[u8]>`
type Bytes: AsRef<[u8]>;
@@ -61,21 +10,21 @@ pub trait ToBytes<'a> {
fn to_bytes(&self) -> Result<Self::Bytes, Error>;
}
impl ToBytes<'_> for () {
impl<'a> ToBytes<'a> for () {
type Bytes = [u8; 0];
fn to_bytes(&self) -> Result<Self::Bytes, Error> {
Ok([])
}
}
impl ToBytes<'_> for Vec<u8> {
impl<'a> ToBytes<'a> for Vec<u8> {
type Bytes = Vec<u8>;
fn to_bytes(&self) -> Result<Self::Bytes, Error> {
Ok(self.clone())
}
}
impl ToBytes<'_> for String {
impl<'a> ToBytes<'a> for String {
type Bytes = String;
fn to_bytes(&self) -> Result<Self::Bytes, Error> {
Ok(self.clone())
@@ -96,7 +45,7 @@ impl<'a> ToBytes<'a> for &'a str {
}
}
impl ToBytes<'_> for f64 {
impl<'a> ToBytes<'a> for f64 {
type Bytes = [u8; 8];
fn to_bytes(&self) -> Result<Self::Bytes, Error> {
@@ -104,7 +53,7 @@ impl ToBytes<'_> for f64 {
}
}
impl ToBytes<'_> for f32 {
impl<'a> ToBytes<'a> for f32 {
type Bytes = [u8; 4];
fn to_bytes(&self) -> Result<Self::Bytes, Error> {
@@ -112,7 +61,7 @@ impl ToBytes<'_> for f32 {
}
}
impl ToBytes<'_> for i64 {
impl<'a> ToBytes<'a> for i64 {
type Bytes = [u8; 8];
fn to_bytes(&self) -> Result<Self::Bytes, Error> {
@@ -120,7 +69,7 @@ impl ToBytes<'_> for i64 {
}
}
impl ToBytes<'_> for i32 {
impl<'a> ToBytes<'a> for i32 {
type Bytes = [u8; 4];
fn to_bytes(&self) -> Result<Self::Bytes, Error> {
@@ -128,7 +77,7 @@ impl ToBytes<'_> for i32 {
}
}
impl ToBytes<'_> for u64 {
impl<'a> ToBytes<'a> for u64 {
type Bytes = [u8; 8];
fn to_bytes(&self) -> Result<Self::Bytes, Error> {
@@ -136,7 +85,7 @@ impl ToBytes<'_> for u64 {
}
}
impl ToBytes<'_> for u32 {
impl<'a> ToBytes<'a> for u32 {
type Bytes = [u8; 4];
fn to_bytes(&self) -> Result<Self::Bytes, Error> {
@@ -144,14 +93,6 @@ impl ToBytes<'_> for u32 {
}
}
impl ToBytes<'_> for bool {
type Bytes = [u8; 1];
fn to_bytes(&self) -> Result<Self::Bytes, Error> {
Ok([*self as u8])
}
}
impl<'a, T: ToBytes<'a>> ToBytes<'a> for &'a T {
type Bytes = T::Bytes;
@@ -159,26 +100,3 @@ impl<'a, T: ToBytes<'a>> ToBytes<'a> for &'a T {
<T as ToBytes>::to_bytes(self)
}
}
impl<'a, T: ToBytes<'a>> ToBytes<'a> for Option<T> {
type Bytes = Vec<u8>;
fn to_bytes(&self) -> Result<Self::Bytes, Error> {
match self {
Some(x) => x.to_bytes().map(|x| x.as_ref().to_vec()),
None => Ok(vec![]),
}
}
}
#[test]
fn test() {
use extism_convert::{Json, ToBytes};
use serde::Serialize;
#[derive(ToBytes, Serialize)]
#[encoding(Json)]
struct Struct {
hello: String,
}
}

View File

@@ -1,28 +0,0 @@
# yaml-language-server: $schema=https://xtp.dylibso.com/assets/wasm/schema.json
# Learn more at https://docs.xtp.dylibso.com/docs/concepts/xtp-schema
version: v1-draft
exports:
CountVowels:
input:
type: string
contentType: text/plain; charset=utf-8
output:
$ref: "#/components/schemas/VowelReport"
contentType: application/json
components:
schemas:
VowelReport:
description: The result of counting vowels on the Vowels input.
properties:
count:
type: integer
format: int32
description: The count of vowels for input string.
total:
type: integer
format: int32
description: The cumulative amount of vowels counted, if this keeps state across multiple function calls.
nullable: true
vowels:
type: string
description: The set of vowels used to get the count, e.g. "aAeEiIoOuU"

94
flake.lock generated Normal file
View File

@@ -0,0 +1,94 @@
{
"nodes": {
"flake-utils": {
"inputs": {
"systems": "systems"
},
"locked": {
"lastModified": 1701680307,
"narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "4022d587cbbfd70fe950c1e2083a02621806a725",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"naersk": {
"inputs": {
"nixpkgs": "nixpkgs"
},
"locked": {
"lastModified": 1698420672,
"narHash": "sha256-/TdeHMPRjjdJub7p7+w55vyABrsJlt5QkznPYy55vKA=",
"owner": "nix-community",
"repo": "naersk",
"rev": "aeb58d5e8faead8980a807c840232697982d47b9",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "naersk",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1701693815,
"narHash": "sha256-7BkrXykVWfkn6+c1EhFA3ko4MLi3gVG0p9G96PNnKTM=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "09ec6a0881e1a36c29d67497693a67a16f4da573",
"type": "github"
},
"original": {
"id": "nixpkgs",
"type": "indirect"
}
},
"nixpkgs_2": {
"locked": {
"lastModified": 1701693815,
"narHash": "sha256-7BkrXykVWfkn6+c1EhFA3ko4MLi3gVG0p9G96PNnKTM=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "09ec6a0881e1a36c29d67497693a67a16f4da573",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixpkgs-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"flake-utils": "flake-utils",
"naersk": "naersk",
"nixpkgs": "nixpkgs_2"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
}
},
"root": "root",
"version": 7
}

32
flake.nix Normal file
View File

@@ -0,0 +1,32 @@
{
inputs = {
flake-utils.url = "github:numtide/flake-utils";
naersk.url = "github:nix-community/naersk";
nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
};
outputs = { self, flake-utils, naersk, nixpkgs }:
flake-utils.lib.eachDefaultSystem (system:
let
pkgs = (import nixpkgs) {
inherit system;
};
naersk' = pkgs.callPackage naersk {};
in {
# For `nix build` & `nix run`:
defaultPackage = naersk'.buildPackage {
name = "libextism";
src = ./.;
copyLibs = true;
postInstall = "mkdir -p $out/include; cp runtime/extism.h $out/include";
};
# For `nix develop`:
devShell = pkgs.mkShell {
nativeBuildInputs = with pkgs; [ rustc cargo clippy gnumake ];
};
}
);
}

7
kernel/Cargo.lock generated Normal file
View File

@@ -0,0 +1,7 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 3
[[package]]
name = "extism-runtime-kernel"
version = "0.1.0"

View File

@@ -5,9 +5,6 @@ edition = "2021"
[dependencies]
[dev-dependencies]
wasm-bindgen-test = "0.3.39"
[features]
default = ["bounds-checking"]
bounds-checking = []

View File

@@ -17,9 +17,6 @@ done
cargo build --package extism-runtime-kernel --bin extism-runtime --release --target wasm32-unknown-unknown $CARGO_FLAGS
cp target/wasm32-unknown-unknown/release/extism-runtime.wasm .
wasm-strip extism-runtime.wasm
mv extism-runtime.wasm ../runtime/src/extism-runtime.wasm
wasm-tools parse extism-context.wat -o extism-context.wasm
wasm-merge --enable-reference-types ./extism-runtime.wasm runtime extism-context.wasm context -o ../runtime/src/extism-runtime.wasm
rm extism-context.wasm
rm extism-runtime.wasm
wasm-strip ../runtime/src/extism-runtime.wasm

View File

@@ -1,3 +0,0 @@
(module
(global (export "extism_context") (mut externref) (ref.null extern))
)

View File

@@ -3,7 +3,7 @@
pub use extism_runtime_kernel::*;
#[cfg(all(target_arch = "wasm32", not(test)))]
#[cfg(target_arch = "wasm32")]
#[panic_handler]
fn panic(_info: &core::panic::PanicInfo) -> ! {
core::arch::wasm32::unreachable()

View File

@@ -39,7 +39,7 @@
use core::sync::atomic::*;
pub type Pointer = u64;
pub type Handle = u64;
pub type Length = u64;
/// WebAssembly page size
const PAGE_SIZE: usize = 65536;
@@ -81,13 +81,13 @@ pub struct MemoryRoot {
/// Offset of error block
pub error: AtomicU64,
/// Input position in memory
pub input_offset: Handle,
pub input_offset: Pointer,
/// Input length
pub input_length: u64,
pub input_length: Length,
/// Output position in memory
pub output_offset: Pointer,
/// Output length
pub output_length: u64,
pub output_length: Length,
/// A pointer to the start of the first block
pub blocks: [MemoryBlock; 0],
}
@@ -139,9 +139,7 @@ impl MemoryRoot {
}
// Ensure that at least one page is allocated to store the `MemoryRoot` data
if core::arch::wasm32::memory_size(0) == 0
&& core::arch::wasm32::memory_grow(0, 1) == usize::MAX
{
if core::arch::wasm32::memory_size(0) == 0 && core::arch::wasm32::memory_grow(0, 1) == usize::MAX {
core::arch::wasm32::unreachable()
}
@@ -170,15 +168,12 @@ impl MemoryRoot {
/// Resets the position of the allocator and zeroes out all allocations
pub unsafe fn reset(&mut self) {
// Clear allocated data
let self_position = self.position.fetch_and(0, Ordering::SeqCst);
core::ptr::write_bytes(
self.blocks.as_mut_ptr() as *mut u8,
MemoryStatus::Unused as u8,
self_position as usize,
0,
self.length.load(Ordering::Acquire) as usize,
);
// Clear extism runtime metadata
self.position.store(0, Ordering::Release);
self.error.store(0, Ordering::Release);
self.input_offset = 0;
self.input_length = 0;
@@ -198,15 +193,15 @@ impl MemoryRoot {
fn pointer_in_bounds_fast(p: Pointer) -> bool {
// Similar to `pointer_in_bounds` but less accurate on the upper bound. This uses the total memory size,
// instead of checking `MemoryRoot::length`
let end = (core::arch::wasm32::memory_size(0) as u64) << 16;
p >= core::mem::size_of::<Self>() as Pointer && p <= end as u64
let end = core::arch::wasm32::memory_size(0) << 16;
p >= core::mem::size_of::<Self>() as Pointer && p <= end as Pointer
}
// Find a block that is free to use, this can be a new block or an existing freed block. The `self_position` argument
// is used to avoid loading the allocators position more than once when performing an allocation.
unsafe fn find_free_block(
&mut self,
length: u64,
length: Length,
self_position: u64,
) -> Option<&'static mut MemoryBlock> {
// Get the first block
@@ -228,7 +223,7 @@ impl MemoryRoot {
if status == MemoryStatus::Free as u8 && b.size >= length as usize {
// Split block if there is too much excess
if b.size - length as usize >= 128 {
b.size -= length as usize + core::mem::size_of::<MemoryBlock>();
b.size -= length as usize;
b.used = 0;
let block1 = b.data.as_mut_ptr().add(b.size) as *mut MemoryBlock;
@@ -252,37 +247,30 @@ impl MemoryRoot {
/// Create a new `MemoryBlock`, when `Some(block)` is returned, `block` will contain at least enough room for `length` bytes
/// but may be as large as `length` + `BLOCK_SPLIT_SIZE` bytes. When `None` is returned the allocation has failed.
pub unsafe fn alloc(&mut self, length: u64) -> Option<&'static mut MemoryBlock> {
pub unsafe fn alloc(&mut self, length: Length) -> Option<&'static mut MemoryBlock> {
let self_position = self.position.load(Ordering::Acquire);
let self_length = self.length.load(Ordering::Acquire);
let b = self.find_free_block(length, self_position);
// If there's a free block then re-use it
if let Some(b) = b {
b.used = length as usize;
b.status
.store(MemoryStatus::Active as u8, Ordering::Release);
return Some(b);
}
// Get the current index for a new block
let curr = self.blocks.as_ptr() as u64 + self_position;
// Get the number of bytes available
let mem_left = self_length - self_position - core::mem::size_of::<MemoryRoot>() as u64;
let length_with_block = length + core::mem::size_of::<MemoryBlock>() as u64;
// If the current position is large enough to hold the length of the block being
// allocated then check for existing free blocks that can be re-used before
// growing memory
if length_with_block <= self_position {
let b = self.find_free_block(length, self_position);
// If there's a free block then re-use it
if let Some(b) = b {
b.used = length as usize;
b.status
.store(MemoryStatus::Active as u8, Ordering::Release);
return Some(b);
}
}
// When the allocation is larger than the number of bytes available
// we will need to try to grow the memory
if length_with_block >= mem_left {
if length >= mem_left {
// Calculate the number of pages needed to cover the remaining bytes
let npages = num_pages(length_with_block - mem_left);
let npages = num_pages(length - mem_left);
let x = core::arch::wasm32::memory_grow(0, npages);
if x == usize::MAX {
return None;
@@ -313,26 +301,9 @@ impl MemoryRoot {
if !Self::pointer_in_bounds_fast(offs) {
return None;
}
// Get the first block
let mut block = self.blocks.as_mut_ptr();
// Only loop while the block pointer is less then the current position
while (block as u64) < self.blocks.as_ptr() as u64 + offs {
let b = &mut *block;
// Get the block status, this lets us know if we are able to re-use it
let status = b.status.load(Ordering::Acquire);
if status == MemoryStatus::Active as u8 && b.data.as_ptr() as Pointer == offs {
return Some(b);
}
// Get the next block
block = b.next_ptr();
}
None
let ptr = offs - core::mem::size_of::<MemoryBlock>() as u64;
let ptr = ptr as *mut MemoryBlock;
Some(&mut *ptr)
}
}
@@ -357,21 +328,21 @@ impl MemoryBlock {
/// Allocate a block of memory and return the offset
#[no_mangle]
pub unsafe fn alloc(n: u64) -> Handle {
pub unsafe fn alloc(n: Length) -> Pointer {
if n == 0 {
return 0;
}
let region = MemoryRoot::new();
let block = region.alloc(n);
match block {
Some(block) => block.data.as_mut_ptr() as Handle,
Some(block) => block.data.as_mut_ptr() as Pointer,
None => 0,
}
}
/// Free allocated memory
#[no_mangle]
pub unsafe fn free(p: Handle) {
pub unsafe fn free(p: Pointer) {
if p == 0 {
return;
}
@@ -389,42 +360,13 @@ pub unsafe fn free(p: Handle) {
}
/// Get the length of an allocated memory block
///
/// Note: this should only be called on memory handles returned
/// by a call to `alloc` - it will return garbage on invalid offsets
#[no_mangle]
pub unsafe fn length_unsafe(p: Handle) -> u64 {
pub unsafe fn length(p: Pointer) -> Length {
if p == 0 {
return 0;
}
if !MemoryRoot::pointer_in_bounds_fast(p) {
return 0;
}
let ptr = p - core::mem::size_of::<MemoryBlock>() as u64;
let block = &mut *(ptr as *mut MemoryBlock);
// Simplest sanity check to verify the pointer is a block
if block.status.load(Ordering::Acquire) != MemoryStatus::Active as u8 {
return 0;
}
block.used as u64
}
/// Get the length but returns 0 if the offset is not a valid handle.
///
/// Note: this function walks each node in the allocations list, which ensures correctness, but is also
/// slow
#[no_mangle]
pub unsafe fn length(p: Pointer) -> u64 {
if p == 0 {
return 0;
}
if let Some(block) = MemoryRoot::new().find_block(p) {
block.used as u64
block.used as Length
} else {
0
}
@@ -452,24 +394,24 @@ pub unsafe fn load_u64(p: Pointer) -> u64 {
/// Load a byte from the input data
#[no_mangle]
pub unsafe fn input_load_u8(offset: u64) -> u8 {
pub unsafe fn input_load_u8(p: Pointer) -> u8 {
let root = MemoryRoot::new();
#[cfg(feature = "bounds-checking")]
if offset >= root.input_length {
if p >= root.input_length {
return 0;
}
*((root.input_offset + offset) as *mut u8)
*((root.input_offset + p) as *mut u8)
}
/// Load a u64 from the input data
#[no_mangle]
pub unsafe fn input_load_u64(offset: u64) -> u64 {
pub unsafe fn input_load_u64(p: Pointer) -> u64 {
let root = MemoryRoot::new();
#[cfg(feature = "bounds-checking")]
if offset + core::mem::size_of::<u64>() as u64 > root.input_length {
if p + core::mem::size_of::<u64>() as Pointer > root.input_length {
return 0;
}
*((root.input_offset + offset) as *mut u64)
*((root.input_offset + p) as *mut u64)
}
/// Write a byte in Extism-managed memory
@@ -493,28 +435,22 @@ pub unsafe fn store_u64(p: Pointer, x: u64) {
}
/// Set the range of the input data in memory
/// h must always be a handle so that length works on it
/// len must match length(handle)
/// **Note**: this function takes ownership of the handle passed in
/// the caller should not `free` this value
#[no_mangle]
pub unsafe fn input_set(h: Handle, len: u64) {
pub unsafe fn input_set(p: Pointer, len: Length) {
let root = MemoryRoot::new();
#[cfg(feature = "bounds-checking")]
{
if !root.pointer_in_bounds(h) || !root.pointer_in_bounds(h + len - 1) {
if !root.pointer_in_bounds(p) || !root.pointer_in_bounds(p + len - 1) {
return;
}
}
root.input_offset = h;
root.input_offset = p;
root.input_length = len;
}
/// Set the range of the output data in memory
/// **Note**: this function takes ownership of the handle passed in
/// the caller should not `free` this value
#[no_mangle]
pub unsafe fn output_set(p: Pointer, len: u64) {
pub unsafe fn output_set(p: Pointer, len: Length) {
let root = MemoryRoot::new();
#[cfg(feature = "bounds-checking")]
{
@@ -528,25 +464,25 @@ pub unsafe fn output_set(p: Pointer, len: u64) {
/// Get the input length
#[no_mangle]
pub fn input_length() -> u64 {
pub fn input_length() -> Length {
unsafe { MemoryRoot::new().input_length }
}
/// Get the input offset in Exitsm-managed memory
#[no_mangle]
pub fn input_offset() -> Handle {
pub fn input_offset() -> Length {
unsafe { MemoryRoot::new().input_offset }
}
/// Get the output length
#[no_mangle]
pub fn output_length() -> u64 {
pub fn output_length() -> Length {
unsafe { MemoryRoot::new().output_length }
}
/// Get the output offset in Extism-managed memory
#[no_mangle]
pub unsafe fn output_offset() -> Pointer {
pub unsafe fn output_offset() -> Length {
MemoryRoot::new().output_offset
}
@@ -556,60 +492,32 @@ pub unsafe fn reset() {
MemoryRoot::new().reset()
}
/// Set the error message offset, the handle passed to this
/// function should not be freed after this call
/// **Note**: this function takes ownership of the handle passed in
/// the caller should not `free` this value
/// Set the error message offset
#[no_mangle]
pub unsafe fn error_set(h: Handle) {
pub unsafe fn error_set(ptr: Pointer) {
let root = MemoryRoot::new();
// Allow ERROR to be set to 0
if h == 0 {
root.error.store(h, Ordering::SeqCst);
if ptr == 0 {
root.error.store(ptr, Ordering::SeqCst);
return;
}
#[cfg(feature = "bounds-checking")]
if !root.pointer_in_bounds(h) {
if !root.pointer_in_bounds(ptr) {
return;
}
root.error.store(h, Ordering::SeqCst);
root.error.store(ptr, Ordering::SeqCst);
}
/// Get the error message offset, if it's `0` then no error has been set
#[no_mangle]
pub unsafe fn error_get() -> Handle {
pub unsafe fn error_get() -> Pointer {
MemoryRoot::new().error.load(Ordering::SeqCst)
}
/// Get the position of the allocator, this can be used as an indication of how many bytes are currently in-use
#[no_mangle]
pub unsafe fn memory_bytes() -> u64 {
pub unsafe fn memory_bytes() -> Length {
MemoryRoot::new().length.load(Ordering::Acquire)
}
#[cfg(test)]
mod test {
use crate::*;
use wasm_bindgen_test::*;
#[wasm_bindgen_test]
fn test_oom() {
let size = 1024 * 1024 * 5;
let mut last = 0;
for _ in 0..1024 {
unsafe {
let ptr = alloc(size);
last = ptr;
if ptr == 0 {
break;
}
assert_eq!(length(ptr), size);
}
}
assert_eq!(last, 0);
}
}

View File

@@ -1,5 +0,0 @@
# install wasm-bindgen-cli to get wasm-bindgen-runner if it is not installed yet
which wasm-bindgen-test-runner 1>/dev/null || cargo install -f wasm-bindgen-cli
# run tests with the wasm-bindgen-runner
CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER=wasm-bindgen-test-runner cargo test --release --target=wasm32-unknown-unknown

View File

@@ -1,2 +0,0 @@
BasedOnStyle: LLVM
IndentWidth: 2

View File

@@ -14,9 +14,7 @@ Create a new plugin.
- `functions`: is an array of `ExtismFunction*`
- `n_functions`: is the number of functions
- `with_wasi`: enables/disables WASI
- `errmsg`: error message during plugin creation, this should be freed with
`extism_plugin_new_error_free`
- `errmsg`: error message during plugin creation
```c
ExtismPlugin extism_plugin_new(const uint8_t *wasm,
@@ -26,17 +24,6 @@ ExtismPlugin extism_plugin_new(const uint8_t *wasm,
bool with_wasi,
char **errmsg);
```
---
### `extism_plugin_new_error_free`
Frees the error message returned when creating a plugin
```c
void extism_plugin_new_error_free(char *err);
```
---
### `extism_plugin_free`
@@ -46,8 +33,6 @@ Remove a plugin from the registry and free associated memory.
void extism_plugin_free(ExtismPlugin *plugin);
```
---
### `extism_plugin_config`
Update plugin config values, this will merge with the existing values.
@@ -58,8 +43,6 @@ bool extism_plugin_config(ExtismPlugin *plugin,
ExtismSize json_size);
```
---
### `extism_plugin_function_exists`
Returns true if `func_name` exists.
@@ -69,8 +52,6 @@ bool extism_plugin_function_exists(ExtismPlugin *plugin,
const char *func_name);
```
---
### `extism_plugin_call`
Call a function.
@@ -78,8 +59,6 @@ Call a function.
- `data`: is the input data
- `data_len`: is the length of `data`
Returns `0` when the call is successful.
```c
int32_t extism_plugin_call(ExtismPlugin *plugin,
const char *func_name,
@@ -87,28 +66,6 @@ int32_t extism_plugin_call(ExtismPlugin *plugin,
ExtismSize data_len);
```
---
### `extism_plugin_call_with_host_context`
Call a function with additional host context that can be accessed from inside host functions.
- `func_name`: is the function to call
- `data`: is the input data
- `data_len`: is the length of `data`
- `host_ctx`: an opaque pointer that can be accessed in host functions
Returns `0` when the call is successful.
```c
int32_t extism_plugin_call_with_host_context(ExtismPlugin *plugin,
const char *func_name,
const uint8_t *data,
ExtismSize data_len,
void *host_ctx);
```
---
### `extism_plugin_error`
Get the error associated with a `Plugin`
@@ -117,8 +74,6 @@ Get the error associated with a `Plugin`
const char *extism_plugin_error(ExtismPlugin *plugin);
```
---
### `extism_plugin_output_length`
Get the length of a plugin's output data.
@@ -127,8 +82,6 @@ Get the length of a plugin's output data.
ExtismSize extism_plugin_output_length(ExtismPlugin *plugin);
```
---
### `extism_plugin_output_data`
Get the plugin's output data.
@@ -137,18 +90,6 @@ Get the plugin's output data.
const uint8_t *extism_plugin_output_data(ExtismPlugin *plugin);
```
---
### `extism_plugin_reset`
Reset the Extism runtime, this will invalidate all allocated memory.
```c
bool extism_plugin_reset(ExtismPlugin *plugin);
```
---
### `extism_log_file`
Set log file and level.
@@ -157,8 +98,6 @@ Set log file and level.
bool extism_log_file(const char *filename, const char *log_level);
```
---
### `extism_log_custom`
Enable a custom log handler, this will buffer logs until `extism_log_drain`
@@ -168,8 +107,6 @@ is called Log level should be one of: info, error, trace, debug, warn
bool extism_log_custom(const char *log_level);
```
---
### `extism_log_drain`
Calls the provided callback function for each buffered log line.
@@ -179,8 +116,6 @@ This is only needed when `extism_log_custom` is used.
void extism_log_drain(void (*handler)(const char *, uintptr_t));
```
---
### `extism_version`
Get the Extism version string.
@@ -189,8 +124,6 @@ Get the Extism version string.
const char *extism_version(void);
```
---
### `extism_current_plugin_memory`
Returns a pointer to the memory of the currently running plugin
@@ -199,19 +132,6 @@ Returns a pointer to the memory of the currently running plugin
uint8_t *extism_current_plugin_memory(ExtismCurrentPlugin *plugin);
```
---
### `extism_current_plugin_host_context`
Get access to the host context, passed in using `extism_plugin_call_with_host_context`
```c
void *extism_current_plugin_host_context(ExtismCurrentPlugin *plugin);
```
---
### `extism_current_plugin_memory_alloc`
Allocate a memory block in the currently running plugin
@@ -220,8 +140,6 @@ Allocate a memory block in the currently running plugin
uint64_t extism_current_plugin_memory_alloc(ExtismCurrentPlugin *plugin, ExtismSize n);
```
---
### `extism_current_plugin_memory_length`
Get the length of an allocated block
@@ -230,8 +148,6 @@ Get the length of an allocated block
ExtismSize extism_current_plugin_memory_length(ExtismCurrentPlugin *plugin, ExtismSize n);
```
---
### `extism_current_plugin_memory_free`
Free an allocated memory block
@@ -240,8 +156,6 @@ Free an allocated memory block
void extism_current_plugin_memory_free(ExtismCurrentPlugin *plugin, uint64_t ptr);
```
---
### `extism_function_new`
Create a new host function
- `name`: function name, this should be valid UTF-8
@@ -268,8 +182,6 @@ ExtismFunction *extism_function_new(const char *name,
void (*free_user_data)(void *_));
```
---
### `extism_function_set_namespace`
Set the namespace of an `ExtismFunction`
@@ -278,8 +190,6 @@ Set the namespace of an `ExtismFunction`
void extism_function_set_namespace(ExtismFunction *ptr, const char *namespace_);
```
---
### `extism_function_free`
Free an `ExtismFunction`
@@ -288,8 +198,6 @@ Free an `ExtismFunction`
void extism_function_free(ExtismFunction *ptr);
```
---
### `extism_plugin_cancel_handle`
Get handle for plugin cancellation
@@ -298,8 +206,6 @@ Get handle for plugin cancellation
const ExtismCancelHandle *extism_plugin_cancel_handle(const ExtismPlugin *plugin);
```
---
### `extism_plugin_cancel`
Cancel a running plugin from another thread
@@ -308,14 +214,12 @@ Cancel a running plugin from another thread
bool extism_plugin_cancel(const ExtismCancelHandle *handle);
```
---
## Type definitions:
### `ExtismPlugin`
```c
typedef struct ExtismPlugin ExtismPlugin;
typedef int32_t ExtismPlugin;
```
### `ExtismSize`
@@ -347,5 +251,3 @@ typedef struct ExtismCurrentPlugin ExtismCurrentPlugin;
```c
typedef struct ExtismCancelHandle ExtismCancelHandle;
```

View File

@@ -4,7 +4,7 @@ build:
.PHONY: static
static:
$(CC) -g -o example example.c -l:libextism.a -lm -lpthread
$(CC) -g -o example example.c -l:libextism.a -lm
# if needed, set PKG_CONFIG_PATH= to the directory with extism*.pc installed
LDFLAGS=`pkg-config --libs extism`

View File

@@ -56,6 +56,7 @@ Since you may not have an Extism plug-in on hand to test, let's load a demo plug
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
void print_plugin_output(ExtismPlugin *plugin, int32_t rc){
if (rc != EXTISM_SUCCESS) {
@@ -63,9 +64,9 @@ void print_plugin_output(ExtismPlugin *plugin, int32_t rc){
return;
}
ExtismSize outlen = extism_plugin_output_length(plugin);
size_t outlen = extism_plugin_output_length(plugin);
const uint8_t *out = extism_plugin_output_data(plugin);
fwrite(out, 1, outlen, stdout);
write(STDOUT_FILENO, out, outlen);
}
int main(void) {
@@ -105,9 +106,9 @@ if (rc != EXTISM_SUCCESS) {
exit(2);
}
ExtismSize outlen = extism_plugin_output_length(plugin);
size_t outlen = extism_plugin_output_length(plugin);
const uint8_t *out = extism_plugin_output_data(plugin);
fwrite(out, 1, outlen, stdout);
write(STDOUT_FILENO, out, outlen);
```
Will print
@@ -170,6 +171,7 @@ We want to expose two functions to our plugin, `kv_write(key: String, value: Byt
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
// A stubbed out KV store
typedef struct KVStore KVStore;
@@ -182,14 +184,14 @@ extern const uint32_t fake_kv_store_get(KVStore *kv, const char *key,
// Our host functions to access the fake KV store
void kv_get(ExtismCurrentPlugin *plugin, const ExtismVal *inputs,
ExtismSize ninputs, ExtismVal *outputs, ExtismSize noutputs,
size_t ninputs, ExtismVal *outputs, size_t noutputs,
void *userdata) {
// Cast the userdata pointer
KVStore *kv = (KVStore *)userdata;
// Get the offset to the key in the plugin memory
uint64_t offs = inputs[0].v.i64;
ExtismSize keylen = extism_current_plugin_memory_length(plugin, offs);
size_t keylen = extism_current_plugin_memory_length(plugin, offs);
// Allocate a new block to return
uint64_t outoffs =
@@ -203,23 +205,23 @@ void kv_get(ExtismCurrentPlugin *plugin, const ExtismVal *inputs,
*(uint64_t *)(extism_current_plugin_memory(plugin) + outoffs) = value;
// Return the offset to our allocated block
outputs[0].t = EXTISM_PTR;
outputs[0].t = PTR;
outputs[0].v.i64 = outoffs;
}
void kv_set(ExtismCurrentPlugin *plugin, const ExtismVal *inputs,
ExtismSize ninputs, ExtismVal *outputs, ExtismSize noutputs,
size_t ninputs, ExtismVal *outputs, size_t noutputs,
void *userdata) {
// Cast the userdata pointer
KVStore *kv = (KVStore *)userdata;
// Get the offset to the key in the plugin memory
uint64_t keyoffs = inputs[0].v.i64;
ExtismSize keylen = extism_current_plugin_memory_length(plugin, keyoffs);
size_t keylen = extism_current_plugin_memory_length(plugin, keyoffs);
// Get the offset to the value in the plugin memory
uint64_t valueoffs = inputs[1].v.i64;
ExtismSize valuelen = extism_current_plugin_memory_length(plugin, valueoffs);
size_t valuelen = extism_current_plugin_memory_length(plugin, valueoffs);
// Set key => value
fake_kv_store_set(
@@ -232,13 +234,13 @@ int main(void) {
const char *manifest = "{\"wasm\": [{\"url\": "
"\"https://github.com/extism/plugins/releases/latest/"
"download/count_vowels_kvstore.wasm\"}]}";
const ExtismValType kv_get_inputs[] = {EXTISM_PTR};
const ExtismValType kv_get_outputs[] = {EXTISM_PTR};
const ExtismValType kv_get_inputs[] = {PTR};
const ExtismValType kv_get_outputs[] = {PTR};
ExtismFunction *kv_get_fn = extism_function_new(
"kv_get", kv_get_inputs, 1, kv_get_outputs, 1, kv_get, kv, NULL);
const ExtismValType kv_set_inputs[] = {EXTISM_PTR};
const ExtismValType kv_set_outputs[] = {EXTISM_PTR};
const ExtismValType kv_set_inputs[] = {PTR};
const ExtismValType kv_set_outputs[] = {PTR};
ExtismFunction *kv_set_fn = extism_function_new(
"kv_set", kv_set_inputs, 1, kv_set_outputs, 1, kv_set, kv, NULL);
const ExtismFunction *functions[] = {kv_get_fn};
@@ -286,3 +288,5 @@ print_plugin_output(plugin, extism_plugin_call(plugin, "count_vowels",
# => Writing value=6 from key=count-vowels"
# => {"count": 3, "total": 6, "vowels": "aeiouAEIOU"}
```

View File

@@ -9,7 +9,7 @@
#include <sys/stat.h>
#include <unistd.h>
void log_handler(const char *line, ExtismSize length) {
void log_handler(const char *line, uintptr_t length) {
fwrite(line, length, 1, stderr);
}
@@ -63,8 +63,8 @@ int main(int argc, char *argv[]) {
size_t len = 0;
uint8_t *data = read_file("../wasm/code-functions.wasm", &len);
ExtismValType inputs[] = {EXTISM_PTR};
ExtismValType outputs[] = {EXTISM_PTR};
ExtismValType inputs[] = {PTR};
ExtismValType outputs[] = {PTR};
ExtismFunction *f =
extism_function_new("hello_world", inputs, 1, outputs, 1, hello_world,
"Hello, again!", free_data);

View File

@@ -5,6 +5,6 @@ includedir=${prefix}/include
Version: 1.0.0
Name: Extism
Description: The framework for building with WebAssembly (wasm).
Libs: ${libdir}/libextism.a
Libs.private: -lm -lpthread
Libs: -L${libdir} -l:libextism.a
Libs.private: -lm
Cflags: -I${includedir}

View File

@@ -6,5 +6,5 @@ Version: 1.0.0
Name: Extism
Description: The framework for building with WebAssembly (wasm).
Libs: -L${libdir} -lextism
Libs.private: -lm -lpthread
Libs.private: -lm
Cflags: -I${includedir}

View File

@@ -10,7 +10,7 @@ version.workspace = true
[dependencies]
serde = { version = "1", features = ["derive"] }
base64 = "~0.22"
base64 = "~0.21"
schemars = { version = "0.8", optional = true }
serde_json = "1"

View File

@@ -1,11 +1,9 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Manifest",
"description": "The `Manifest` type is used to configure the runtime and specify how to load modules.",
"type": "object",
"properties": {
"allowed_hosts": {
"description": "Specifies which hosts may be accessed via HTTP, if this is empty then no hosts may be accessed. Wildcards may be used.",
"default": null,
"type": [
"array",
@@ -16,7 +14,6 @@
}
},
"allowed_paths": {
"description": "Specifies which paths should be made available on disk when using WASI. This is a mapping from the path on disk to the path it should be available inside the plugin. For example, `\".\": \"/tmp\"` would mount the current directory as `/tmp` inside the module",
"default": null,
"type": [
"object",
@@ -27,7 +24,6 @@
}
},
"config": {
"description": "Config values are made accessible using the PDK `extism_config_get` function",
"default": {},
"type": "object",
"additionalProperties": {
@@ -35,11 +31,8 @@
}
},
"memory": {
"description": "Memory options",
"default": {
"max_http_response_bytes": null,
"max_pages": null,
"max_var_bytes": null
"max_pages": null
},
"allOf": [
{
@@ -48,8 +41,6 @@
]
},
"timeout_ms": {
"description": "The plugin timeout in milliseconds",
"default": null,
"type": [
"integer",
"null"
@@ -58,7 +49,6 @@
"minimum": 0.0
},
"wasm": {
"description": "WebAssembly modules, the `main` module should be named `main` or listed last",
"default": [],
"type": "array",
"items": {
@@ -66,63 +56,35 @@
}
}
},
"additionalProperties": false,
"definitions": {
"MemoryOptions": {
"description": "Configure memory settings",
"type": "object",
"properties": {
"max_http_response_bytes": {
"description": "The maximum number of bytes allowed in an HTTP response",
"default": null,
"type": [
"integer",
"null"
],
"format": "uint64",
"minimum": 0.0
},
"max_pages": {
"description": "The max number of WebAssembly pages that should be allocated",
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 0.0
},
"max_var_bytes": {
"description": "The maximum number of bytes allowed to be used by plugin vars. Setting this to 0 will disable Extism vars. The default value is 1mb.",
"default": 1048576,
"type": [
"integer",
"null"
],
"format": "uint64",
"minimum": 0.0
}
},
"additionalProperties": false
}
},
"Wasm": {
"description": "The `Wasm` type specifies how to access a WebAssembly module",
"anyOf": [
{
"description": "From disk",
"type": "object",
"required": [
"path"
],
"properties": {
"hash": {
"description": "Module hash, if the data loaded from disk or via HTTP doesn't match an error will be raised",
"type": [
"string",
"null"
]
},
"name": {
"description": "Module name, this is used by Extism to determine which is the `main` module",
"type": [
"string",
"null"
@@ -131,72 +93,45 @@
"path": {
"type": "string"
}
},
"additionalProperties": false
}
},
{
"description": "From memory",
"type": "object",
"required": [
"data"
],
"properties": {
"data": {
"type": [
"string",
"object"
],
"required": [
"len",
"ptr"
],
"properties": {
"len": {
"type": "integer",
"format": "uint64",
"minimum": 0.0
},
"ptr": {
"type": "integer",
"format": "uint64",
"minimum": 0.0
}
},
"additionalProperties": false
"type": "string",
"format": "string"
},
"hash": {
"description": "Module hash, if the data loaded from disk or via HTTP doesn't match an error will be raised",
"type": [
"string",
"null"
]
},
"name": {
"description": "Module name, this is used by Extism to determine which is the `main` module",
"type": [
"string",
"null"
]
}
},
"additionalProperties": false
}
},
{
"description": "Via HTTP",
"type": "object",
"required": [
"url"
],
"properties": {
"hash": {
"description": "Module hash, if the data loaded from disk or via HTTP doesn't match an error will be raised",
"type": [
"string",
"null"
]
},
"headers": {
"description": "Request headers",
"default": {},
"type": "object",
"additionalProperties": {
@@ -204,25 +139,21 @@
}
},
"method": {
"description": "Request method",
"type": [
"string",
"null"
]
},
"name": {
"description": "Module name, this is used by Extism to determine which is the `main` module",
"type": [
"string",
"null"
]
},
"url": {
"description": "The request URL",
"type": "string"
}
},
"additionalProperties": false
}
}
]
}

View File

@@ -10,45 +10,8 @@ pub type ManifestMemory = MemoryOptions;
#[serde(deny_unknown_fields)]
pub struct MemoryOptions {
/// The max number of WebAssembly pages that should be allocated
#[serde(alias = "max")]
pub max_pages: Option<u32>,
/// The maximum number of bytes allowed in an HTTP response
#[serde(default)]
pub max_http_response_bytes: Option<u64>,
/// The maximum number of bytes allowed to be used by plugin vars. Setting this to 0
/// will disable Extism vars. The default value is 1mb.
#[serde(default = "default_var_bytes")]
pub max_var_bytes: Option<u64>,
}
impl MemoryOptions {
/// Create an empty `MemoryOptions` value
pub fn new() -> Self {
Default::default()
}
/// Set max pages
pub fn with_max_pages(mut self, pages: u32) -> Self {
self.max_pages = Some(pages);
self
}
/// Set max HTTP response size
pub fn with_max_http_response_bytes(mut self, bytes: u64) -> Self {
self.max_http_response_bytes = Some(bytes);
self
}
/// Set max size of Extism vars
pub fn with_max_var_bytes(mut self, bytes: u64) -> Self {
self.max_var_bytes = Some(bytes);
self
}
}
fn default_var_bytes() -> Option<u64> {
Some(1024 * 1024)
}
/// Generic HTTP request structure
@@ -61,6 +24,7 @@ pub struct HttpRequest {
/// Request headers
#[serde(default)]
#[serde(alias = "header")]
pub headers: std::collections::BTreeMap<String, String>,
/// Request method
@@ -147,8 +111,8 @@ pub enum Wasm {
/// From memory
Data {
#[serde(with = "wasmdata")]
#[cfg_attr(feature = "json_schema", schemars(schema_with = "wasmdata_schema"))]
#[serde(with = "base64")]
#[cfg_attr(feature = "json_schema", schemars(schema_with = "base64_schema"))]
data: Vec<u8>,
#[serde(flatten)]
meta: WasmMetadata,
@@ -231,25 +195,11 @@ impl Wasm {
}
}
#[derive(Default, Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
#[cfg_attr(feature = "json_schema", derive(schemars::JsonSchema))]
#[serde(deny_unknown_fields)]
struct DataPtrLength {
ptr: u64,
len: u64,
}
#[cfg(feature = "json_schema")]
fn wasmdata_schema(gen: &mut schemars::gen::SchemaGenerator) -> schemars::schema::Schema {
fn base64_schema(gen: &mut schemars::gen::SchemaGenerator) -> schemars::schema::Schema {
use schemars::{schema::SchemaObject, JsonSchema};
let mut schema: SchemaObject = <String>::json_schema(gen).into();
let objschema: SchemaObject = <DataPtrLength>::json_schema(gen).into();
let types = schemars::schema::SingleOrVec::<schemars::schema::InstanceType>::Vec(vec![
schemars::schema::InstanceType::String,
schemars::schema::InstanceType::Object,
]);
schema.instance_type = Some(types);
schema.object = objschema.object;
schema.format = Some("string".to_owned());
schema.into()
}
@@ -261,7 +211,6 @@ pub struct Manifest {
/// WebAssembly modules, the `main` module should be named `main` or listed last
#[serde(default)]
pub wasm: Vec<Wasm>,
/// Memory options
#[serde(default)]
pub memory: MemoryOptions,
@@ -269,8 +218,8 @@ pub struct Manifest {
/// Config values are made accessible using the PDK `extism_config_get` function
#[serde(default)]
pub config: BTreeMap<String, String>,
#[serde(default)]
/// Specifies which hosts may be accessed via HTTP, if this is empty then
/// no hosts may be accessed. Wildcards may be used.
pub allowed_hosts: Option<Vec<String>>,
@@ -279,18 +228,23 @@ pub struct Manifest {
/// the path on disk to the path it should be available inside the plugin.
/// For example, `".": "/tmp"` would mount the current directory as `/tmp` inside the module
#[serde(default)]
pub allowed_paths: Option<BTreeMap<String, PathBuf>>,
pub allowed_paths: Option<BTreeMap<PathBuf, PathBuf>>,
/// The plugin timeout in milliseconds
#[serde(default)]
/// The plugin timeout, by default this is set to 30s
#[serde(default = "default_timeout")]
pub timeout_ms: Option<u64>,
}
fn default_timeout() -> Option<u64> {
Some(30000)
}
impl Manifest {
/// Create a new manifest
pub fn new(wasm: impl IntoIterator<Item = impl Into<Wasm>>) -> Manifest {
Manifest {
wasm: wasm.into_iter().map(|x| x.into()).collect(),
timeout_ms: default_timeout(),
..Default::default()
}
}
@@ -337,7 +291,8 @@ impl Manifest {
}
/// Add a path to `allowed_paths`
pub fn with_allowed_path(mut self, src: String, dest: impl AsRef<Path>) -> Self {
pub fn with_allowed_path(mut self, src: impl AsRef<Path>, dest: impl AsRef<Path>) -> Self {
let src = src.as_ref().to_path_buf();
let dest = dest.as_ref().to_path_buf();
match &mut self.allowed_paths {
Some(p) => {
@@ -354,7 +309,7 @@ impl Manifest {
}
/// Set `allowed_paths`
pub fn with_allowed_paths(mut self, paths: impl Iterator<Item = (String, PathBuf)>) -> Self {
pub fn with_allowed_paths(mut self, paths: impl Iterator<Item = (PathBuf, PathBuf)>) -> Self {
self.allowed_paths = Some(paths.collect());
self
}
@@ -385,12 +340,10 @@ impl Manifest {
}
}
mod wasmdata {
use crate::DataPtrLength;
mod base64 {
use base64::{engine::general_purpose, Engine as _};
use serde::{Deserialize, Serialize};
use serde::{Deserializer, Serializer};
use std::slice;
pub fn serialize<S: Serializer>(v: &Vec<u8>, s: S) -> Result<S::Ok, S::Error> {
let base64 = general_purpose::STANDARD.encode(v.as_slice());
@@ -398,33 +351,21 @@ mod wasmdata {
}
pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Vec<u8>, D::Error> {
#[derive(Deserialize)]
#[serde(untagged)]
enum WasmDataTypes {
String(String),
DataPtrLength(DataPtrLength),
}
Ok(match WasmDataTypes::deserialize(d)? {
WasmDataTypes::String(string) => general_purpose::STANDARD
.decode(string.as_bytes())
.map_err(serde::de::Error::custom)?,
WasmDataTypes::DataPtrLength(ptrlen) => {
let slice =
unsafe { slice::from_raw_parts(ptrlen.ptr as *const u8, ptrlen.len as usize) };
slice.to_vec()
}
})
let base64 = String::deserialize(d)?;
general_purpose::STANDARD
.decode(base64.as_bytes())
.map_err(serde::de::Error::custom)
}
}
impl From<Manifest> for std::borrow::Cow<'_, [u8]> {
impl<'a> From<Manifest> for std::borrow::Cow<'a, [u8]> {
fn from(m: Manifest) -> Self {
let s = serde_json::to_vec(&m).unwrap();
std::borrow::Cow::Owned(s)
}
}
impl From<&Manifest> for std::borrow::Cow<'_, [u8]> {
impl<'a> From<&Manifest> for std::borrow::Cow<'a, [u8]> {
fn from(m: &Manifest) -> Self {
let s = serde_json::to_vec(&m).unwrap();
std::borrow::Cow::Owned(s)

View File

@@ -9,21 +9,20 @@ repository.workspace = true
version.workspace = true
[dependencies]
wasmtime = {version = ">= 27.0.0, < 31.0.0"}
wasi-common = {version = ">= 27.0.0, < 31.0.0"}
wiggle = {version = ">= 27.0.0, < 31.0.0"}
wasmtime = ">= 14.0.0, < 16.0.0"
wasmtime-wasi = ">= 14.0.0, < 16.0.0"
anyhow = "1"
serde = {version = "1", features = ["derive"]}
serde_json = "1"
toml = "0.8"
sha2 = "0.10"
tracing = "0.1"
tracing-subscriber = {version = "0.3.18", features = ["std", "env-filter", "fmt"]}
tracing-subscriber = {version = "0.3", features = ["std", "env-filter", "fmt"]}
url = "2"
glob = "0.3"
ureq = {version = "3.0", optional=true}
ureq = {version = "2.5", optional=true}
extism-manifest = { workspace = true }
extism-convert = { workspace = true, features = ["extism-path"] }
extism-convert = { workspace = true }
uuid = { version = "1", features = ["v4"] }
libc = "0.2"
@@ -34,12 +33,10 @@ register-filesystem = [] # enables wasm to be loaded from disk
http = ["ureq"] # enables extism_http_request
[build-dependencies]
cbindgen = { version = "0.29", default-features = false }
cbindgen = "0.26"
[dev-dependencies]
criterion = "0.6.0"
quickcheck = "1"
rand = "0.9.0"
criterion = "0.5.1"
[[bench]]
name = "bench"

View File

@@ -12,7 +12,7 @@ To use the `extism` crate, you can add it to your Cargo file:
```toml
[dependencies]
extism = "1.4.1"
extism = "^1.0.0-rc3"
```
## Environment variables
@@ -24,7 +24,7 @@ There are a few environment variables that can be used for debugging purposes:
- `EXTISM_COREDUMP=extism.core`: write [coredump](https://github.com/WebAssembly/tool-conventions/blob/main/Coredump.md) to a file when a WebAssembly function traps
- `EXTISM_DEBUG=1`: generate debug information
- `EXTISM_PROFILE=perf|jitdump|vtune`: enable Wasmtime profiling
- `EXTISM_CACHE_CONFIG=path/to/config.toml`: enable Wasmtime cache, details [here](#wasmtime-caching)
- `EXTISM_CACHE_CONFIG=path/to/config.toml`: enable Wasmtime cache, see [the docs](https://docs.wasmtime.dev/cli-cache.html) for details about configuration. Setting this to an empty string will disable caching.
> *Note*: The debug and coredump info will only be written if the plug-in has an error.
@@ -88,7 +88,7 @@ println!("{:?}", res);
### Plug-in State
Plug-ins may be stateful or stateless. Plug-ins can maintain state between calls by the use of variables. Our count vowels plug-in remembers the total number of vowels it's ever counted in the "total" key in the result. You can see this by making subsequent calls to the export:
Plug-ins may be stateful or stateless. Plug-ins can maintain state b/w calls by the use of variables. Our count vowels plug-in remembers the total number of vowels it's ever counted in the "total" key in the result. You can see this by making subsequent calls to the export:
```rust
let res = plugin.call::<&str, &str>("count_vowels", "Hello, world!").unwrap();
@@ -201,7 +201,7 @@ fn main() {
}
```
> *Note*: In order to write host functions you should get familiar with the methods on the [CurrentPlugin](https://docs.rs/extism/latest/extism/struct.CurrentPlugin.html) and [UserData](https://docs.rs/extism/latest/extism/enum.UserData.html) types.
> *Note*: In order to write host functions you should get familiar with the methods on the [Extism::CurrentPlugin](https://docs.rs/extism/latest/extism/struct.CurrentPlugin.html) and [Extism::CurrentPlugin](https://docs.rs/extism/latest/extism/struct.UserData.html) types.
Now we can invoke the event:
@@ -219,52 +219,5 @@ println!("{}", res);
# => {"count": 3, "total": 6, "vowels": "aeiouAEIOU"}
```
### Logging
Plug-ins can't directly print anything to the console. They can however use Extism's built-in logging functionality, for example the [`log!` macro in the rust-pdk](https://docs.rs/extism-pdk/1.4.0/extism_pdk/macro.log.html) or the [`logInfo` function in the haskell-pdk](https://hackage.haskell.org/package/extism-pdk-1.2.0.0/docs/Extism-PDK.html#v:logInfo).
Inside your host application, the rust-sdk emits these as [tracing](https://github.com/tokio-rs/tracing) events. The simplest way to make the logged messages visible is by adding the `tracing_subscriber` dependency to your crate and then initializing a tracing subscriber at the top of your main function:
```rust
tracing_subscriber::fmt::init();
```
### Wasmtime Caching
To enable or disable caching for plugin compilation, you need to provide a configuration file that will be used by the [wasmtime crate](https://github.com/bytecodealliance/wasmtime).
For more information and values that can be used for configuring caching, take a look at [the docs](https://docs.wasmtime.dev/cli-cache.html).
> *Note*: As of now extism uses wasmtime [`version = ">= 27.0.0, < 31.0.0"`](https://github.com/extism/extism/blob/v1.11.1/runtime/Cargo.toml#L12), but the `enabled` key requirement [was removed](https://github.com/bytecodealliance/wasmtime/pull/10859) from `wasmtime` and its documentation, this could explain the `failed to parse config file` error you might encounter without it.
An example configuration for caching would be:
```toml
[cache]
enabled = true # This value is required
directory = "/some/path"
```
You can :
- [Create a global `wasmtime` configuration file](#using-a-configuration-file) in `$HOME/.config/wasmtime/config.toml`.
- [Set the `EXTISM_CACHE_CONFIG` environment variable](#using-an-environment-variable)
- [Set the configuration file path using `PluginBuilder`](#using-pluginbuilder)
#### Using a configuration file
The [wasmtime](https://github.com/bytecodealliance/wasmtime) crate, by default, will look for a configuration file in your systems' default configuration directory (for example on UNIX systems: `$HOME/.config/wasmtime/config.toml`),
for more [information on this behaviour](`https://docs.rs/wasmtime/31.0.0/wasmtime/struct.Config.html#method.cache_config_load_default`).
#### Using an environment variable
You can set the `EXTISM_CACHE_CONFIG=path/to/config.toml` environment variable to set the path of the configuration file used by [wasmtime](https://github.com/bytecodealliance/wasmtime).
Setting the variable to an empty string will disable caching (it won't load any configuration file).
> *Note*: If the environment variable is not set, `wasmtime` will still try to read from a configuration file that may exist in your system's default configuration folder (e.g. `$HOME/.config/wasmtime/config.toml`).
The environment variable does not override the path you might have set using `PluginBuilder`. will only be checked for if you did not specify a cache configuration path in `PluginBuilder`.
#### Using PluginBuilder
If you use a [PluginBuilder](https://docs.rs/extism/latest/extism/struct.PluginBuilder.html), you can set the `wasmtime` configuration path using the [with_cache_config](https://docs.rs/extism/latest/extism/struct.PluginBuilder.html#method.with_cache_config) method.
This will override the `EXTISM_CACHE_CONFIG` environment variable if it's set, so you could have a "global" and per plugin configuration if needed.

View File

@@ -6,7 +6,6 @@ const COUNT_VOWELS: &[u8] = include_bytes!("../../wasm/code.wasm");
const REFLECT: &[u8] = include_bytes!("../../wasm/reflect.wasm");
const ECHO: &[u8] = include_bytes!("../../wasm/echo.wasm");
const CONSUME: &[u8] = include_bytes!("../../wasm/consume.wasm");
const ALLOCATIONS: &[u8] = include_bytes!("../../wasm/allocations.wasm");
host_fn!(hello_world (a: String) -> String { Ok(a) });
@@ -36,46 +35,6 @@ pub fn create_plugin(c: &mut Criterion) {
});
}
pub fn create_compiled(c: &mut Criterion) {
let mut g = c.benchmark_group("create");
g.noise_threshold(1.0);
g.significance_level(0.2);
g.bench_function("create_compiled", |b| {
b.iter(|| {
let plugin = PluginBuilder::new(COUNT_VOWELS).with_wasi(true);
let _compiled = CompiledPlugin::new(plugin).unwrap();
})
});
}
pub fn create_plugin_compiled(c: &mut Criterion) {
let mut g = c.benchmark_group("create");
g.noise_threshold(1.0);
g.significance_level(0.2);
let plugin = PluginBuilder::new(COUNT_VOWELS).with_wasi(true);
let compiled = CompiledPlugin::new(plugin).unwrap();
g.bench_function("create_plugin_compiled", |b| {
b.iter(|| {
let _plugin = Plugin::new_from_compiled(&compiled).unwrap();
})
});
}
pub fn create_plugin_no_cache(c: &mut Criterion) {
let mut g = c.benchmark_group("create");
g.noise_threshold(1.0);
g.significance_level(0.2);
g.bench_function("create_plugin_no_cache", |b| {
b.iter(|| {
let _plugin = PluginBuilder::new(COUNT_VOWELS)
.with_cache_disabled()
.with_wasi(true)
.build()
.unwrap();
})
});
}
#[derive(Debug, serde::Deserialize, PartialEq)]
struct Count {
count: u32,
@@ -159,7 +118,9 @@ pub fn echo(c: &mut Criterion) {
pub fn reflect(c: &mut Criterion) {
let mut g = c.benchmark_group("reflect");
g.sample_size(500);
g.noise_threshold(1.0);
g.significance_level(0.2);
let mut plugin = PluginBuilder::new(REFLECT)
.with_wasi(true)
.with_function(
@@ -171,40 +132,28 @@ pub fn reflect(c: &mut Criterion) {
)
.build()
.unwrap();
for (i, elements) in [
b"a".repeat(65536),
b"a".repeat(65536 * 10),
b"a".repeat(65536 * 100),
b"a".repeat(65536),
]
.iter()
.enumerate()
{
g.throughput(criterion::Throughput::Bytes(elements.len() as u64));
g.bench_with_input(
format!("{i}: reflect {} bytes", elements.len()),
format!("reflect {} bytes", 10u32.pow(i as u32) * 65536),
elements,
|b, elems| {
b.iter(|| {
assert_eq!(elems, plugin.call::<_, &[u8]>("reflect", &elems).unwrap());
// plugin.reset().unwrap();
});
},
);
}
}
pub fn allocations(c: &mut Criterion) {
let mut g = c.benchmark_group("allocations");
let mut plugin = PluginBuilder::new(ALLOCATIONS).build().unwrap();
g.bench_function("allocations", |b| {
b.iter(|| {
plugin.call::<_, ()>("allocations", "").unwrap();
})
});
}
// This is an apples-to-apples comparison of a linked wasm "reflect" function to our host "reflect"
// function.
pub fn reflect_linked(c: &mut Criterion) {
@@ -297,16 +246,12 @@ pub fn reflect_linked(c: &mut Criterion) {
criterion_group!(
benches,
allocations,
consume,
echo,
reflect,
reflect_linked,
basic,
create_plugin,
create_plugin_compiled,
create_plugin_no_cache,
create_compiled,
count_vowels
);
criterion_main!(benches);

View File

@@ -7,7 +7,7 @@ fn main() {
#define EXTISM_SUCCESS 0
/** An alias for I64 to signify an Extism pointer */
#define EXTISM_PTR ExtismValType_I64
#define PTR I64
";
if let Ok(x) = cbindgen::Builder::new()
.with_crate(".")
@@ -24,7 +24,6 @@ fn main() {
.rename_item("CurrentPlugin", "ExtismCurrentPlugin")
.rename_item("CancelHandle", "ExtismCancelHandle")
.rename_item("Plugin", "ExtismPlugin")
.rename_item("CompiledPlugin", "ExtismCompiledPlugin")
.rename_item("Function", "ExtismFunction")
.with_style(cbindgen::Style::Type)
.generate()

View File

@@ -1,36 +0,0 @@
use extism::*;
fn main() {
let url = Wasm::file("../wasm/read_write.wasm");
let manifest = Manifest::new([url])
// This will fail because we're using a readonly path (specified with the `ro:` prefix)
// to overwrite the data file, remove `ro:` from the path on the following line
.with_allowed_path("ro:src/tests/data".to_string(), "/data")
.with_config_key("path", "/data/data.txt");
let mut plugin = PluginBuilder::new(manifest)
.with_wasi(true)
.build()
.unwrap();
println!("trying to read file: ");
let res = plugin.call::<&str, &str>("try_read", "").unwrap();
println!("{res:?}");
println!("-----------------------------------------------------");
// If the allowed path is readonly then writing back to the file should fail
println!("trying to write file: ");
let line = format!(
"Hello World at {:?}\n",
std::time::SystemTime::now()
.duration_since(std::time::SystemTime::UNIX_EPOCH)
.unwrap()
);
let res2 = plugin.call::<&str, &str>("try_write", &line).unwrap();
println!("{res2:?}");
println!("done!");
}

View File

@@ -2,8 +2,6 @@ use extism::*;
fn main() {
let manifest = Manifest::new([
// upper.wat provides an export called `host_reflect` that takes a string
// and returns the same string uppercased
Wasm::File {
// See https://github.com/extism/plugins/blob/main/upper.wat
path: "../wasm/upper.wasm".into(),
@@ -12,9 +10,6 @@ fn main() {
hash: None,
},
},
// reflect expects host_reflect to be imported: https://github.com/extism/plugins/blob/e5578bbbdd87f9936a0a8d36df629768b2eff6bb/reflect/src/lib.rs#L5
// Extism will link the export from upper.wat to the import of reflect.rs at runtime so it
// can call it
Wasm::File {
// See https://github.com/extism/plugins/tree/main/reflect
path: "../wasm/reflect.wasm".into(),
@@ -30,6 +25,6 @@ fn main() {
let res = plugin
.call::<&str, &str>("reflect", "Hello, world!")
.unwrap();
println!("{res}");
println!("{}", res);
}
}

View File

@@ -25,6 +25,6 @@ fn main() {
println!("Dumping logs");
for line in LOGS.lock().unwrap().iter() {
print!("{line}");
print!("{}", line);
}
}

View File

@@ -52,6 +52,6 @@ fn main() {
let res = plugin
.call::<&str, &str>("count_vowels", "Hello, world!")
.unwrap();
println!("{res}");
println!("{}", res);
}
}

View File

@@ -10,7 +10,7 @@
#define EXTISM_SUCCESS 0
/** An alias for I64 to signify an Extism pointer */
#define EXTISM_PTR ExtismValType_I64
#define PTR I64
/**
@@ -20,31 +20,31 @@ typedef enum {
/**
* Signed 32 bit integer.
*/
ExtismValType_I32,
I32,
/**
* Signed 64 bit integer.
*/
ExtismValType_I64,
I64,
/**
* Floating point 32 bit integer.
*/
ExtismValType_F32,
F32,
/**
* Floating point 64 bit integer.
*/
ExtismValType_F64,
F64,
/**
* A 128 bit number.
*/
ExtismValType_V128,
V128,
/**
* A reference to a Wasm function.
*/
ExtismValType_FuncRef,
FuncRef,
/**
* A reference to opaque data in the Wasm instance.
*/
ExtismValType_ExternRef,
ExternRef,
} ExtismValType;
/**
@@ -52,8 +52,6 @@ typedef enum {
*/
typedef struct ExtismCancelHandle ExtismCancelHandle;
typedef struct ExtismCompiledPlugin ExtismCompiledPlugin;
/**
* CurrentPlugin stores data that is available to the caller in PDK functions, this should
* only be accessed from inside a host function
@@ -99,11 +97,6 @@ typedef void (*ExtismFunctionType)(ExtismCurrentPlugin *plugin,
ExtismSize n_outputs,
void *data);
/**
* Log drain callback
*/
typedef void (*ExtismLogDrainFunctionType)(const char *data, ExtismSize size);
#ifdef __cplusplus
@@ -115,12 +108,6 @@ extern "C" {
*/
const uint8_t *extism_plugin_id(ExtismPlugin *plugin);
/**
* Get the current plugin's associated host context data. Returns null if call was made without
* host context.
*/
void *extism_current_plugin_host_context(ExtismCurrentPlugin *plugin);
/**
* Returns a pointer to the memory of the currently running plugin
* NOTE: this should only be called from host functions.
@@ -156,7 +143,7 @@ void extism_current_plugin_memory_free(ExtismCurrentPlugin *plugin, ExtismMemory
* - `n_outputs`: number of return types
* - `func`: the function to call
* - `user_data`: a pointer that will be passed to the function when it's called
* this value should live as long as the function exists
* this value should live as long as the function exists
* - `free_user_data`: a callback to release the `user_data` value when the resulting
* `ExtismFunction` is freed.
*
@@ -181,21 +168,6 @@ void extism_function_free(ExtismFunction *f);
*/
void extism_function_set_namespace(ExtismFunction *ptr, const char *namespace_);
/**
* Pre-compile an Extism plugin
*/
ExtismCompiledPlugin *extism_compiled_plugin_new(const uint8_t *wasm,
ExtismSize wasm_size,
const ExtismFunction **functions,
ExtismSize n_functions,
bool with_wasi,
char **errmsg);
/**
* Free `ExtismCompiledPlugin`
*/
void extism_compiled_plugin_free(ExtismCompiledPlugin *plugin);
/**
* Create a new plugin with host functions, the functions passed to this function no longer need to be manually freed using
*
@@ -212,34 +184,13 @@ ExtismPlugin *extism_plugin_new(const uint8_t *wasm,
bool with_wasi,
char **errmsg);
/**
* Create a new plugin from an `ExtismCompiledPlugin`
*/
ExtismPlugin *extism_plugin_new_from_compiled(const ExtismCompiledPlugin *compiled, char **errmsg);
/**
* Create a new plugin and set the number of instructions a plugin is allowed to execute
*/
ExtismPlugin *extism_plugin_new_with_fuel_limit(const uint8_t *wasm,
ExtismSize wasm_size,
const ExtismFunction **functions,
ExtismSize n_functions,
bool with_wasi,
uint64_t fuel_limit,
char **errmsg);
/**
* Enable HTTP response headers in plugins using `extism:host/env::http_request`
*/
void extism_plugin_allow_http_response_headers(ExtismPlugin *plugin);
/**
* Free the error returned by `extism_plugin_new`, errors returned from `extism_plugin_error` don't need to be freed
*/
void extism_plugin_new_error_free(char *err);
/**
* Free `ExtismPlugin`
* Remove a plugin from the registry and free associated memory
*/
void extism_plugin_free(ExtismPlugin *plugin);
@@ -275,20 +226,6 @@ int32_t extism_plugin_call(ExtismPlugin *plugin,
const uint8_t *data,
ExtismSize data_len);
/**
* Call a function with host context.
*
* `func_name`: is the function to call
* `data`: is the input data
* `data_len`: is the length of `data`
* `host_context`: a pointer to context data that will be available in host functions
*/
int32_t extism_plugin_call_with_host_context(ExtismPlugin *plugin,
const char *func_name,
const uint8_t *data,
ExtismSize data_len,
void *host_context);
/**
* Get the error associated with a `Plugin`
*/
@@ -327,12 +264,7 @@ bool extism_log_custom(const char *log_level);
* Calls the provided callback function for each buffered log line.
* This is only needed when `extism_log_custom` is used.
*/
void extism_log_drain(ExtismLogDrainFunctionType handler);
/**
* Reset the Extism runtime, this will invalidate all allocated memory
*/
bool extism_plugin_reset(ExtismPlugin *plugin);
void extism_log_drain(void (*handler)(const char*, uintptr_t));
/**
* Get the Extism version string
@@ -340,5 +272,5 @@ bool extism_plugin_reset(ExtismPlugin *plugin);
const char *extism_version(void);
#ifdef __cplusplus
} // extern "C"
#endif // __cplusplus
} // extern "C"
#endif // __cplusplus

View File

@@ -1,5 +1,3 @@
use anyhow::Context;
use crate::*;
/// CurrentPlugin stores data that is available to the caller in PDK functions, this should
@@ -14,11 +12,9 @@ pub struct CurrentPlugin {
pub(crate) linker: *mut wasmtime::Linker<CurrentPlugin>,
pub(crate) wasi: Option<Wasi>,
pub(crate) http_status: u16,
pub(crate) http_headers: Option<std::collections::BTreeMap<String, String>>,
pub(crate) available_pages: Option<u32>,
pub(crate) memory_limiter: Option<MemoryLimiter>,
pub(crate) id: uuid::Uuid,
pub(crate) start_time: std::time::Instant,
}
unsafe impl Send for CurrentPlugin {}
@@ -56,12 +52,7 @@ impl wasmtime::ResourceLimiter for MemoryLimiter {
Ok(true)
}
fn table_growing(
&mut self,
_current: usize,
desired: usize,
maximum: Option<usize>,
) -> Result<bool> {
fn table_growing(&mut self, _current: u32, desired: u32, maximum: Option<u32>) -> Result<bool> {
if let Some(max) = maximum {
return Ok(desired <= max);
}
@@ -71,11 +62,6 @@ impl wasmtime::ResourceLimiter for MemoryLimiter {
}
impl CurrentPlugin {
/// Gets `Plugin`'s ID
pub fn id(&self) -> uuid::Uuid {
self.id
}
/// Get a `MemoryHandle` from a memory offset
pub fn memory_handle(&mut self, offs: u64) -> Option<MemoryHandle> {
if offs == 0 {
@@ -101,15 +87,9 @@ impl CurrentPlugin {
}
/// Access memory bytes as `str`
pub fn memory_str_mut(&mut self, handle: MemoryHandle) -> Result<&mut str, Error> {
let bytes = self.memory_bytes_mut(handle)?;
let s = std::str::from_utf8_mut(bytes)?;
Ok(s)
}
pub fn memory_str(&mut self, handle: MemoryHandle) -> Result<&str, Error> {
pub fn memory_str(&mut self, handle: MemoryHandle) -> Result<&mut str, Error> {
let bytes = self.memory_bytes(handle)?;
let s = std::str::from_utf8(bytes)?;
let s = std::str::from_utf8_mut(bytes)?;
Ok(s)
}
@@ -117,11 +97,8 @@ impl CurrentPlugin {
pub fn memory_new<'a, T: ToBytes<'a>>(&mut self, t: T) -> Result<MemoryHandle, Error> {
let data = t.to_bytes()?;
let data = data.as_ref();
if data.is_empty() {
return Ok(MemoryHandle::null());
}
let handle = self.memory_alloc(data.len() as u64)?;
let bytes = self.memory_bytes_mut(handle)?;
let bytes = self.memory_bytes(handle)?;
bytes.copy_from_slice(data.as_ref());
Ok(handle)
}
@@ -167,11 +144,11 @@ impl CurrentPlugin {
Ok(())
}
pub fn memory_bytes_mut(&mut self, handle: MemoryHandle) -> Result<&mut [u8], Error> {
let (linker, store) = self.linker_and_store();
if let Some(mem) = linker.get(&mut *store, EXTISM_ENV_MODULE, "memory") {
pub fn memory_bytes(&mut self, handle: MemoryHandle) -> Result<&mut [u8], Error> {
let (linker, mut store) = self.linker_and_store();
if let Some(mem) = linker.get(&mut store, EXTISM_ENV_MODULE, "memory") {
let mem = mem.into_memory().unwrap();
let ptr = unsafe { mem.data_ptr(&*store).add(handle.offset() as usize) };
let ptr = unsafe { mem.data_ptr(&store).add(handle.offset() as usize) };
if ptr.is_null() {
return Ok(&mut []);
}
@@ -181,44 +158,6 @@ impl CurrentPlugin {
anyhow::bail!("{} unable to locate extism memory", self.id)
}
pub fn memory_bytes(&mut self, handle: MemoryHandle) -> Result<&[u8], Error> {
let (linker, store) = self.linker_and_store();
if let Some(mem) = linker.get(&mut *store, EXTISM_ENV_MODULE, "memory") {
let mem = mem.into_memory().unwrap();
let ptr = unsafe { mem.data_ptr(&*store).add(handle.offset() as usize) };
if ptr.is_null() {
return Ok(&[]);
}
return Ok(unsafe { std::slice::from_raw_parts(ptr, handle.len()) });
}
anyhow::bail!("{} unable to locate extism memory", self.id)
}
pub fn host_context<T: 'static>(&mut self) -> Result<&mut T, Error> {
let (linker, store) = self.linker_and_store();
let Some(Extern::Global(xs)) = linker.get(&mut *store, EXTISM_ENV_MODULE, "extism_context")
else {
anyhow::bail!("unable to locate an extism kernel global: extism_context",)
};
let Val::ExternRef(Some(xs)) = xs.get(&mut *store) else {
anyhow::bail!("expected extism_context to be an externref value",)
};
if let Some(d) = xs.data_mut(&mut *store)? {
match d.downcast_mut::<Box<dyn std::any::Any + Send + Sync>>() {
Some(xs) => match xs.downcast_mut::<T>() {
Some(xs) => Ok(xs),
None => anyhow::bail!("could not downcast extism_context inner value"),
},
None => anyhow::bail!("could not downcast extism_context"),
}
} else {
anyhow::bail!("extism_context not found")
}
}
pub fn memory_alloc(&mut self, n: u64) -> Result<MemoryHandle, Error> {
if n == 0 {
return Ok(MemoryHandle {
@@ -229,13 +168,9 @@ impl CurrentPlugin {
let (linker, mut store) = self.linker_and_store();
let output = &mut [Val::I64(0)];
if let Some(f) = linker.get(&mut store, EXTISM_ENV_MODULE, "alloc") {
catch_out_of_fuel!(
&store,
f.into_func()
.unwrap()
.call(&mut *store, &[Val::I64(n as i64)], output)
.context("failed to allocate extism memory")
)?;
f.into_func()
.unwrap()
.call(&mut store, &[Val::I64(n as i64)], output)?;
} else {
anyhow::bail!("{} unable to allocate memory", self.id);
}
@@ -257,15 +192,11 @@ impl CurrentPlugin {
/// Free a block of Extism plugin memory
pub fn memory_free(&mut self, handle: MemoryHandle) -> Result<(), Error> {
let (linker, store) = self.linker_and_store();
if let Some(f) = linker.get(&mut *store, EXTISM_ENV_MODULE, "free") {
catch_out_of_fuel!(
&store,
f.into_func()
.unwrap()
.call(&mut *store, &[Val::I64(handle.offset as i64)], &mut [])
.context("failed to free extism memory")
)?;
let (linker, mut store) = self.linker_and_store();
if let Some(f) = linker.get(&mut store, EXTISM_ENV_MODULE, "free") {
f.into_func()
.unwrap()
.call(&mut store, &[Val::I64(handle.offset as i64)], &mut [])?;
} else {
anyhow::bail!("unable to locate an extism kernel function: free",)
}
@@ -273,16 +204,12 @@ impl CurrentPlugin {
}
pub fn memory_length(&mut self, offs: u64) -> Result<u64, Error> {
let (linker, store) = self.linker_and_store();
let (linker, mut store) = self.linker_and_store();
let output = &mut [Val::I64(0)];
if let Some(f) = linker.get(&mut *store, EXTISM_ENV_MODULE, "length") {
catch_out_of_fuel!(
&store,
f.into_func()
.unwrap()
.call(&mut *store, &[Val::I64(offs as i64)], output)
.context("failed to get length of extism memory handle")
)?;
if let Some(f) = linker.get(&mut store, EXTISM_ENV_MODULE, "length") {
f.into_func()
.unwrap()
.call(&mut store, &[Val::I64(offs as i64)], output)?;
} else {
anyhow::bail!("unable to locate an extism kernel function: length",)
}
@@ -296,30 +223,6 @@ impl CurrentPlugin {
Ok(len)
}
pub fn memory_length_unsafe(&mut self, offs: u64) -> Result<u64, Error> {
let (linker, store) = self.linker_and_store();
let output = &mut [Val::I64(0)];
if let Some(f) = linker.get(&mut *store, EXTISM_ENV_MODULE, "length_unsafe") {
catch_out_of_fuel!(
&store,
f.into_func()
.unwrap()
.call(&mut *store, &[Val::I64(offs as i64)], output)
.context("failed to get length of extism memory using length_unsafe")
)?;
} else {
anyhow::bail!("unable to locate an extism kernel function: length_unsafe",)
}
let len = output[0].unwrap_i64() as u64;
trace!(
plugin = self.id.to_string(),
"memory_length_unsafe({}) = {}",
offs,
len
);
Ok(len)
}
/// Access a plugin's variables
pub fn vars(&self) -> &std::collections::BTreeMap<String, Vec<u8>> {
&self.vars
@@ -339,51 +242,35 @@ impl CurrentPlugin {
manifest: extism_manifest::Manifest,
wasi: bool,
available_pages: Option<u32>,
allow_http_response_headers: bool,
id: uuid::Uuid,
) -> Result<Self, Error> {
let wasi = if wasi {
let auth = wasi_common::sync::ambient_authority();
let random = wasi_common::sync::random_ctx();
let clocks = wasi_common::sync::clocks_ctx();
let sched = wasi_common::sync::sched_ctx();
let table = wasi_common::Table::new();
let ctx = wasi_common::WasiCtx::new(random, clocks, sched, table);
let auth = wasmtime_wasi::ambient_authority();
let mut ctx = wasmtime_wasi::WasiCtxBuilder::new();
for (k, v) in manifest.config.iter() {
ctx.env(k, v)?;
}
if let Some(a) = &manifest.allowed_paths {
for (k, v) in a.iter() {
let readonly = k.starts_with("ro:");
let dir_path = if readonly { &k[3..] } else { k };
let dir = wasi_common::sync::dir::Dir::from_cap_std(
wasi_common::sync::Dir::open_ambient_dir(dir_path, auth)?,
);
let file: Box<dyn wasi_common::dir::WasiDir> = if readonly {
Box::new(readonly_dir::ReadOnlyDir::new(dir))
} else {
Box::new(dir)
};
ctx.push_preopened_dir(file, v)?;
let d = wasmtime_wasi::Dir::open_ambient_dir(k, auth)?;
ctx.preopened_dir(d, v)?;
}
}
// Enable WASI output, typically used for debugging purposes
if std::env::var("EXTISM_ENABLE_WASI_OUTPUT").is_ok() {
ctx.set_stderr(Box::new(wasi_common::sync::stdio::stderr()));
ctx.set_stdout(Box::new(wasi_common::sync::stdio::stdout()));
ctx.inherit_stdout().inherit_stderr();
}
Some(Wasi { ctx })
Some(Wasi { ctx: ctx.build() })
} else {
None
};
let memory_limiter = if let Some(pgs) = available_pages {
let n = pgs as usize * 65536;
Some(MemoryLimiter {
Some(crate::current_plugin::MemoryLimiter {
max_bytes: n,
bytes_left: n,
})
@@ -401,12 +288,6 @@ impl CurrentPlugin {
available_pages,
memory_limiter,
id,
start_time: std::time::Instant::now(),
http_headers: if allow_http_response_headers {
Some(BTreeMap::new())
} else {
None
},
})
}
@@ -451,12 +332,12 @@ impl CurrentPlugin {
/// Clear the current plugin error
pub fn clear_error(&mut self) {
trace!(plugin = self.id.to_string(), "CurrentPlugin::clear_error");
let (linker, store) = self.linker_and_store();
if let Some(f) = linker.get(&mut *store, EXTISM_ENV_MODULE, "error_set") {
let (linker, mut store) = self.linker_and_store();
if let Some(f) = linker.get(&mut store, EXTISM_ENV_MODULE, "error_set") {
let res = f
.into_func()
.unwrap()
.call(&mut *store, &[Val::I64(0)], &mut []);
.call(&mut store, &[Val::I64(0)], &mut []);
if let Err(e) = res {
error!(
plugin = self.id.to_string(),
@@ -477,22 +358,23 @@ impl CurrentPlugin {
offset: offs,
length,
});
s.ok()
match s {
Ok(s) => Some(s),
Err(_) => None,
}
}
#[doc(hidden)]
pub fn set_error(&mut self, s: impl AsRef<str>) -> Result<(u64, u64), Error> {
let s = s.as_ref();
debug!(plugin = self.id.to_string(), "set error: {:?}", s);
let handle = self.memory_new(s)?;
let (linker, store) = self.linker_and_store();
if let Some(f) = linker.get(&mut *store, EXTISM_ENV_MODULE, "error_set") {
catch_out_of_fuel!(
&store,
f.into_func()
.unwrap()
.call(&mut *store, &[Val::I64(handle.offset() as i64)], &mut [])
.context("failed to set extism error")
let handle = self.current_plugin_mut().memory_new(s)?;
let (linker, mut store) = self.linker_and_store();
if let Some(f) = linker.get(&mut store, EXTISM_ENV_MODULE, "error_set") {
f.into_func().unwrap().call(
&mut store,
&[Val::I64(handle.offset() as i64)],
&mut [],
)?;
Ok((handle.offset(), s.len() as u64))
} else {
@@ -501,13 +383,10 @@ impl CurrentPlugin {
}
pub(crate) fn get_error_position(&mut self) -> (u64, u64) {
let (linker, store) = self.linker_and_store();
let (linker, mut store) = self.linker_and_store();
let output = &mut [Val::I64(0)];
if let Some(f) = linker.get(&mut *store, EXTISM_ENV_MODULE, "error_get") {
if let Err(e) = catch_out_of_fuel!(
&store,
f.into_func().unwrap().call(&mut *store, &[], output)
) {
if let Some(f) = linker.get(&mut store, EXTISM_ENV_MODULE, "error_get") {
if let Err(e) = f.into_func().unwrap().call(&mut store, &[], output) {
error!(
plugin = self.id.to_string(),
"unable to call extism:host/env::error_get: {:?}", e
@@ -519,18 +398,6 @@ impl CurrentPlugin {
let length = self.memory_length(offs).unwrap_or_default();
(offs, length)
}
/// Returns the remaining time before a plugin will timeout, or
/// `None` if no timeout is configured in the manifest
pub fn time_remaining(&self) -> Option<std::time::Duration> {
if let Some(x) = &self.manifest.timeout_ms {
let elapsed = &self.start_time.elapsed().as_millis();
let ms_left = x.saturating_sub(*elapsed as u64);
return Some(std::time::Duration::from_millis(ms_left));
}
None
}
}
impl Internal for CurrentPlugin {
@@ -542,6 +409,14 @@ impl Internal for CurrentPlugin {
unsafe { &mut *self.store }
}
fn linker(&self) -> &Linker<CurrentPlugin> {
unsafe { &*self.linker }
}
fn linker_mut(&mut self) -> &mut Linker<CurrentPlugin> {
unsafe { &mut *self.linker }
}
fn linker_and_store(&mut self) -> (&mut Linker<CurrentPlugin>, &mut Store<CurrentPlugin>) {
unsafe { (&mut *self.linker, &mut *self.store) }
}

BIN
runtime/src/extism-runtime.wasm Normal file → Executable file

Binary file not shown.

View File

@@ -4,7 +4,6 @@ use wasmtime::Caller;
use crate::{error, trace, CurrentPlugin, Error};
/// An enumeration of all possible value types in WebAssembly.
/// cbindgen:prefix-with-name
#[derive(Debug, Clone, Hash, Eq, PartialEq)]
#[repr(C)]
pub enum ValType {
@@ -38,13 +37,8 @@ impl From<wasmtime::ValType> for ValType {
F32 => ValType::F32,
F64 => ValType::F64,
V128 => ValType::V128,
Ref(t) => {
if t.heap_type().is_func() {
ValType::FuncRef
} else {
ValType::ExternRef
}
}
FuncRef => ValType::FuncRef,
ExternRef => ValType::ExternRef,
}
}
}
@@ -58,8 +52,8 @@ impl From<ValType> for wasmtime::ValType {
F32 => wasmtime::ValType::F32,
F64 => wasmtime::ValType::F64,
V128 => wasmtime::ValType::V128,
FuncRef => wasmtime::ValType::FUNCREF,
ExternRef => wasmtime::ValType::EXTERNREF,
FuncRef => wasmtime::ValType::FuncRef,
ExternRef => wasmtime::ValType::ExternRef,
}
}
}
@@ -77,9 +71,7 @@ pub struct CPtr {
/// UserDataHandle is an untyped version of `UserData` that is stored inside `Function` to keep a live reference.
#[derive(Clone)]
pub(crate) enum UserDataHandle {
#[allow(dead_code)]
C(Arc<CPtr>),
#[allow(dead_code)]
Rust(Arc<std::sync::Mutex<dyn std::any::Any>>),
}
@@ -176,8 +168,8 @@ pub struct Function {
/// Module name
pub(crate) namespace: Option<String>,
pub(crate) params: Vec<ValType>,
pub(crate) results: Vec<ValType>,
/// Function type
pub(crate) ty: wasmtime::FuncType,
/// Function handle
pub(crate) f: Arc<FunctionInner>,
@@ -190,8 +182,8 @@ impl Function {
/// Create a new host function
pub fn new<T: 'static, F>(
name: impl Into<String>,
params: impl IntoIterator<Item = ValType>,
results: impl IntoIterator<Item = ValType>,
args: impl IntoIterator<Item = ValType>,
returns: impl IntoIterator<Item = ValType>,
user_data: UserData<T>,
f: F,
) -> Function
@@ -203,13 +195,13 @@ impl Function {
{
let data = user_data.clone();
let name = name.into();
let params = params.into_iter().collect();
let results = results.into_iter().collect();
trace!("Creating function {name}: params={params:?}, results={results:?}");
let args = args.into_iter().map(wasmtime::ValType::from);
let returns = returns.into_iter().map(wasmtime::ValType::from);
let ty = wasmtime::FuncType::new(args, returns);
trace!("Creating function {name}: type={ty:?}");
Function {
name,
params,
results,
ty,
f: Arc::new(
move |mut caller: Caller<_>, inp: &[Val], outp: &mut [Val]| {
let x = data.clone();
@@ -224,22 +216,6 @@ impl Function {
}
}
pub(crate) fn ty(&self, engine: &wasmtime::Engine) -> wasmtime::FuncType {
wasmtime::FuncType::new(
engine,
self.params
.iter()
.cloned()
.map(wasmtime::ValType::from)
.collect::<Vec<_>>(),
self.results
.iter()
.cloned()
.map(wasmtime::ValType::from)
.collect::<Vec<_>>(),
)
}
/// Host function name
pub fn name(&self) -> &str {
&self.name
@@ -263,14 +239,9 @@ impl Function {
self
}
/// Get param types
pub fn params(&self) -> &[ValType] {
&self.params
}
/// Get result types
pub fn results(&self) -> &[ValType] {
&self.results
/// Get function type
pub fn ty(&self) -> &wasmtime::FuncType {
&self.ty
}
}
@@ -279,13 +250,13 @@ impl Function {
/// For example, the following defines a host function named `add_newline` that takes a
/// string parameter and returns a string result:
/// ```rust
/// extism::host_fn!(add_newline(_user_data: (); a: String) -> String { Ok(a + "\n") });
/// extism::host_fn!(add_newline(_user_data: (), a: String) -> String { Ok(a + "\n") });
/// ```
/// A few things worth noting:
/// - The function always returns a `Result` that wraps the specified return type
/// - If a first parameter and type are passed (`_user_data` above) followed by a semicolon it will be
/// the name of the `UserData` parameter and can be used from inside the function
// definition.
/// the name of the `UserData` parameter and can be used from inside the function
// definition.
#[macro_export]
macro_rules! host_fn {
($pub:vis $name: ident ($($arg:ident : $argty:ty),*) $(-> $ret:ty)? $b:block) => {

View File

@@ -3,7 +3,7 @@ use crate::*;
/// WASI context
pub struct Wasi {
/// wasi
pub ctx: wasi_common::WasiCtx,
pub ctx: wasmtime_wasi::WasiCtx,
}
/// InternalExt provides a unified way of acessing `memory`, `store` and `internal` values
@@ -12,6 +12,10 @@ pub(crate) trait Internal {
fn store_mut(&mut self) -> &mut Store<CurrentPlugin>;
fn linker(&self) -> &Linker<CurrentPlugin>;
fn linker_mut(&mut self) -> &mut Linker<CurrentPlugin>;
fn linker_and_store(&mut self) -> (&mut Linker<CurrentPlugin>, &mut Store<CurrentPlugin>);
fn current_plugin(&self) -> &CurrentPlugin {

View File

@@ -1,23 +1,8 @@
// Makes proc-macros able to resolve `::extism` correctly
extern crate self as extism;
macro_rules! catch_out_of_fuel {
($store: expr, $x:expr) => {{
let y = $x;
if y.is_err() && $store.get_fuel().is_ok_and(|x| x == 0) {
Err(Error::msg("plugin ran out of fuel"))
} else {
y
}
}};
}
pub(crate) use extism_convert::*;
pub(crate) use std::collections::BTreeMap;
use std::str::FromStr;
pub(crate) use wasmtime::*;
#[doc(inline)]
pub use extism_convert as convert;
pub use anyhow::Error;
@@ -29,8 +14,6 @@ pub(crate) mod manifest;
pub(crate) mod pdk;
mod plugin;
mod plugin_builder;
mod pool;
mod readonly_dir;
mod timer;
/// Extism C API
@@ -40,11 +23,8 @@ pub use current_plugin::CurrentPlugin;
pub use extism_convert::{FromBytes, FromBytesOwned, ToBytes};
pub use extism_manifest::{Manifest, Wasm, WasmMetadata};
pub use function::{Function, UserData, Val, ValType, PTR};
pub use plugin::{
CancelHandle, CompiledPlugin, Plugin, WasmInput, EXTISM_ENV_MODULE, EXTISM_USER_MODULE,
};
pub use plugin::{CancelHandle, Plugin, WasmInput, EXTISM_ENV_MODULE, EXTISM_USER_MODULE};
pub use plugin_builder::{DebugOptions, PluginBuilder};
pub use pool::{Pool, PoolBuilder, PoolPlugin};
pub(crate) use internal::{Internal, Wasi};
pub(crate) use timer::{Timer, TimerAction};
@@ -93,16 +73,11 @@ pub fn set_log_callback<F: 'static + Clone + Fn(&str)>(
filter: impl AsRef<str>,
) -> Result<(), Error> {
let filter = filter.as_ref();
let is_level = tracing::Level::from_str(filter).is_ok();
let cfg = tracing_subscriber::FmtSubscriber::builder().with_env_filter({
let x = tracing_subscriber::EnvFilter::builder()
.with_default_directive(tracing::Level::ERROR.into());
if is_level {
x.parse_lossy(format!("extism={filter}"))
} else {
x.parse_lossy(filter)
}
});
let cfg = tracing_subscriber::FmtSubscriber::builder().with_env_filter(
tracing_subscriber::EnvFilter::builder()
.with_default_directive(tracing::Level::ERROR.into())
.parse_lossy(filter),
);
let w = LogFunction { func };
cfg.with_ansi(false)
.with_writer(move || w.clone())

View File

@@ -10,7 +10,7 @@ use crate::*;
fn hex(data: &[u8]) -> String {
let mut s = String::new();
for &byte in data {
write!(&mut s, "{byte:02x}").unwrap();
write!(&mut s, "{:02x}", byte).unwrap();
}
s
}
@@ -47,13 +47,9 @@ fn to_module(engine: &Engine, wasm: &extism_manifest::Wasm) -> Result<(String, M
let name = meta.name.as_deref().unwrap_or(MAIN_KEY).to_string();
// Load file
let buf = std::fs::read(path).map_err(|err| {
Error::msg(format!(
"Unable to load Wasm file \"{}\": {}",
path.display(),
err.kind()
))
})?;
let mut buf = Vec::new();
let mut file = std::fs::File::open(path)?;
file.read_to_end(&mut buf)?;
check_hash(&meta.hash, &buf)?;
Ok((name, Module::new(engine, buf)?))
@@ -86,16 +82,14 @@ fn to_module(engine: &Engine, wasm: &extism_manifest::Wasm) -> Result<(String, M
#[cfg(feature = "register-http")]
{
// Setup request
let mut req = ureq::http::request::Builder::new()
.method(method.as_deref().unwrap_or("GET").to_uppercase().as_str())
.uri(url);
let mut req = ureq::request(method.as_deref().unwrap_or("GET"), url);
for (k, v) in headers.iter() {
req = req.header(k, v);
req = req.set(k, v);
}
// Fetch WASM code
let mut r = ureq::run(req.body(())?)?.into_body().into_reader();
let mut r = req.call()?.into_reader();
let mut data = Vec::new();
r.read_to_end(&mut data)?;
@@ -123,17 +117,10 @@ pub(crate) fn load(
match input {
WasmInput::Data(data) => {
let has_magic = data.len() >= 4 && data[0..4] == WASM_MAGIC;
let s = std::str::from_utf8(&data);
let is_wat = s.is_ok_and(|s| {
let s = s.trim_start();
let starts_with_module = s.len() > 2
&& data[0] == b'(' // First character is `(`
&& s[1..].trim_start().starts_with("module"); // Then `module` (after any whitespace)
starts_with_module || s.starts_with(";;") || s.starts_with("(;")
});
let is_wat = data.starts_with(b"(module") || data.starts_with(b";;");
if !has_magic && !is_wat {
trace!("Loading manifest");
if let Ok(s) = s {
if let Ok(s) = std::str::from_utf8(&data) {
let t = if let Ok(t) = toml::from_str::<extism_manifest::Manifest>(s) {
trace!("Manifest is TOML");
modules(engine, &t, &mut mods)?;

View File

@@ -20,8 +20,6 @@ macro_rules! args {
/// Get a configuration value
/// Params: i64 (offset)
/// Returns: i64 (offset)
/// **Note**: this function takes ownership of the handle passed in
/// the caller should not `free` this value
pub(crate) fn config_get(
mut caller: Caller<CurrentPlugin>,
input: &[Val],
@@ -40,7 +38,6 @@ pub(crate) fn config_get(
};
let val = data.manifest.config.get(key);
let ptr = val.map(|x| (x.len(), x.as_ptr()));
data.memory_free(handle)?;
let mem = match ptr {
Some((len, ptr)) => {
let bytes = unsafe { std::slice::from_raw_parts(ptr, len) };
@@ -58,9 +55,6 @@ pub(crate) fn config_get(
/// Get a variable
/// Params: i64 (offset)
/// Returns: i64 (offset)
/// **Note**: this function takes ownership of the handle passed in
/// the caller should not `free` this value, but the return value
/// will need to be freed
pub(crate) fn var_get(
mut caller: Caller<CurrentPlugin>,
input: &[Val],
@@ -79,8 +73,6 @@ pub(crate) fn var_get(
};
let val = data.vars.get(key);
let ptr = val.map(|x| (x.len(), x.as_ptr()));
data.memory_free(handle)?;
let mem = match ptr {
Some((len, ptr)) => {
let bytes = unsafe { std::slice::from_raw_parts(ptr, len) };
@@ -98,8 +90,6 @@ pub(crate) fn var_get(
/// Set a variable, if the value offset is 0 then the provided key will be removed
/// Params: i64 (key offset), i64 (value offset)
/// Returns: none
/// **Note**: this function takes ownership of the handles passed in
/// the caller should not `free` these values
pub(crate) fn var_set(
mut caller: Caller<CurrentPlugin>,
input: &[Val],
@@ -107,19 +97,25 @@ pub(crate) fn var_set(
) -> Result<(), Error> {
let data: &mut CurrentPlugin = caller.data_mut();
if data.manifest.memory.max_var_bytes.is_some_and(|x| x == 0) {
anyhow::bail!("Vars are disabled by this host")
let mut size = 0;
for v in data.vars.values() {
size += v.len();
}
let voffset = args!(input, 1, i64) as u64;
let key_offs = args!(input, 0, i64) as u64;
let key_handle = match data.memory_handle(key_offs) {
Some(h) => h,
None => anyhow::bail!("invalid handle offset for var key: {key_offs}"),
};
// If the store is larger than 100MB then stop adding things
if size > 1024 * 1024 * 100 && voffset != 0 {
return Err(Error::msg("Variable store is full"));
}
let key_offs = args!(input, 0, i64) as u64;
let key = {
let key = data.memory_str(key_handle)?;
let handle = match data.memory_handle(key_offs) {
Some(h) => h,
None => anyhow::bail!("invalid handle offset for var key: {key_offs}"),
};
let key = data.memory_str(handle)?;
let key_len = key.len();
let key_ptr = key.as_ptr();
unsafe { std::str::from_utf8_unchecked(std::slice::from_raw_parts(key_ptr, key_len)) }
@@ -128,7 +124,6 @@ pub(crate) fn var_set(
// Remove if the value offset is 0
if voffset == 0 {
data.vars.remove(key);
data.memory_free(key_handle)?;
return Ok(());
}
@@ -137,27 +132,8 @@ pub(crate) fn var_set(
None => anyhow::bail!("invalid handle offset for var value: {voffset}"),
};
let mut size = std::mem::size_of::<String>()
+ std::mem::size_of::<Vec<u8>>()
+ key.len()
+ handle.length as usize;
for (k, v) in data.vars.iter() {
size += k.len();
size += v.len();
size += std::mem::size_of::<String>() + std::mem::size_of::<Vec<u8>>();
}
// If the store is larger than the configured size, or 1mb by default, then stop adding things
if size > data.manifest.memory.max_var_bytes.unwrap_or(1024 * 1024) as usize && voffset != 0 {
return Err(Error::msg("Variable store is full"));
}
let value = data.memory_bytes(handle)?.to_vec();
data.memory_free(handle)?;
data.memory_free(key_handle)?;
// Insert the value from memory into the `vars` map
data.vars.insert(key.to_string(), value);
@@ -167,9 +143,6 @@ pub(crate) fn var_set(
/// Make an HTTP request
/// Params: i64 (offset to JSON encoded HttpRequest), i64 (offset to body or 0)
/// Returns: i64 (offset)
/// **Note**: this function takes ownership of the handles passed in
/// the caller should not `free` these values, the result will need to
/// be freed.
pub(crate) fn http_request(
#[allow(unused_mut)] mut caller: Caller<CurrentPlugin>,
input: &[Val],
@@ -183,7 +156,6 @@ pub(crate) fn http_request(
Some(h) => h,
None => anyhow::bail!("http_request input is invalid: {http_req_offset}"),
};
data.memory_free(handle)?;
let req: extism_manifest::HttpRequest = serde_json::from_slice(data.memory_bytes(handle)?)?;
output[0] = Val::I64(0);
anyhow::bail!(
@@ -194,16 +166,12 @@ pub(crate) fn http_request(
#[cfg(feature = "http")]
{
data.http_headers.iter_mut().for_each(|x| x.clear());
data.http_status = 0;
use std::io::Read;
let handle = match data.memory_handle(http_req_offset) {
Some(h) => h,
None => anyhow::bail!("invalid handle offset for http request: {http_req_offset}"),
};
let req: extism_manifest::HttpRequest = serde_json::from_slice(data.memory_bytes(handle)?)?;
data.memory_free(handle)?;
let body_offset = args!(input, 1, i64) as u64;
@@ -233,22 +201,12 @@ pub(crate) fn http_request(
)));
}
let mut r = ureq::http::request::Builder::new()
.method(
req.method
.as_deref()
.unwrap_or("GET")
.to_uppercase()
.as_str(),
)
.uri(&req.url);
let mut r = ureq::request(req.method.as_deref().unwrap_or("GET"), &req.url);
for (k, v) in req.headers.iter() {
r = r.header(k, v);
r = r.set(k, v);
}
// Set HTTP timeout to respect the manifest timeout
let timeout = data.time_remaining();
let res = if body_offset > 0 {
let handle = match data.memory_handle(body_offset) {
Some(h) => h,
@@ -257,63 +215,31 @@ pub(crate) fn http_request(
}
};
let buf: &[u8] = data.memory_bytes(handle)?;
let agent = ureq::agent();
let config = agent.configure_request(r.body(buf)?);
let req = config.timeout_global(timeout).build();
ureq::run(req)
r.send_bytes(buf)
} else {
let agent = ureq::agent();
let config = agent.configure_request(r.body(())?);
let req = config.timeout_global(timeout).build();
ureq::run(req)
r.call()
};
if let Some(handle) = data.memory_handle(body_offset) {
data.memory_free(handle)?;
}
let reader = match res {
Ok(res) => {
if let Some(headers) = &mut data.http_headers {
for (name, h) in res.headers() {
if let Ok(h) = h.to_str() {
headers.insert(name.as_str().to_string(), h.to_string());
}
}
}
data.http_status = res.status().as_u16();
Some(res.into_body().into_reader())
data.http_status = res.status();
Some(res.into_reader())
}
Err(e) => {
// Catch timeout and return
if let Some(d) = data.time_remaining() {
if matches!(e, ureq::Error::Timeout(_)) && d.as_nanos() == 0 {
anyhow::bail!("timeout");
}
}
let msg = e.to_string();
if let ureq::Error::StatusCode(res) = e {
data.http_status = res;
None
if let Some(res) = e.into_response() {
data.http_status = res.status();
Some(res.into_reader())
} else {
return Err(Error::msg(msg));
None
}
}
};
if let Some(reader) = reader {
let mut buf = Vec::new();
let max = if let Some(max) = &data.manifest.memory.max_http_response_bytes {
reader.take(*max + 1).read_to_end(&mut buf)?;
*max
} else {
reader.take(1024 * 1024 * 50 + 1).read_to_end(&mut buf)?;
1024 * 1024 * 50
};
if buf.len() > max as usize {
anyhow::bail!("HTTP response exceeds the configured maximum number of bytes: {max}")
}
reader
.take(1024 * 1024 * 50) // TODO: make this limit configurable
.read_to_end(&mut buf)?;
let mem = data.memory_new(&buf)?;
output[0] = Val::I64(mem.offset() as i64);
@@ -338,24 +264,6 @@ pub(crate) fn http_status_code(
Ok(())
}
/// Get the HTTP response headers from the last HTTP request
/// Params: none
/// Returns: i64 (offset)
pub(crate) fn http_headers(
mut caller: Caller<CurrentPlugin>,
_input: &[Val],
output: &mut [Val],
) -> Result<(), Error> {
let data: &mut CurrentPlugin = caller.data_mut();
if let Some(h) = &data.http_headers {
let headers = serde_json::to_string(h)?;
data.memory_set_val(&mut output[0], headers)?;
} else {
output[0] = Val::I64(0);
}
Ok(())
}
pub fn log(
level: tracing::Level,
mut caller: Caller<CurrentPlugin>,
@@ -363,22 +271,13 @@ pub fn log(
_output: &mut [Val],
) -> Result<(), Error> {
let data: &mut CurrentPlugin = caller.data_mut();
let offset = args!(input, 0, i64) as u64;
// Check if the current log level should be logged
let global_log_level = tracing::level_filters::LevelFilter::current();
if global_log_level == tracing::level_filters::LevelFilter::OFF || level > global_log_level {
if let Some(handle) = data.memory_handle(offset) {
data.memory_free(handle)?;
}
return Ok(());
}
let handle = match data.memory_handle(offset) {
Some(h) => h,
None => anyhow::bail!("invalid handle offset for log message: {offset}"),
};
let id = data.id.to_string();
let buf = data.memory_str(handle);
@@ -402,16 +301,12 @@ pub fn log(
},
Err(_) => tracing::error!(plugin = id, "unable to log message: {:?}", buf),
}
data.memory_free(handle)?;
Ok(())
}
/// Write to logs (warning)
/// Params: i64 (offset)
/// Returns: none
/// **Note**: this function takes ownership of the handle passed in
/// the caller should not `free` this value
pub(crate) fn log_warn(
caller: Caller<CurrentPlugin>,
input: &[Val],
@@ -423,8 +318,6 @@ pub(crate) fn log_warn(
/// Write to logs (info)
/// Params: i64 (offset)
/// Returns: none
/// **Note**: this function takes ownership of the handle passed in
/// the caller should not `free` this value
pub(crate) fn log_info(
caller: Caller<CurrentPlugin>,
input: &[Val],
@@ -436,8 +329,6 @@ pub(crate) fn log_info(
/// Write to logs (debug)
/// Params: i64 (offset)
/// Returns: none
/// **Note**: this function takes ownership of the handle passed in
/// the caller should not `free` this value
pub(crate) fn log_debug(
caller: Caller<CurrentPlugin>,
input: &[Val],
@@ -449,8 +340,6 @@ pub(crate) fn log_debug(
/// Write to logs (error)
/// Params: i64 (offset)
/// Returns: none
/// **Note**: this function takes ownership of the handle passed in
/// the caller should not `free` this value
pub(crate) fn log_error(
caller: Caller<CurrentPlugin>,
input: &[Val],
@@ -458,46 +347,3 @@ pub(crate) fn log_error(
) -> Result<(), Error> {
log(tracing::Level::ERROR, caller, input, _output)
}
/// Write to logs (trace)
/// Params: i64 (offset)
/// Returns: none
/// **Note**: this function takes ownership of the handle passed in
/// the caller should not `free` this value
pub(crate) fn log_trace(
caller: Caller<CurrentPlugin>,
input: &[Val],
_output: &mut [Val],
) -> Result<(), Error> {
log(tracing::Level::TRACE, caller, input, _output)
}
/// Get the log level
/// Params: none
/// Returns: i32 (log level)
pub(crate) fn get_log_level(
mut _caller: Caller<CurrentPlugin>,
_input: &[Val],
output: &mut [Val],
) -> Result<(), Error> {
let level = tracing::level_filters::LevelFilter::current();
if level == tracing::level_filters::LevelFilter::OFF {
output[0] = Val::I32(i32::MAX)
} else {
output[0] = Val::I32(log_level_to_int(
level.into_level().unwrap_or(tracing::Level::ERROR),
));
}
Ok(())
}
/// Convert log level to integer
pub(crate) const fn log_level_to_int(level: tracing::Level) -> i32 {
match level {
tracing::Level::TRACE => 0,
tracing::Level::DEBUG => 1,
tracing::Level::INFO => 2,
tracing::Level::WARN => 3,
tracing::Level::ERROR => 4,
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -33,21 +33,12 @@ impl Default for DebugOptions {
}
/// PluginBuilder is used to configure and create `Plugin` instances
#[derive(Clone)]
pub struct PluginBuilder<'a> {
pub(crate) source: WasmInput<'a>,
pub(crate) config: Option<wasmtime::Config>,
pub(crate) options: PluginBuilderOptions,
}
#[derive(Clone)]
pub(crate) struct PluginBuilderOptions {
pub(crate) wasi: bool,
pub(crate) functions: Vec<Function>,
pub(crate) debug_options: DebugOptions,
pub(crate) cache_config: Option<Option<PathBuf>>,
pub(crate) fuel: Option<u64>,
pub(crate) http_response_headers: bool,
source: WasmInput<'a>,
wasi: bool,
functions: Vec<Function>,
debug_options: DebugOptions,
cache_config: Option<Option<PathBuf>>,
}
impl<'a> PluginBuilder<'a> {
@@ -55,21 +46,16 @@ impl<'a> PluginBuilder<'a> {
pub fn new(plugin: impl Into<WasmInput<'a>>) -> Self {
PluginBuilder {
source: plugin.into(),
config: None,
options: PluginBuilderOptions {
wasi: false,
functions: vec![],
debug_options: DebugOptions::default(),
cache_config: None,
fuel: None,
http_response_headers: false,
},
wasi: false,
functions: vec![],
debug_options: DebugOptions::default(),
cache_config: None,
}
}
/// Enables WASI if the argument is set to `true`
pub fn with_wasi(mut self, wasi: bool) -> Self {
self.options.wasi = wasi;
self.wasi = wasi;
self
}
@@ -88,8 +74,7 @@ impl<'a> PluginBuilder<'a> {
+ Sync
+ Send,
{
self.options
.functions
self.functions
.push(Function::new(name, args, returns, user_data, f));
self
}
@@ -110,97 +95,67 @@ impl<'a> PluginBuilder<'a> {
+ Sync
+ Send,
{
self.options
.functions
self.functions
.push(Function::new(name, args, returns, user_data, f).with_namespace(namespace));
self
}
/// Add multiple host functions
pub fn with_functions(mut self, f: impl IntoIterator<Item = Function>) -> Self {
self.options.functions.extend(f);
self.functions.extend(f);
self
}
/// Set profiling strategy
pub fn with_profiling_strategy(mut self, p: wasmtime::ProfilingStrategy) -> Self {
self.options.debug_options.profiling_strategy = p;
self.debug_options.profiling_strategy = p;
self
}
/// Enable Wasmtime coredump on trap
pub fn with_coredump(mut self, path: impl Into<std::path::PathBuf>) -> Self {
self.options.debug_options.coredump = Some(path.into());
self.debug_options.coredump = Some(path.into());
self
}
/// Enable Extism memory dump when plugin calls return an error
pub fn with_memdump(mut self, path: impl Into<std::path::PathBuf>) -> Self {
self.options.debug_options.memdump = Some(path.into());
self.debug_options.memdump = Some(path.into());
self
}
/// Compile with debug info
pub fn with_debug_info(mut self) -> Self {
self.options.debug_options.debug_info = true;
self.debug_options.debug_info = true;
self
}
/// Configure debug options
pub fn with_debug_options(mut self, options: DebugOptions) -> Self {
self.options.debug_options = options;
self.debug_options = options;
self
}
/// Set wasmtime compilation cache config path
pub fn with_cache_config(mut self, dir: impl Into<PathBuf>) -> Self {
self.options.cache_config = Some(Some(dir.into()));
self.cache_config = Some(Some(dir.into()));
self
}
/// Turn wasmtime compilation caching off
pub fn with_cache_disabled(mut self) -> Self {
self.options.cache_config = Some(None);
self
}
/// Limit the number of instructions that can be executed
pub fn with_fuel_limit(mut self, fuel: u64) -> Self {
self.options.fuel = Some(fuel);
self
}
/// Configure an initial wasmtime config to be passed to the plugin
///
/// **Warning**: some values might be overwritten by the Extism runtime. In particular:
/// - async_support
/// - epoch_interruption
/// - debug_info
/// - coredump_on_trap
/// - profiler
/// - wasm_tail_call
/// - wasm_function_references
/// - wasm_gc
///
/// See the implementation details of [PluginBuilder::build] and [Plugin::build_new] to verify which values are overwritten.
pub fn with_wasmtime_config(mut self, config: wasmtime::Config) -> Self {
self.config = Some(config);
self
}
/// Enables `http_response_headers`, which allows for plugins to access response headers when using `extism:host/env::http_request`
pub fn with_http_response_headers(mut self, allow: bool) -> Self {
self.options.http_response_headers = allow;
self.cache_config = Some(None);
self
}
/// Generate a new plugin with the configured settings
pub fn build(self) -> Result<Plugin, Error> {
Plugin::new_from_compiled(&CompiledPlugin::new(self)?)
}
/// Build new `CompiledPlugin`
pub fn compile(self) -> Result<CompiledPlugin, Error> {
CompiledPlugin::new(self)
Plugin::build_new(
self.source,
self.functions,
self.wasi,
self.debug_options,
self.cache_config,
)
}
}

View File

@@ -1,174 +0,0 @@
use crate::{Error, FromBytesOwned, Plugin, ToBytes};
// `PoolBuilder` is used to configure and create `Pool`s
#[derive(Debug, Clone)]
pub struct PoolBuilder {
/// Max number of concurrent instances for a plugin - by default this is set to
/// the output of `std::thread::available_parallelism`
pub max_instances: usize,
}
impl PoolBuilder {
/// Create a `PoolBuilder` with default values
pub fn new() -> Self {
Self::default()
}
/// Set the max number of parallel instances
pub fn with_max_instances(mut self, n: usize) -> Self {
self.max_instances = n;
self
}
/// Create a new `Pool` with the given configuration
pub fn build<F: 'static + Fn() -> Result<Plugin, Error>>(self, source: F) -> Pool {
Pool::new_from_builder(source, self)
}
}
impl Default for PoolBuilder {
fn default() -> Self {
PoolBuilder {
max_instances: std::thread::available_parallelism()
.expect("available parallelism")
.into(),
}
}
}
/// `PoolPlugin` is used by the pool to track the number of live instances of a particular plugin
#[derive(Clone, Debug)]
pub struct PoolPlugin(std::rc::Rc<std::cell::RefCell<Plugin>>);
impl PoolPlugin {
fn new(plugin: Plugin) -> Self {
Self(std::rc::Rc::new(std::cell::RefCell::new(plugin)))
}
/// Access the underlying plugin
pub fn plugin(&self) -> std::cell::RefMut<Plugin> {
self.0.borrow_mut()
}
/// Helper to call a plugin function on the underlying plugin
pub fn call<'a, Input: ToBytes<'a>, Output: FromBytesOwned>(
&self,
name: impl AsRef<str>,
input: Input,
) -> Result<Output, Error> {
self.plugin().call(name.as_ref(), input)
}
/// Helper to get the underlying plugin's ID
pub fn id(&self) -> uuid::Uuid {
self.plugin().id
}
}
type PluginSource = dyn Fn() -> Result<Plugin, Error>;
struct PoolInner {
plugin_source: Box<PluginSource>,
instances: Vec<PoolPlugin>,
}
unsafe impl Send for PoolInner {}
unsafe impl Sync for PoolInner {}
/// `Pool` manages threadsafe access to a limited number of instances of multiple plugins
#[derive(Clone)]
pub struct Pool {
config: PoolBuilder,
inner: std::sync::Arc<std::sync::Mutex<PoolInner>>,
}
unsafe impl Send for Pool {}
unsafe impl Sync for Pool {}
impl Pool {
/// Create a new pool with the default configuration
pub fn new<F: 'static + Fn() -> Result<Plugin, Error>>(source: F) -> Self {
Pool {
config: Default::default(),
inner: std::sync::Arc::new(std::sync::Mutex::new(PoolInner {
plugin_source: Box::new(source),
instances: Default::default(),
})),
}
}
/// Create a new pool configured using a `PoolBuilder`
pub fn new_from_builder<F: 'static + Fn() -> Result<Plugin, Error>>(
source: F,
builder: PoolBuilder,
) -> Self {
Pool {
config: builder,
inner: std::sync::Arc::new(std::sync::Mutex::new(PoolInner {
plugin_source: Box::new(source),
instances: Default::default(),
})),
}
}
fn find_available(&self) -> Result<Option<PoolPlugin>, Error> {
let pool = self.inner.lock().unwrap();
for instance in pool.instances.iter() {
if std::rc::Rc::strong_count(&instance.0) == 1 {
return Ok(Some(instance.clone()));
}
}
Ok(None)
}
/// Get the number of live instances for a plugin
pub fn count(&self) -> usize {
self.inner.lock().unwrap().instances.len()
}
/// Get access to a plugin, this will create a new instance if needed (and allowed by the specified
/// max_instances). `Ok(None)` is returned if the timeout is reached before an available plugin could be
/// acquired
pub fn get(&self, timeout: std::time::Duration) -> Result<Option<PoolPlugin>, Error> {
let start = std::time::Instant::now();
let max = self.config.max_instances;
if let Some(avail) = self.find_available()? {
return Ok(Some(avail));
}
{
let mut pool = self.inner.lock().unwrap();
if pool.instances.len() < max {
let plugin = (*pool.plugin_source)()?;
let instance = PoolPlugin::new(plugin);
pool.instances.push(instance);
return Ok(Some(pool.instances.last().unwrap().clone()));
}
}
loop {
if let Ok(Some(x)) = self.find_available() {
return Ok(Some(x));
}
if std::time::Instant::now() - start > timeout {
return Ok(None);
}
std::thread::sleep(std::time::Duration::from_millis(100));
}
}
/// Access a plugin in a callback function. This calls `Pool::get` then the provided
/// callback. `Ok(None)` is returned if the timeout is reached before an available
/// plugin could be acquired
pub fn with_plugin<T>(
&self,
timeout: std::time::Duration,
f: impl FnOnce(&mut Plugin) -> Result<T, Error>,
) -> Result<Option<T>, Error> {
if let Some(plugin) = self.get(timeout)? {
return f(&mut plugin.plugin()).map(Some);
}
Ok(None)
}
}

View File

@@ -1,109 +0,0 @@
use crate::*;
use wasi_common::{Error, ErrorExt};
pub struct ReadOnlyDir<D: wasi_common::WasiDir> {
inner: std::sync::Arc<D>,
}
impl<D: wasi_common::WasiDir> ReadOnlyDir<D> {
pub fn new(inner: D) -> Self {
ReadOnlyDir {
inner: std::sync::Arc::new(inner),
}
}
}
#[wiggle::async_trait]
impl<D: wasi_common::WasiDir> wasi_common::WasiDir for ReadOnlyDir<D> {
fn as_any(&self) -> &dyn std::any::Any {
self.inner.as_any()
}
async fn open_file(
&self,
symlink_follow: bool,
path: &str,
oflags: wasi_common::file::OFlags,
read: bool,
write: bool,
fdflags: wasi_common::file::FdFlags,
) -> Result<wasi_common::dir::OpenResult, Error> {
if write {
return Err(Error::not_supported());
}
self.inner
.open_file(symlink_follow, path, oflags, read, false, fdflags)
.await
}
async fn create_dir(&self, _path: &str) -> Result<(), Error> {
Err(Error::not_supported())
}
async fn readdir(
&self,
cursor: wasi_common::dir::ReaddirCursor,
) -> Result<
Box<dyn Iterator<Item = Result<wasi_common::dir::ReaddirEntity, Error>> + Send>,
Error,
> {
self.inner.readdir(cursor).await
}
async fn symlink(&self, _old_path: &str, _new_path: &str) -> Result<(), Error> {
Err(Error::not_supported())
}
async fn remove_dir(&self, _path: &str) -> Result<(), Error> {
Err(Error::not_supported())
}
async fn unlink_file(&self, _path: &str) -> Result<(), Error> {
Err(Error::not_supported())
}
async fn read_link(&self, path: &str) -> Result<std::path::PathBuf, Error> {
self.inner.read_link(path).await
}
async fn get_filestat(&self) -> Result<wasi_common::file::Filestat, Error> {
self.inner.get_filestat().await
}
async fn get_path_filestat(
&self,
path: &str,
follow_symlinks: bool,
) -> Result<wasi_common::file::Filestat, Error> {
self.inner.get_path_filestat(path, follow_symlinks).await
}
async fn rename(
&self,
_path: &str,
_dest_dir: &dyn wasi_common::WasiDir,
_dest_path: &str,
) -> Result<(), Error> {
Err(wasi_common::Error::not_supported())
}
async fn hard_link(
&self,
_path: &str,
_target_dir: &dyn wasi_common::WasiDir,
_target_path: &str,
) -> Result<(), Error> {
Err(wasi_common::Error::not_supported())
}
async fn set_times(
&self,
_path: &str,
_atime: std::option::Option<wasi_common::SystemTimeSpec>,
_mtime: std::option::Option<wasi_common::SystemTimeSpec>,
_follow_symlinks: bool,
) -> Result<(), Error> {
Err(wasi_common::Error::not_supported())
}
}

View File

@@ -1,6 +1,6 @@
#![allow(clippy::missing_safety_doc)]
use std::{os::raw::c_char, ptr::null_mut};
use std::os::raw::c_char;
use crate::*;
@@ -11,12 +11,6 @@ pub struct ExtismFunction(std::cell::Cell<Option<Function>>);
/// The return code used to specify a successful plugin call
pub static EXTISM_SUCCESS: i32 = 0;
fn make_error_msg(s: String) -> Vec<u8> {
let mut s = s.into_bytes();
s.push(0);
s
}
/// A union type for host function argument/return values
#[repr(C)]
pub union ValUnion {
@@ -44,36 +38,33 @@ pub type ExtismFunctionType = extern "C" fn(
data: *mut std::ffi::c_void,
);
/// Log drain callback
pub type ExtismLogDrainFunctionType = extern "C" fn(data: *const std::ffi::c_char, size: Size);
impl ExtismVal {
fn from_val(value: &wasmtime::Val, ctx: impl AsContext) -> Result<Self, Error> {
match value.ty(ctx)? {
wasmtime::ValType::I32 => Ok(ExtismVal {
impl From<&wasmtime::Val> for ExtismVal {
fn from(value: &wasmtime::Val) -> Self {
match value.ty() {
wasmtime::ValType::I32 => ExtismVal {
t: ValType::I32,
v: ValUnion {
i32: value.unwrap_i32(),
},
}),
wasmtime::ValType::I64 => Ok(ExtismVal {
},
wasmtime::ValType::I64 => ExtismVal {
t: ValType::I64,
v: ValUnion {
i64: value.unwrap_i64(),
},
}),
wasmtime::ValType::F32 => Ok(ExtismVal {
},
wasmtime::ValType::F32 => ExtismVal {
t: ValType::F32,
v: ValUnion {
f32: value.unwrap_f32(),
},
}),
wasmtime::ValType::F64 => Ok(ExtismVal {
},
wasmtime::ValType::F64 => ExtismVal {
t: ValType::F64,
v: ValUnion {
f64: value.unwrap_f64(),
},
}),
},
t => todo!("{}", t),
}
}
@@ -90,24 +81,6 @@ pub unsafe extern "C" fn extism_plugin_id(plugin: *mut Plugin) -> *const u8 {
plugin.id.as_bytes().as_ptr()
}
/// Get the current plugin's associated host context data. Returns null if call was made without
/// host context.
#[no_mangle]
pub unsafe extern "C" fn extism_current_plugin_host_context(
plugin: *mut CurrentPlugin,
) -> *mut std::ffi::c_void {
if plugin.is_null() {
return std::ptr::null_mut();
}
let plugin = &mut *plugin;
if let Ok(CVoidContainer(ptr)) = plugin.host_context::<CVoidContainer>() {
*ptr
} else {
std::ptr::null_mut()
}
}
/// Returns a pointer to the memory of the currently running plugin
/// NOTE: this should only be called from host functions.
#[no_mangle]
@@ -180,7 +153,7 @@ pub unsafe extern "C" fn extism_current_plugin_memory_free(
/// - `n_outputs`: number of return types
/// - `func`: the function to call
/// - `user_data`: a pointer that will be passed to the function when it's called
/// this value should live as long as the function exists
/// this value should live as long as the function exists
/// - `free_user_data`: a callback to release the `user_data` value when the resulting
/// `ExtismFunction` is freed.
///
@@ -224,11 +197,7 @@ pub unsafe extern "C" fn extism_function_new(
output_types.clone(),
user_data,
move |plugin, inputs, outputs, user_data| {
let store = &*plugin.store;
let inputs: Vec<_> = inputs
.iter()
.map(|x| ExtismVal::from_val(x, store).unwrap())
.collect();
let inputs: Vec<_> = inputs.iter().map(ExtismVal::from).collect();
let mut output_tmp: Vec<_> = output_types
.iter()
.map(|t| ExtismVal {
@@ -237,28 +206,12 @@ pub unsafe extern "C" fn extism_function_new(
})
.collect();
// We cannot simply "get" the Vec's storage pointer because
// the underlying storage might be invalid when the Vec is empty.
// In that case, we return (null, 0).
let (inputs_ptr, inputs_len) = if inputs.is_empty() {
(core::ptr::null(), 0 as Size)
} else {
(inputs.as_ptr(), inputs.len() as Size)
};
let (output_ptr, output_len) = if output_tmp.is_empty() {
(null_mut(), 0 as Size)
} else {
(output_tmp.as_mut_ptr(), output_tmp.len() as Size)
};
func(
plugin,
inputs_ptr,
inputs_len,
output_ptr,
output_len,
inputs.as_ptr(),
inputs.len() as Size,
output_tmp.as_mut_ptr(),
output_tmp.len() as Size,
user_data.as_ptr(),
);
@@ -266,8 +219,8 @@ pub unsafe extern "C" fn extism_function_new(
match tmp.t {
ValType::I32 => *out = Val::I32(tmp.v.i32),
ValType::I64 => *out = Val::I64(tmp.v.i64),
ValType::F32 => *out = Val::F32(tmp.v.f32.to_bits()),
ValType::F64 => *out = Val::F64(tmp.v.f64.to_bits()),
ValType::F32 => *out = Val::F32(tmp.v.f32 as u32),
ValType::F64 => *out = Val::F64(tmp.v.f64 as u64),
_ => todo!(),
}
}
@@ -302,79 +255,6 @@ pub unsafe extern "C" fn extism_function_set_namespace(
}
}
/// Pre-compile an Extism plugin
#[no_mangle]
pub unsafe extern "C" fn extism_compiled_plugin_new(
wasm: *const u8,
wasm_size: Size,
functions: *mut *const ExtismFunction,
n_functions: Size,
with_wasi: bool,
errmsg: *mut *mut std::ffi::c_char,
) -> *mut CompiledPlugin {
trace!("Call to extism_plugin_new with wasm pointer {:?}", wasm);
let data = std::slice::from_raw_parts(wasm, wasm_size as usize);
let mut builder = PluginBuilder::new(data).with_wasi(with_wasi);
if !functions.is_null() {
let funcs = (0..n_functions)
.map(|i| unsafe { *functions.add(i as usize) })
.map(|ptr| {
if ptr.is_null() {
return Err("Cannot pass null pointer");
}
let ExtismFunction(func) = &*ptr;
let Some(func) = func.take() else {
return Err("Function cannot be registered with multiple different Plugins");
};
Ok(func)
})
.collect::<Result<Vec<_>, _>>()
.unwrap_or_else(|e| {
if !errmsg.is_null() {
let e = std::ffi::CString::new(e.to_string()).unwrap();
*errmsg = e.into_raw();
}
Vec::new()
});
if funcs.len() != n_functions as usize {
return std::ptr::null_mut();
}
builder = builder.with_functions(funcs);
}
CompiledPlugin::new(builder)
.map(|v| Box::into_raw(Box::new(v)))
.unwrap_or_else(|e| {
if !errmsg.is_null() {
let e = std::ffi::CString::new(format!(
"Unable to compile Extism plugin: {}",
e.root_cause(),
))
.unwrap();
*errmsg = e.into_raw();
}
std::ptr::null_mut()
})
}
/// Free `ExtismCompiledPlugin`
#[no_mangle]
pub unsafe extern "C" fn extism_compiled_plugin_free(plugin: *mut CompiledPlugin) {
if plugin.is_null() {
return;
}
let plugin = Box::from_raw(plugin);
trace!("called extism_compiled_plugin_free");
drop(plugin)
}
/// Create a new plugin with host functions, the functions passed to this function no longer need to be manually freed using
///
/// `wasm`: is a WASM module (wat or wasm) or a JSON encoded manifest
@@ -391,70 +271,36 @@ pub unsafe extern "C" fn extism_plugin_new(
with_wasi: bool,
errmsg: *mut *mut std::ffi::c_char,
) -> *mut Plugin {
trace!("Call to extism_plugin_new with wasm pointer {:?}", wasm);
let data = std::slice::from_raw_parts(wasm, wasm_size as usize);
let funcs = if functions.is_null() {
vec![]
} else {
let funcs = (0..n_functions)
.map(|i| unsafe { *functions.add(i as usize) })
.map(|ptr| {
if ptr.is_null() {
return Err("Cannot pass null pointer");
let mut funcs = vec![];
if !functions.is_null() {
for i in 0..n_functions {
unsafe {
let f = *functions.add(i as usize);
if f.is_null() {
continue;
}
let ExtismFunction(func) = &*ptr;
let Some(func) = func.take() else {
return Err("Function cannot be registered with multiple different Plugins");
};
Ok(func)
})
.collect::<Result<Vec<_>, _>>()
.unwrap_or_else(|e| {
if !errmsg.is_null() {
let e = std::ffi::CString::new(e.to_string()).unwrap();
if let Some(f) = (*f).0.take() {
funcs.push(f);
} else {
let e = std::ffi::CString::new(
"Function cannot be registered with multiple different Plugins",
)
.unwrap();
*errmsg = e.into_raw();
}
Vec::new()
});
if funcs.len() != n_functions as usize {
return std::ptr::null_mut();
}
funcs
};
Plugin::new(data, funcs, with_wasi)
.map(|v| Box::into_raw(Box::new(v)))
.unwrap_or_else(|e| {
if !errmsg.is_null() {
let e = std::ffi::CString::new(format!(
"Unable to compile Extism plugin: {}",
e.root_cause(),
))
.unwrap();
*errmsg = e.into_raw();
}
std::ptr::null_mut()
})
}
}
}
/// Create a new plugin from an `ExtismCompiledPlugin`
#[no_mangle]
pub unsafe extern "C" fn extism_plugin_new_from_compiled(
compiled: *const CompiledPlugin,
errmsg: *mut *mut std::ffi::c_char,
) -> *mut Plugin {
let plugin = Plugin::new_from_compiled(&*compiled);
let plugin = Plugin::new(data, funcs, with_wasi);
match plugin {
Err(e) => {
if !errmsg.is_null() {
let e = std::ffi::CString::new(format!(
"Unable to create Extism plugin: {}",
e.root_cause(),
))
.unwrap();
let e = std::ffi::CString::new(format!("Unable to create Extism plugin: {}", e))
.unwrap();
*errmsg = e.into_raw();
}
std::ptr::null_mut()
@@ -463,100 +309,6 @@ pub unsafe extern "C" fn extism_plugin_new_from_compiled(
}
}
/// Create a new plugin and set the number of instructions a plugin is allowed to execute
#[no_mangle]
pub unsafe extern "C" fn extism_plugin_new_with_fuel_limit(
wasm: *const u8,
wasm_size: Size,
functions: *mut *const ExtismFunction,
n_functions: Size,
with_wasi: bool,
fuel_limit: u64,
errmsg: *mut *mut std::ffi::c_char,
) -> *mut Plugin {
trace!(
"Call to extism_plugin_new_with_fuel_limit with wasm pointer {:?}",
wasm
);
let data = std::slice::from_raw_parts(wasm, wasm_size as usize);
let funcs = if functions.is_null() {
vec![]
} else {
let funcs = (0..n_functions)
.map(|i| unsafe { *functions.add(i as usize) })
.map(|ptr| {
if ptr.is_null() {
return Err("Cannot pass null pointer");
}
let ExtismFunction(func) = &*ptr;
let Some(func) = func.take() else {
return Err("Function cannot be registered with multiple different Plugins");
};
Ok(func)
})
.collect::<Result<Vec<_>, _>>()
.unwrap_or_else(|e| {
if !errmsg.is_null() {
let e = std::ffi::CString::new(e.to_string()).unwrap();
*errmsg = e.into_raw();
}
Vec::new()
});
if funcs.len() != n_functions as usize {
return std::ptr::null_mut();
}
funcs
};
let compiled = match CompiledPlugin::new(
PluginBuilder::new(data)
.with_functions(funcs)
.with_wasi(with_wasi)
.with_fuel_limit(fuel_limit),
) {
Ok(x) => x,
Err(e) => {
if !errmsg.is_null() {
let e = std::ffi::CString::new(format!(
"Unable to compile Extism plugin: {}",
e.root_cause(),
))
.unwrap();
*errmsg = e.into_raw();
}
return std::ptr::null_mut();
}
};
let plugin = Plugin::new_from_compiled(&compiled);
match plugin {
Err(e) => {
if !errmsg.is_null() {
let e = std::ffi::CString::new(format!(
"Unable to create Extism plugin: {}",
e.root_cause(),
))
.unwrap();
*errmsg = e.into_raw();
}
std::ptr::null_mut()
}
Ok(p) => Box::into_raw(Box::new(p)),
}
}
/// Enable HTTP response headers in plugins using `extism:host/env::http_request`
#[no_mangle]
pub unsafe extern "C" fn extism_plugin_allow_http_response_headers(plugin: *mut Plugin) {
let plugin = &mut *plugin;
plugin.store.data_mut().http_headers = Some(BTreeMap::new());
}
/// Free the error returned by `extism_plugin_new`, errors returned from `extism_plugin_error` don't need to be freed
#[no_mangle]
pub unsafe extern "C" fn extism_plugin_new_error_free(err: *mut std::ffi::c_char) {
@@ -566,7 +318,7 @@ pub unsafe extern "C" fn extism_plugin_new_error_free(err: *mut std::ffi::c_char
drop(std::ffi::CString::from_raw(err))
}
/// Free `ExtismPlugin`
/// Remove a plugin from the registry and free associated memory
#[no_mangle]
pub unsafe extern "C" fn extism_plugin_free(plugin: *mut Plugin) {
if plugin.is_null() {
@@ -617,6 +369,8 @@ pub unsafe extern "C" fn extism_plugin_config(
return false;
}
let plugin = &mut *plugin;
let _lock = plugin.instance.clone();
let mut lock = _lock.lock().unwrap();
trace!(
plugin = plugin.id.to_string(),
@@ -627,11 +381,25 @@ pub unsafe extern "C" fn extism_plugin_config(
let json: std::collections::BTreeMap<String, Option<String>> =
match serde_json::from_slice(data) {
Ok(x) => x,
Err(_) => {
return false;
Err(e) => {
return plugin.return_error(&mut lock, e, false);
}
};
let wasi = &mut plugin.current_plugin_mut().wasi;
if let Some(Wasi { ctx, .. }) = wasi {
for (k, v) in json.iter() {
match v {
Some(v) => {
let _ = ctx.push_env(k, v);
}
None => {
let _ = ctx.push_env(k, "");
}
}
}
}
let id = plugin.id;
let config = &mut plugin.current_plugin_mut().manifest.config;
for (k, v) in json.into_iter() {
@@ -661,6 +429,9 @@ pub unsafe extern "C" fn extism_plugin_function_exists(
return false;
}
let plugin = &mut *plugin;
let _lock = plugin.instance.clone();
let mut lock = _lock.lock().unwrap();
let name = std::ffi::CStr::from_ptr(func_name);
trace!(
plugin = plugin.id.to_string(),
@@ -670,8 +441,8 @@ pub unsafe extern "C" fn extism_plugin_function_exists(
let name = match name.to_str() {
Ok(x) => x,
Err(_) => {
return false;
Err(e) => {
return plugin.return_error(&mut lock, e, false);
}
};
@@ -690,31 +461,6 @@ pub unsafe extern "C" fn extism_plugin_call(
func_name: *const c_char,
data: *const u8,
data_len: Size,
) -> i32 {
extism_plugin_call_with_host_context(plugin, func_name, data, data_len, std::ptr::null_mut())
}
#[derive(Clone)]
#[repr(transparent)]
struct CVoidContainer(*mut std::ffi::c_void);
// "You break it, you buy it."
unsafe impl Send for CVoidContainer {}
unsafe impl Sync for CVoidContainer {}
/// Call a function with host context.
///
/// `func_name`: is the function to call
/// `data`: is the input data
/// `data_len`: is the length of `data`
/// `host_context`: a pointer to context data that will be available in host functions
#[no_mangle]
pub unsafe extern "C" fn extism_plugin_call_with_host_context(
plugin: *mut Plugin,
func_name: *const c_char,
data: *const u8,
data_len: Size,
host_context: *mut std::ffi::c_void,
) -> i32 {
if plugin.is_null() {
return -1;
@@ -727,10 +473,7 @@ pub unsafe extern "C" fn extism_plugin_call_with_host_context(
let name = std::ffi::CStr::from_ptr(func_name);
let name = match name.to_str() {
Ok(name) => name,
Err(e) => {
plugin.error_msg = Some(make_error_msg(e.to_string()));
return -1;
}
Err(e) => return plugin.return_error(&mut lock, e, -1),
};
trace!(
@@ -739,17 +482,10 @@ pub unsafe extern "C" fn extism_plugin_call_with_host_context(
name
);
let input = std::slice::from_raw_parts(data, data_len as usize);
let r = if host_context.is_null() {
None
} else {
Some(CVoidContainer(host_context))
};
let res = plugin.raw_call(&mut lock, name, input, r);
let res = plugin.raw_call(&mut lock, name, input);
match res {
Err((e, rc)) => {
plugin.error_msg = Some(make_error_msg(e.to_string()));
rc
}
Err((e, rc)) => plugin.return_error(&mut lock, e, rc),
Ok(x) => x,
}
}
@@ -772,26 +508,14 @@ pub unsafe extern "C" fn extism_plugin_error(plugin: *mut Plugin) -> *const c_ch
let _lock = _lock.lock().unwrap();
if plugin.output.error_offset == 0 {
if let Some(err) = &plugin.error_msg {
return err.as_ptr() as *const _;
}
trace!(plugin = plugin.id.to_string(), "error is NULL");
return std::ptr::null();
}
let offs = plugin.output.error_offset;
let ptr = plugin.current_plugin_mut().memory_ptr().add(offs as usize) as *const _;
let len = plugin
plugin
.current_plugin_mut()
.memory_length(offs)
.unwrap_or_default();
let mut data = std::slice::from_raw_parts(ptr, len as usize).to_vec();
data.push(0);
plugin.error_msg = Some(data);
plugin.error_msg.as_ref().unwrap().as_ptr() as *const _
.memory_ptr()
.add(plugin.output.error_offset as usize) as *const _
}
/// Get the length of a plugin's output data
@@ -866,12 +590,13 @@ pub unsafe extern "C" fn extism_log_file(
fn set_log_file(log_file: impl Into<std::path::PathBuf>, filter: &str) -> Result<(), Error> {
let log_file = log_file.into();
let s = log_file.to_str();
let is_level = tracing::Level::from_str(filter).is_ok();
let cfg = tracing_subscriber::FmtSubscriber::builder().with_env_filter({
let x = tracing_subscriber::EnvFilter::builder()
.with_default_directive(tracing::Level::ERROR.into());
if is_level {
x.parse_lossy(format!("extism={filter}"))
x.parse_lossy(format!("extism={}", filter))
} else {
x.parse_lossy(filter)
}
@@ -899,7 +624,7 @@ fn set_log_file(log_file: impl Into<std::path::PathBuf>, filter: &str) -> Result
Ok(())
}
static LOG_BUFFER: std::sync::Mutex<Option<LogBuffer>> = std::sync::Mutex::new(None);
static mut LOG_BUFFER: Option<LogBuffer> = None;
/// Enable a custom log handler, this will buffer logs until `extism_log_drain` is called
/// Log level should be one of: info, error, trace, debug, warn
@@ -916,7 +641,6 @@ pub unsafe extern "C" fn extism_log_custom(log_level: *const c_char) -> bool {
} else {
"error"
};
set_log_buffer(level).is_ok()
}
@@ -926,13 +650,13 @@ unsafe fn set_log_buffer(filter: &str) -> Result<(), Error> {
let x = tracing_subscriber::EnvFilter::builder()
.with_default_directive(tracing::Level::ERROR.into());
if is_level {
x.parse_lossy(format!("extism={filter}"))
x.parse_lossy(format!("extism={}", filter))
} else {
x.parse_lossy(filter)
}
});
*LOG_BUFFER.lock().unwrap() = Some(LogBuffer::default());
let buf = LOG_BUFFER.lock().unwrap().clone().unwrap();
LOG_BUFFER = Some(LogBuffer::default());
let buf = LOG_BUFFER.clone().unwrap();
cfg.with_ansi(false)
.with_writer(move || buf.clone())
.try_init()
@@ -943,11 +667,11 @@ unsafe fn set_log_buffer(filter: &str) -> Result<(), Error> {
#[no_mangle]
/// Calls the provided callback function for each buffered log line.
/// This is only needed when `extism_log_custom` is used.
pub unsafe extern "C" fn extism_log_drain(handler: ExtismLogDrainFunctionType) {
if let Some(buf) = LOG_BUFFER.lock().unwrap().as_mut() {
pub unsafe extern "C" fn extism_log_drain(handler: extern "C" fn(*const std::ffi::c_char, usize)) {
if let Some(buf) = &mut LOG_BUFFER {
if let Ok(mut buf) = buf.buffer.lock() {
for (line, len) in buf.drain(..) {
handler(line.as_ptr(), len as u64);
handler(line.as_ptr(), len);
}
}
}
@@ -978,30 +702,6 @@ impl std::io::Write for LogBuffer {
}
}
/// Reset the Extism runtime, this will invalidate all allocated memory
#[no_mangle]
pub unsafe extern "C" fn extism_plugin_reset(plugin: *mut Plugin) -> bool {
let plugin = &mut *plugin;
if let Err(e) = plugin.reset() {
error!(
plugin = plugin.id.to_string(),
"unable to reset plugin: {}",
e.to_string()
);
if let Err(e) = plugin.current_plugin_mut().set_error(e.to_string()) {
error!(
plugin = plugin.id.to_string(),
"unable to set error after failed plugin reset: {}",
e.to_string()
);
}
false
} else {
true
}
}
/// Get the Extism version string
#[no_mangle]
pub unsafe extern "C" fn extism_version() -> *const c_char {

View File

@@ -1 +0,0 @@
hello world!

View File

@@ -1,7 +1,6 @@
use crate::*;
const WASM_EMPTY: &[u8] = include_bytes!("../../../wasm/empty.wasm");
const WASM_UNREACHABLE: &[u8] = include_bytes!("../../../wasm/unreachable.wasm");
// https://github.com/extism/extism/issues/620
#[test]
@@ -16,7 +15,7 @@ fn test_issue_620() {
// Call test method, this does not work
let p = plugin.call::<(), String>("test", ()).unwrap();
println!("{p}");
println!("{}", p);
}
// https://github.com/extism/extism/issues/619
@@ -27,31 +26,3 @@ host_fn!(
Ok(path.display().to_string())
}
);
// https://github.com/extism/extism/issues/775
#[test]
fn test_issue_775() {
// Load and build plugin
let url = Wasm::data(WASM_UNREACHABLE);
let manifest = Manifest::new([url]);
let mut plugin = PluginBuilder::new(manifest)
.with_wasi(true)
.build()
.unwrap();
// Call test method
let lock = plugin.instance.clone();
let mut lock = lock.lock().unwrap();
let res = plugin.raw_call(&mut lock, "do_unreachable", b"", None::<()>);
let p = match res {
Err(e) => {
if e.1 == 0 {
Err(e.1)
} else {
Ok(e.1)
}
}
Ok(code) => Err(code),
}
.unwrap();
println!("{p}");
}

View File

@@ -1,34 +1,23 @@
use crate::*;
use quickcheck::*;
const KERNEL: &[u8] = include_bytes!("../extism-runtime.wasm");
fn extism_alloc<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, n: u64) -> u64 {
fn extism_alloc<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance, n: u64) -> u64 {
let out_alloc = &mut [Val::I64(0)];
instance
.get_func(&mut *store, "alloc")
.get_func(&mut store, "alloc")
.unwrap()
.call(store, &[Val::I64(n as i64)], out_alloc)
.call(&mut store, &[Val::I64(n as i64)], out_alloc)
.unwrap();
out_alloc[0].unwrap_i64() as u64
}
fn extism_length<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) -> u64 {
fn extism_length<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) -> u64 {
let out = &mut [Val::I64(0)];
instance
.get_func(&mut *store, "length")
.get_func(&mut store, "length")
.unwrap()
.call(store, &[Val::I64(p as i64)], out)
.unwrap();
out[0].unwrap_i64() as u64
}
fn extism_length_unsafe<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) -> u64 {
let out = &mut [Val::I64(0)];
instance
.get_func(&mut *store, "length_unsafe")
.unwrap()
.call(store, &[Val::I64(p as i64)], out)
.call(&mut store, &[Val::I64(p as i64)], out)
.unwrap();
out[0].unwrap_i64() as u64
}
@@ -43,96 +32,122 @@ fn extism_load_u8<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance
out[0].unwrap_i32() as u8
}
fn extism_load_u64<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) -> u64 {
fn extism_load_u64<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) -> u64 {
let out = &mut [Val::I32(0)];
instance
.get_func(&mut *store, "load_u64")
.get_func(&mut store, "load_u64")
.unwrap()
.call(store, &[Val::I64(p as i64)], out)
.call(&mut store, &[Val::I64(p as i64)], out)
.unwrap();
out[0].unwrap_i64() as u64
}
fn extism_input_load_u8<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) -> u8 {
fn extism_input_load_u8<T>(
mut store: &mut wasmtime::Store<T>,
instance: &mut Instance,
p: u64,
) -> u8 {
let out = &mut [Val::I32(0)];
instance
.get_func(&mut *store, "input_load_u8")
.get_func(&mut store, "input_load_u8")
.unwrap()
.call(store, &[Val::I64(p as i64)], out)
.call(&mut store, &[Val::I64(p as i64)], out)
.unwrap();
out[0].unwrap_i32() as u8
}
fn extism_input_load_u64<T>(
store: &mut wasmtime::Store<T>,
mut store: &mut wasmtime::Store<T>,
instance: &mut Instance,
p: u64,
) -> u64 {
let out = &mut [Val::I32(0)];
instance
.get_func(&mut *store, "input_load_u64")
.get_func(&mut store, "input_load_u64")
.unwrap()
.call(store, &[Val::I64(p as i64)], out)
.call(&mut store, &[Val::I64(p as i64)], out)
.unwrap();
out[0].unwrap_i64() as u64
}
fn extism_store_u8<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64, x: u8) {
fn extism_store_u8<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64, x: u8) {
instance
.get_func(&mut *store, "store_u8")
.get_func(&mut store, "store_u8")
.unwrap()
.call(store, &[Val::I64(p as i64), Val::I32(x as i32)], &mut [])
.call(
&mut store,
&[Val::I64(p as i64), Val::I32(x as i32)],
&mut [],
)
.unwrap();
}
fn extism_store_u64<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64, x: u64) {
fn extism_store_u64<T>(
mut store: &mut wasmtime::Store<T>,
instance: &mut Instance,
p: u64,
x: u64,
) {
instance
.get_func(&mut *store, "store_u64")
.get_func(&mut store, "store_u64")
.unwrap()
.call(store, &[Val::I64(p as i64), Val::I64(x as i64)], &mut [])
.call(
&mut store,
&[Val::I64(p as i64), Val::I64(x as i64)],
&mut [],
)
.unwrap();
}
fn extism_free<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) {
fn extism_free<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) {
instance
.get_func(&mut *store, "free")
.get_func(&mut store, "free")
.unwrap()
.call(store, &[Val::I64(p as i64)], &mut [])
.call(&mut store, &[Val::I64(p as i64)], &mut [])
.unwrap();
}
fn extism_error_set<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) {
fn extism_error_set<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64) {
instance
.get_func(&mut *store, "error_set")
.get_func(&mut store, "error_set")
.unwrap()
.call(store, &[Val::I64(p as i64)], &mut [])
.call(&mut store, &[Val::I64(p as i64)], &mut [])
.unwrap();
}
fn extism_error_get<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance) -> u64 {
fn extism_error_get<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance) -> u64 {
let out = &mut [Val::I64(0)];
instance
.get_func(&mut *store, "error_get")
.get_func(&mut store, "error_get")
.unwrap()
.call(store, &[], out)
.call(&mut store, &[], out)
.unwrap();
out[0].unwrap_i64() as u64
}
fn extism_reset<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance) {
fn extism_reset<T>(mut store: &mut wasmtime::Store<T>, instance: &mut Instance) {
instance
.get_func(&mut *store, "reset")
.get_func(&mut store, "reset")
.unwrap()
.call(store, &[], &mut [])
.call(&mut store, &[], &mut [])
.unwrap();
}
fn extism_input_set<T>(store: &mut wasmtime::Store<T>, instance: &mut Instance, p: u64, l: u64) {
fn extism_input_set<T>(
mut store: &mut wasmtime::Store<T>,
instance: &mut Instance,
p: u64,
l: u64,
) {
instance
.get_func(&mut *store, "input_set")
.get_func(&mut store, "input_set")
.unwrap()
.call(store, &[Val::I64(p as i64), Val::I64(l as i64)], &mut [])
.call(
&mut store,
&[Val::I64(p as i64), Val::I64(l as i64)],
&mut [],
)
.unwrap();
}
@@ -153,32 +168,11 @@ fn test_kernel_allocations() {
// Test allocations
assert_eq!(extism_alloc(&mut store, instance, 0), 0);
// 512 bytes, test block re-use + splitting
let p = extism_alloc(&mut store, instance, 65535);
let first_alloc = p;
assert_eq!(extism_length(&mut store, instance, p), 65535);
assert_eq!(extism_length(&mut store, instance, p + 1), 0);
assert_eq!(extism_length(&mut store, instance, p + 2), 0);
assert_eq!(extism_length(&mut store, instance, p + 3), 0);
assert_eq!(extism_length(&mut store, instance, p + 4), 0);
extism_free(&mut store, instance, p);
// Should re-use the previous block
let q = extism_alloc(&mut store, instance, 65535);
assert_eq!(q, p);
assert_eq!(extism_length(&mut store, instance, q), 65535);
extism_free(&mut store, instance, q);
let r = extism_alloc(&mut store, instance, 65535 - 24);
assert_eq!(r, q);
assert_eq!(extism_length(&mut store, instance, q), 65535 - 24);
extism_free(&mut store, instance, r);
// 1 byte
let p = extism_alloc(&mut store, instance, 1);
let first_alloc = p;
assert!(p > 0);
assert_eq!(extism_length(&mut store, instance, p), 1);
assert_eq!(extism_length_unsafe(&mut store, instance, p), 1);
extism_free(&mut store, instance, p);
// 2 bytes
@@ -186,17 +180,39 @@ fn test_kernel_allocations() {
assert!(x > 0);
assert!(x != p);
assert_eq!(extism_length(&mut store, instance, x), 2);
assert_eq!(extism_length_unsafe(&mut store, instance, x), 2);
extism_free(&mut store, instance, x);
for i in 0..64 {
let p = extism_alloc(&mut store, instance, 64 - i);
assert!(p > 0);
assert_eq!(extism_length(&mut store, instance, p), 64 - i);
assert_eq!(extism_length_unsafe(&mut store, instance, p), 64 - i);
extism_free(&mut store, instance, p);
// should re-use the last allocation
let q = extism_alloc(&mut store, instance, 64 - i);
assert_eq!(p, q);
assert_eq!(extism_length(&mut store, instance, q), 64 - i);
extism_free(&mut store, instance, q);
}
// 512 bytes, test block re-use + splitting
let p = extism_alloc(&mut store, instance, 512);
assert_eq!(extism_length(&mut store, instance, p), 512);
extism_free(&mut store, instance, p);
// 128 bytes, should be split off the 512 byte block
let q = extism_alloc(&mut store, instance, 128);
assert!(p <= q && q < p + 512);
assert_eq!(extism_length(&mut store, instance, q), 128);
extism_free(&mut store, instance, q);
// 128 bytes, same as above
let r = extism_alloc(&mut store, instance, 128);
assert!(p <= r && r < p + 512);
assert!(r > p);
assert_eq!(extism_length(&mut store, instance, q), 128);
extism_free(&mut store, instance, q);
// 100 pages
let p = extism_alloc(&mut store, instance, 6553600);
assert!(p > 0);
@@ -225,26 +241,6 @@ fn test_kernel_allocations() {
extism_free(&mut store, instance, q);
}
#[test]
fn test_kernel_page_allocations() {
let (mut store, mut instance) = init_kernel_test();
let instance = &mut instance;
let a = extism_alloc(&mut store, instance, 65500 * 4);
let a_size = extism_length(&mut store, instance, a);
let b = extism_alloc(&mut store, instance, 65539);
extism_free(&mut store, instance, a);
let c = extism_alloc(&mut store, instance, 65500 * 2);
let c_size = extism_length(&mut store, instance, c);
let d = extism_alloc(&mut store, instance, 65500);
assert_eq!(a + (a_size - c_size), c);
assert!(c < b);
assert!(d < b);
}
#[test]
fn test_kernel_error() {
let (mut store, mut instance) = init_kernel_test();
@@ -301,186 +297,3 @@ fn test_load_input() {
// Out of bounds should return 0
assert_eq!(extism_input_load_u64(&mut store, instance, 123457), 0);
}
#[test]
fn test_failed_quickcheck1() {
let (mut store, mut instance) = init_kernel_test();
let allocs = [
20622, 23162, 58594, 32421, 25928, 44611, 26318, 24455, 5798, 60202, 42126, 64928, 57832,
50888, 63256, 37562, 46334, 47985, 60836, 28132, 65535, 37800, 33150, 48768, 38457, 57249,
5734, 58587, 26294, 26653, 24519, 1,
];
extism_reset(&mut store, &mut instance);
for a in allocs {
println!("Alloc: {a}");
let n = extism_alloc(&mut store, &mut instance, a);
if n == 0 {
continue;
}
assert_eq!(a, extism_length(&mut store, &mut instance, n));
}
}
#[test]
fn test_failed_quickcheck2() {
let (mut store, mut instance) = init_kernel_test();
let allocs = [352054710, 1248853976, 2678441931, 14567928];
extism_reset(&mut store, &mut instance);
for a in allocs {
println!("Alloc: {a}");
let n = extism_alloc(&mut store, &mut instance, a);
if n == 0 {
continue;
}
assert_eq!(a, extism_length(&mut store, &mut instance, n));
}
}
quickcheck! {
fn check_alloc(amounts: Vec<u16>) -> bool {
let (mut store, mut instance) = init_kernel_test();
let instance = &mut instance;
for a in amounts {
let ptr = extism_alloc(&mut store, instance, a as u64);
if ptr == 0 || ptr == u64::MAX {
continue
}
if extism_length(&mut store, instance, ptr) != a as u64 {
return false
}
}
true
}
}
quickcheck! {
fn check_large_alloc(amounts: Vec<u32>) -> bool {
let (mut store, mut instance) = init_kernel_test();
let instance = &mut instance;
for a in amounts {
let ptr = extism_alloc(&mut store, instance, a as u64);
if ptr == 0 {
continue
}
let len = extism_length_unsafe(&mut store, instance, ptr);
if len != a as u64 {
return false
}
}
true
}
}
quickcheck! {
fn check_alloc_with_frees(amounts: Vec<u16>) -> bool {
let (mut store, mut instance) = init_kernel_test();
let instance = &mut instance;
let mut prev = 0;
for a in amounts {
let ptr = extism_alloc(&mut store, instance, a as u64);
if ptr == 0 {
continue
}
if extism_length(&mut store, instance, ptr) != a as u64 {
return false
}
if a % 2 == 0 {
extism_free(&mut store, instance, ptr);
} else if a % 3 == 0 {
extism_free(&mut store, instance, prev);
}
prev = ptr;
}
true
}
}
quickcheck! {
fn check_large_alloc_with_frees(amounts: Vec<u32>) -> bool {
let (mut store, mut instance) = init_kernel_test();
let instance = &mut instance;
let mut prev = 0;
for a in amounts {
let ptr = extism_alloc(&mut store, instance, a as u64);
if ptr == 0 || ptr == u64::MAX {
continue
}
if extism_length(&mut store, instance, ptr) != a as u64 {
return false
}
if a % 2 == 0 {
extism_free(&mut store, instance, ptr);
} else if a % 3 == 0 {
extism_free(&mut store, instance, prev);
}
prev = ptr;
}
true
}
}
quickcheck! {
fn check_alloc_with_load_and_store(amounts: Vec<u16>) -> bool {
use rand::Rng;
let mut rng = rand::rng();
let (mut store, mut instance) = init_kernel_test();
let instance = &mut instance;
for a in amounts {
let ptr = extism_alloc(&mut store, instance, a as u64);
if ptr == 0 || ptr == u64::MAX {
continue
}
if extism_length(&mut store, instance, ptr) != a as u64 {
return false
}
for _ in 0..16 {
let i = rng.random_range(ptr..ptr+a as u64);
extism_store_u8(&mut store, instance, i, i as u8);
if extism_load_u8(&mut store, instance, i as u64) != i as u8 {
return false
}
}
}
true
}
}
quickcheck! {
fn check_block_reuse(allocs: Vec<u16>) -> bool {
let (mut store, mut instance) = init_kernel_test();
let instance = &mut instance;
let init = extism_alloc(&mut store, instance, allocs.iter().map(|x| *x as u64).sum::<u64>() + allocs.len() as u64 * 64);
let bounds = init + extism_length(&mut store, instance, init);
extism_free(&mut store, instance, init);
for a in allocs {
let ptr = extism_alloc(&mut store, instance, a as u64);
if ptr == 0 {
continue
}
if extism_length(&mut store, instance, ptr) != a as u64 {
return false
}
extism_free(&mut store, instance , ptr);
if ptr > bounds {
println!("ptr={ptr}, bounds={bounds}");
return false
}
}
true
}
}

View File

@@ -1,4 +1,3 @@
mod issues;
mod kernel;
mod pool;
mod runtime;

View File

@@ -1,48 +0,0 @@
use crate::*;
fn run_thread(p: Pool, i: u64) -> std::thread::JoinHandle<()> {
std::thread::spawn(move || {
std::thread::sleep(std::time::Duration::from_millis(i));
let s: String = p
.get(std::time::Duration::from_secs(1))
.unwrap()
.unwrap()
.call("count_vowels", "abc")
.unwrap();
println!("{s}");
})
}
#[test]
fn test_threads() {
for i in 1..=3 {
let data = include_bytes!("../../../wasm/code.wasm");
let plugin_builder =
extism::PluginBuilder::new(extism::Manifest::new([extism::Wasm::data(data)]))
.with_wasi(true);
let pool: Pool = PoolBuilder::new()
.with_max_instances(i)
.build(move || plugin_builder.clone().build());
let threads = vec![
run_thread(pool.clone(), 1000),
run_thread(pool.clone(), 1000),
run_thread(pool.clone(), 1000),
run_thread(pool.clone(), 1000),
run_thread(pool.clone(), 1000),
run_thread(pool.clone(), 1000),
run_thread(pool.clone(), 500),
run_thread(pool.clone(), 500),
run_thread(pool.clone(), 500),
run_thread(pool.clone(), 500),
run_thread(pool.clone(), 500),
run_thread(pool.clone(), 0),
];
for t in threads {
t.join().unwrap();
}
assert!(pool.count() <= i);
}
}

View File

@@ -1,7 +1,5 @@
use extism_manifest::{HttpRequest, MemoryOptions};
use crate::*;
use std::{collections::HashMap, io::Write, time::Instant};
use std::{io::Write, time::Instant};
const WASM: &[u8] = include_bytes!("../../../wasm/code-functions.wasm");
const WASM_NO_FUNCTIONS: &[u8] = include_bytes!("../../../wasm/code.wasm");
@@ -9,8 +7,6 @@ const WASM_LOOP: &[u8] = include_bytes!("../../../wasm/loop.wasm");
const WASM_GLOBALS: &[u8] = include_bytes!("../../../wasm/globals.wasm");
const WASM_REFLECT: &[u8] = include_bytes!("../../../wasm/reflect.wasm");
const WASM_HTTP: &[u8] = include_bytes!("../../../wasm/http.wasm");
const WASM_HTTP_HEADERS: &[u8] = include_bytes!("../../../wasm/http_headers.wasm");
const WASM_FS: &[u8] = include_bytes!("../../../wasm/read_write.wasm");
host_fn!(pub hello_world (a: String) -> String { Ok(a) });
@@ -43,12 +39,11 @@ pub struct Count {
#[test]
fn it_works() {
let log = tracing_subscriber::fmt()
tracing_subscriber::fmt()
.with_ansi(false)
.with_env_filter("extism=debug")
.with_writer(std::fs::File::create("test.log").unwrap())
.try_init()
.is_ok();
.init();
let wasm_start = Instant::now();
@@ -133,7 +128,10 @@ fn it_works() {
.unwrap();
let native_avg: std::time::Duration = native_sum / native_num_tests as u32;
println!("native function call (avg, N = {native_num_tests}): {native_avg:?}");
println!(
"native function call (avg, N = {}): {:?}",
native_num_tests, native_avg
);
let num_tests = test_times.len();
let sum: std::time::Duration = test_times
@@ -142,13 +140,11 @@ fn it_works() {
.unwrap();
let avg: std::time::Duration = sum / num_tests as u32;
println!("wasm function call (avg, N = {num_tests}): {avg:?}");
println!("wasm function call (avg, N = {}): {:?}", num_tests, avg);
// Check that log file was written to
if log {
let meta = std::fs::metadata("test.log").unwrap();
assert!(meta.len() > 0);
}
let meta = std::fs::metadata("test.log").unwrap();
assert!(meta.len() > 0);
}
#[test]
@@ -209,7 +205,7 @@ fn test_cancel() {
let _output: Result<&[u8], Error> = plugin.call("loop_forever", "abc123");
let end = std::time::Instant::now();
let time = end - start;
println!("Cancelled plugin ran for {time:?}");
println!("Cancelled plugin ran for {:?}", time);
}
}
@@ -239,68 +235,6 @@ fn test_timeout() {
assert!(err == "timeout");
}
#[test]
fn test_fuel() {
let manifest = Manifest::new([extism_manifest::Wasm::data(WASM_LOOP)]);
let mut plugin = PluginBuilder::new(manifest)
.with_wasi(true)
.with_fuel_limit(1)
.build()
.unwrap();
for _ in 0..10001 {
let output: Result<&[u8], Error> = plugin.call("loop_forever", "abc123");
let err = output.unwrap_err().root_cause().to_string();
println!("Fuel limited plugin exited with error: {:?}", &err);
assert!(err.contains("fuel"));
}
}
#[test]
fn test_fuel_consumption() {
let manifest = Manifest::new([extism_manifest::Wasm::data(WASM_LOOP)]);
let mut plugin = PluginBuilder::new(manifest)
.with_wasi(true)
.with_fuel_limit(10000)
.build()
.unwrap();
let output: Result<&[u8], Error> = plugin.call("loop_forever", "abc123");
assert!(output.is_err());
let fuel_consumed = plugin.fuel_consumed().unwrap();
println!("Fuel consumed: {fuel_consumed}");
assert!(fuel_consumed > 0);
}
#[test]
#[cfg(feature = "http")]
fn test_http_timeout() {
let f = Function::new(
"hello_world",
[PTR],
[PTR],
UserData::default(),
hello_world,
);
let manifest = Manifest::new([extism_manifest::Wasm::data(WASM_HTTP)])
.with_timeout(std::time::Duration::from_millis(1))
.with_allowed_host("www.extism.org");
let mut plugin = Plugin::new(manifest, [f], true).unwrap();
let start = std::time::Instant::now();
let output: Result<&[u8], Error> =
plugin.call("http_request", r#"{"url": "https://www.extism.org"}"#);
let end = std::time::Instant::now();
let time = end - start;
let err = output.unwrap_err().root_cause().to_string();
println!(
"Timed out plugin ran for {:?}, with error: {:?}",
time, &err
);
assert!(err == "timeout");
}
typed_plugin!(pub TestTypedPluginGenerics {
count_vowels<T: FromBytes<'a>>(&str) -> T
});
@@ -349,10 +283,8 @@ fn test_multiple_instantiations() {
#[test]
fn test_globals() {
let mut plugin = Plugin::new(WASM_GLOBALS, [], true).unwrap();
for i in 0..100001 {
let Json(count) = plugin
.call_with_host_context::<_, Json<Count>, _>("globals", "", ())
.unwrap();
for i in 0..1000 {
let Json(count) = plugin.call::<_, Json<Count>>("globals", "").unwrap();
assert_eq!(count.count, i);
}
}
@@ -370,42 +302,6 @@ fn test_toml_manifest() {
assert_eq!(count.get("count").unwrap().as_i64().unwrap(), 1);
}
#[test]
fn test_call_with_host_context() {
#[derive(Clone)]
struct Foo {
message: String,
}
let f = Function::new(
"host_reflect",
[PTR],
[PTR],
UserData::default(),
|current_plugin, _val, ret, _user_data: UserData<()>| {
let foo = current_plugin.host_context::<Foo>()?.clone();
let hnd = current_plugin.memory_new(foo.message)?;
ret[0] = current_plugin.memory_to_val(hnd);
Ok(())
},
);
let mut plugin = Plugin::new(WASM_REFLECT, [f], true).unwrap();
let message = "hello world";
let output: String = plugin
.call_with_host_context(
"reflect",
"anything, really",
Foo {
message: message.to_string(),
},
)
.unwrap();
assert_eq!(output, message);
}
#[test]
fn test_fuzz_reflect_plugin() {
// assert!(set_log_file("stdout", Some(log::Level::Trace)));
@@ -437,7 +333,7 @@ fn test_memory_max() {
assert!(output.is_err());
let err = output.unwrap_err().root_cause().to_string();
println!("{err:?}");
println!("{:?}", err);
assert_eq!(err, "oom");
// Should pass with memory.max set to a large enough number
@@ -461,7 +357,7 @@ fn hello_world_set_error(
_user_data: UserData<()>,
) -> Result<(), Error> {
plugin.set_error("TEST")?;
outputs[0] = inputs[0];
outputs[0] = inputs[0].clone();
Ok(())
}
@@ -500,7 +396,7 @@ fn test_extism_error() {
let mut plugin = Plugin::new(&manifest, [f], true).unwrap();
let output: Result<String, Error> = plugin.call("count_vowels", "a".repeat(1024));
assert!(output.is_err());
println!("{output:?}");
println!("{:?}", output);
assert_eq!(output.unwrap_err().root_cause().to_string(), "TEST");
}
@@ -558,7 +454,7 @@ fn hello_world_user_data(
let mut data = data.lock().unwrap();
let s = _plugin.memory_get_val(&inputs[0])?;
data.write_all(s)?;
outputs[0] = inputs[0];
outputs[0] = inputs[0].clone();
Ok(())
}
@@ -676,165 +572,3 @@ fn test_disable_cache() {
assert!(t < t1);
}
#[test]
fn test_manifest_ptr_len() {
let manifest = serde_json::json!({
"wasm" : [
{
"data" : {
"ptr" : WASM_NO_FUNCTIONS.as_ptr() as u64,
"len" : WASM_NO_FUNCTIONS.len()
}
}
]
});
let mut plugin = Plugin::new(manifest.to_string().as_bytes(), [], true).unwrap();
let output = plugin.call("count_vowels", "abc123").unwrap();
let count: serde_json::Value = serde_json::from_slice(output).unwrap();
assert_eq!(count.get("count").unwrap().as_i64().unwrap(), 1);
}
#[test]
fn test_no_vars() {
let data = br#"
(module
(import "extism:host/env" "var_set" (func $var_set (param i64 i64)))
(import "extism:host/env" "input_offset" (func $input_offset (result i64)))
(func (export "test") (result i32)
(call $input_offset)
(call $input_offset)
(call $var_set)
(i32.const 0)
)
)
"#;
let manifest = Manifest::new([Wasm::data(data)])
.with_memory_options(MemoryOptions::new().with_max_var_bytes(1));
let mut plugin = Plugin::new(manifest, [], true).unwrap();
let output: Result<(), Error> = plugin.call("test", b"A".repeat(1024));
assert!(output.is_err());
let output: Result<(), Error> = plugin.call("test", vec![]);
assert!(output.is_ok());
}
#[test]
fn test_linking() {
let manifest = Manifest::new([
Wasm::Data {
data: br#"
(module
(import "wasi_snapshot_preview1" "random_get" (func $random (param i32 i32) (result i32)))
(import "extism:host/env" "alloc" (func $alloc (param i64) (result i64)))
(import "extism:host/user" "hello" (func $hello))
(global $counter (mut i32) (i32.const 0))
(func $start (export "_start")
(global.set $counter (i32.add (global.get $counter) (i32.const 1)))
)
(func (export "read_counter") (result i32)
(global.get $counter)
)
(start $start)
)
"#.to_vec(),
meta: WasmMetadata {
name: Some("commander".to_string()),
hash: None,
},
},
Wasm::Data {
data: br#"
(module
(import "commander" "_start" (func $commander_start))
(import "commander" "read_counter" (func $commander_read_counter (result i32)))
(import "extism:host/env" "store_u64" (func $store_u64 (param i64 i64)))
(import "extism:host/env" "alloc" (func $alloc (param i64) (result i64)))
(import "extism:host/user" "hello" (func $hello))
(import "extism:host/env" "output_set" (func $output_set (param i64 i64)))
(func (export "run") (result i32)
(local $output i64)
(local.set $output (call $alloc (i64.const 8)))
(call $commander_start)
(call $commander_start)
(call $commander_start)
(call $commander_start)
(call $hello)
(call $store_u64 (local.get $output) (i64.extend_i32_u (call $commander_read_counter)))
(call $output_set (local.get $output) (i64.const 8))
i32.const 0
)
)
"#.to_vec(),
meta: WasmMetadata {
name: Some("main".to_string()),
hash: None,
},
},
]);
let mut plugin = PluginBuilder::new(manifest)
.with_wasi(true)
.with_function("hello", [], [], UserData::new(()), |_, _, _, _| {
eprintln!("hello!");
Ok(())
})
.build()
.unwrap();
for _ in 0..5 {
assert_eq!(plugin.call::<&str, i64>("run", "Hello, world!").unwrap(), 1);
}
}
#[test]
fn test_readonly_dirs() {
let wasm = Wasm::data(WASM_FS);
let manifest = Manifest::new([wasm])
.with_allowed_path("ro:src/tests/data".to_string(), "/data")
.with_config_key("path", "/data/data.txt");
let mut plugin = PluginBuilder::new(manifest)
.with_wasi(true)
.build()
.unwrap();
let res = plugin.call::<&str, &str>("try_read", "").unwrap();
assert_eq!(res, "hello world!");
let line = "hello world 2";
let res2 = plugin.call::<&str, &str>("try_write", line);
assert!(
res2.is_err(),
"Expected try_write to fail, but it succeeded."
);
}
#[test]
#[cfg(feature = "http")]
fn test_http_response_headers() {
let mut plugin = PluginBuilder::new(
Manifest::new([Wasm::data(WASM_HTTP_HEADERS)]).with_allowed_host("extism.org"),
)
.with_http_response_headers(true)
.build()
.unwrap();
let req = HttpRequest::new("https://extism.org");
let Json(res): Json<HashMap<String, String>> = plugin.call("http_get", Json(req)).unwrap();
println!("{res:?}");
assert_eq!(res["content-type"], "text/html; charset=utf-8");
}
#[test]
#[cfg(feature = "http")]
fn test_http_response_headers_disabled() {
let mut plugin = PluginBuilder::new(
Manifest::new([Wasm::data(WASM_HTTP_HEADERS)]).with_allowed_host("extism.org"),
)
.with_http_response_headers(false)
.build()
.unwrap();
let req = HttpRequest::new("https://extism.org");
let Json(res): Json<HashMap<String, String>> = plugin.call("http_get", Json(req)).unwrap();
println!("{res:?}");
assert!(res.is_empty());
}

View File

@@ -22,18 +22,18 @@ pub(crate) struct Timer {
#[cfg(not(target_family = "windows"))]
extern "C" fn cleanup_timer() {
let mut timer = match TIMER.lock() {
let mut timer = match unsafe { TIMER.lock() } {
Ok(x) => x,
Err(e) => e.into_inner(),
};
drop(timer.take());
}
static TIMER: std::sync::Mutex<Option<Timer>> = std::sync::Mutex::new(None);
static mut TIMER: std::sync::Mutex<Option<Timer>> = std::sync::Mutex::new(None);
impl Timer {
pub(crate) fn tx() -> std::sync::mpsc::Sender<TimerAction> {
let mut timer = match TIMER.lock() {
let mut timer = match unsafe { TIMER.lock() } {
Ok(x) => x,
Err(e) => e.into_inner(),
};
@@ -92,39 +92,25 @@ impl Timer {
loop {
if plugins.is_empty() {
if let Ok(x) = rx.recv() {
handle!(x);
handle!(x)
}
}
let mut timeout: Option<std::time::Duration> = None;
plugins.retain(|_k, (engine, end)| {
if let Some(end) = end {
let now = std::time::Instant::now();
if *end <= now {
engine.increment_epoch();
return false;
} else {
let time_left =
(*end - now).saturating_sub(std::time::Duration::from_millis(1));
if let Some(t) = &timeout {
if time_left < *t {
timeout = Some(time_left);
}
} else {
timeout = Some(time_left);
plugins = plugins
.into_iter()
.filter(|(_k, (engine, end))| {
if let Some(end) = end {
let now = std::time::Instant::now();
if end <= &now {
engine.increment_epoch();
return false;
}
}
}
true
})
.collect();
true
});
if let Some(timeout) = timeout {
if let Ok(x) = rx.recv_timeout(timeout) {
handle!(x)
}
} else if let Ok(x) = rx.recv() {
for x in rx.try_iter() {
handle!(x)
}
}

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.