Compare commits

..

6 Commits

Author SHA1 Message Date
zach
91d41d83b6 cleanup: implement more type checks 2024-12-03 16:40:51 -08:00
zach
6350f6e746 wip: fix import validation check 2024-12-02 17:21:53 -08:00
zach
783b32266a wip: improve error messages 2024-12-02 17:05:58 -08:00
zach
097079fab9 wip: check for missing functions when linking 2024-12-02 16:57:52 -08:00
zach
91b7cd7d3a fix: wiggle bounds 2024-12-02 16:57:52 -08:00
zach
198a63044e chore: fix clippy 2024-12-02 16:57:52 -08:00
33 changed files with 217 additions and 598 deletions

View File

@@ -9,8 +9,33 @@ updates:
directory: "/" # Location of package manifests
schedule:
interval: "weekly"
ignore:
- dependency-name: "wasmtime"
- dependency-name: "wasi-common"
- dependency-name: "wiggle"
- package-ecosystem: "pip"
directory: "python"
schedule:
interval: "weekly"
- package-ecosystem: "mix"
directory: "elixir"
schedule:
interval: "weekly"
- package-ecosystem: "npm"
directory: "node"
schedule:
interval: "weekly"
- package-ecosystem: "composer"
directory: "/"
schedule:
interval: "weekly"
- package-ecosystem: "bundler"
directory: "ruby"
schedule:
interval: "weekly"
- package-ecosystem: "gomod"
directory: "/"
schedule:
interval: "weekly"

View File

@@ -52,7 +52,7 @@ jobs:
shell: bash
run: cargo build --release -p ${{ env.LIBEXTISM_CRATE }}
- name: Upload artifact
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: libextism-${{ matrix.os }}
path: |

View File

@@ -1,6 +1,4 @@
on:
release:
types: [published, edited]
workflow_dispatch:
name: Release .NET Native NuGet Packages
@@ -20,13 +18,10 @@ jobs:
uses: actions/setup-dotnet@v3.0.3
with:
dotnet-version: 7.x
- name: download release
run: |
tag='${{ github.ref }}'
tag="${tag/refs\/tags\//}"
gh release download "$tag" -p 'libextism-*.tar.gz'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- uses: dawidd6/action-download-artifact@v6
with:
workflow: release.yml
name: release-artifacts
- name: Extract Archive
run: |
extract_archive() {

View File

@@ -7,6 +7,11 @@ on:
name: Release
env:
RUNTIME_MANIFEST: runtime/Cargo.toml
RUNTIME_CRATE: libextism
RUSTFLAGS: -C target-feature=-crt-static
ARTIFACT_DIR: release-artifacts
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@@ -16,11 +21,6 @@ jobs:
release:
name: ${{ matrix.os }} ${{ matrix.target }}
runs-on: ${{ matrix.os }}-latest
env:
RUNTIME_MANIFEST: runtime/Cargo.toml
RUNTIME_CRATE: libextism
RUSTFLAGS: -C target-feature=-crt-static
ARTIFACT_DIR: release-artifacts-${{ matrix.os }}-${{ matrix.target }}
strategy:
matrix:
include:
@@ -98,11 +98,13 @@ jobs:
pyproject="$(cat extism-maturin/pyproject.toml)"
<<<"$pyproject" >extism-maturin/pyproject.toml sed -e 's/^version = "0.0.0.replaced-by-ci"/version = "'"$version"'"/g'
- uses: actions-rust-lang/setup-rust-toolchain@v1
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
target: ${{ matrix.target }}
override: true
toolchain: stable
profile: minimal
override: true
target: ${{ matrix.target }}
- uses: Swatinem/rust-cache@v2
with:
@@ -111,15 +113,11 @@ jobs:
cache-on-failure: "true"
- name: Build Target (${{ matrix.os }} ${{ matrix.target }})
if: ${{ matrix.os != 'windows' }}
run: |
cargo install cross
cross build --release --target ${{ matrix.target }} -p ${{ env.RUNTIME_CRATE }}
- name: Build Target (${{ matrix.os }} ${{ matrix.target }})
if: ${{ matrix.os == 'windows' }}
run: |
cargo build --release --target ${{ matrix.target }} -p ${{ env.RUNTIME_CRATE }}
uses: actions-rs/cargo@v1
with:
use-cross: ${{ matrix.os != 'windows' }}
command: build
args: --release --target ${{ matrix.target }} -p ${{ env.RUNTIME_CRATE }}
- uses: actions/setup-python@v4
with:
@@ -192,7 +190,7 @@ jobs:
ls -ll ${DEST_DIR}
- name: Upload Artifact to Summary
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: ${{ env.ARTIFACT_DIR }}
path: ${{ env.ARTIFACT_DIR }}
@@ -211,10 +209,9 @@ jobs:
needs: [release]
if: github.ref == 'refs/heads/main'
steps:
- uses: actions/download-artifact@v4
- uses: actions/download-artifact@v3
with:
pattern: release-artifacts-*
merge-multiple: true
name: ${{ env.ARTIFACT_DIR }}
- uses: "marvinpinto/action-automatic-releases@latest"
with:

View File

@@ -36,22 +36,6 @@ function linking, and more. Extism users build:
- web applications
- & much more...
# Supported Targets
We currently provide releases for the following targets:
- aarch64-apple-darwin
- aarch64-unknown-linux-gnu
- aarch64-unknown-linux-musl
- x86_64-apple-darwin
- x86_64-pc-windows-gnu
- x86_64-pc-windows-msvc
- x86_64-unknown-linux-gnu
- x86_64-unknown-linux-musl
For Android we suggest taking a look at the [Chicory SDK](https://github.com/extism/chicory-sdk) for a pure Java
Extism runtime.
# Run WebAssembly In Your App
Pick a SDK to import into your program, and refer to the documentation to get

View File

@@ -34,11 +34,11 @@ fn extract_encoding(attrs: &[Attribute]) -> Result<Path> {
.iter()
.filter(|attr| attr.path().is_ident("encoding"))
.collect();
ensure!(!encodings.is_empty(), "encoding needs to be specified"; try = "`#[encoding(Json)]`");
ensure!(!encodings.is_empty(), "encoding needs to be specified"; try = "`#[encoding(ToJson)]`");
ensure!(encodings.len() < 2, encodings[1], "only one encoding can be specified"; try = "remove `{}`", encodings[1].to_token_stream());
Ok(encodings[0].parse_args().map_err(
|e| error_message!(e.span(), "{e}"; note= "expects a path"; try = "`#[encoding(Json)]`"),
|e| error_message!(e.span(), "{e}"; note= "expects a path"; try = "`#[encoding(ToJson)]`"),
)?)
}

View File

@@ -1,6 +1,6 @@
error: encoding needs to be specified
= try: `#[encoding(Json)]`
= try: `#[encoding(ToJson)]`
--> tests/ui/invalid-encoding.rs:3:10
|
3 | #[derive(ToBytes)]
@@ -11,7 +11,7 @@ error: encoding needs to be specified
error: expected attribute arguments in parentheses: #[encoding(...)]
= note: expects a path
= try: `#[encoding(Json)]`
= try: `#[encoding(ToJson)]`
--> tests/ui/invalid-encoding.rs:7:3
|
7 | #[encoding]
@@ -20,7 +20,7 @@ error: expected attribute arguments in parentheses: #[encoding(...)]
error: expected parentheses: #[encoding(...)]
= note: expects a path
= try: `#[encoding(Json)]`
= try: `#[encoding(ToJson)]`
--> tests/ui/invalid-encoding.rs:11:12
|
11 | #[encoding = "string"]
@@ -29,7 +29,7 @@ error: expected parentheses: #[encoding(...)]
error: unexpected token
= note: expects a path
= try: `#[encoding(Json)]`
= try: `#[encoding(ToJson)]`
--> tests/ui/invalid-encoding.rs:15:21
|
15 | #[encoding(something, else)]

View File

@@ -13,7 +13,7 @@ description = "Traits to make Rust types usable with Extism"
anyhow = "1.0.75"
base64 = "~0.22"
bytemuck = {version = "1.14.0", optional = true }
prost = { version = "0.14.1", optional = true }
prost = { version = "0.13.1", optional = true }
protobuf = { version = "3.2.0", optional = true }
rmp-serde = { version = "1.1.2", optional = true }
serde = "1.0.186"

View File

@@ -141,16 +141,6 @@ impl FromBytesOwned for u32 {
}
}
impl FromBytesOwned for bool {
fn from_bytes_owned(data: &[u8]) -> Result<Self, Error> {
if let Some(x) = data.first() {
Ok(*x != 0)
} else {
Err(Error::msg("Expected one byte to read boolean value"))
}
}
}
impl FromBytesOwned for () {
fn from_bytes_owned(_: &[u8]) -> Result<Self, Error> {
Ok(())

View File

@@ -61,17 +61,6 @@ fn rountrip_option() {
assert_eq!(y.unwrap().0, z.unwrap().0);
}
#[test]
fn check_bool() {
// `None` case
let a = true.to_bytes().unwrap();
let b = false.to_bytes().unwrap();
assert_ne!(a, b);
assert_eq!(a, [1]);
assert_eq!(b, [0]);
}
#[cfg(all(feature = "raw", target_endian = "little"))]
mod raw_tests {
use crate::*;

View File

@@ -144,14 +144,6 @@ impl ToBytes<'_> for u32 {
}
}
impl ToBytes<'_> for bool {
type Bytes = [u8; 1];
fn to_bytes(&self) -> Result<Self::Bytes, Error> {
Ok([*self as u8])
}
}
impl<'a, T: ToBytes<'a>> ToBytes<'a> for &'a T {
type Bytes = T::Bytes;

View File

@@ -9,9 +9,9 @@ repository.workspace = true
version.workspace = true
[dependencies]
wasmtime = {version = ">= 27.0.0, < 31.0.0"}
wasi-common = {version = ">= 27.0.0, < 31.0.0"}
wiggle = {version = ">= 27.0.0, < 31.0.0"}
wasmtime = {version = ">= 26.0.0, < 27.0.0"}
wasi-common = {version = ">= 26.0.0, < 27.0.0"}
wiggle = {version = ">= 26.0.0, < 27.0.0"}
anyhow = "1"
serde = {version = "1", features = ["derive"]}
serde_json = "1"
@@ -21,7 +21,7 @@ tracing = "0.1"
tracing-subscriber = {version = "0.3.18", features = ["std", "env-filter", "fmt"]}
url = "2"
glob = "0.3"
ureq = {version = "3.0", optional=true}
ureq = {version = "2.5", optional=true}
extism-manifest = { workspace = true }
extism-convert = { workspace = true, features = ["extism-path"] }
uuid = { version = "1", features = ["v4"] }
@@ -34,12 +34,12 @@ register-filesystem = [] # enables wasm to be loaded from disk
http = ["ureq"] # enables extism_http_request
[build-dependencies]
cbindgen = { version = "0.29", default-features = false }
cbindgen = { version = "0.27", default-features = false }
[dev-dependencies]
criterion = "0.6.0"
criterion = "0.5.1"
quickcheck = "1"
rand = "0.9.0"
rand = "0.8.5"
[[bench]]
name = "bench"

View File

@@ -24,7 +24,7 @@ There are a few environment variables that can be used for debugging purposes:
- `EXTISM_COREDUMP=extism.core`: write [coredump](https://github.com/WebAssembly/tool-conventions/blob/main/Coredump.md) to a file when a WebAssembly function traps
- `EXTISM_DEBUG=1`: generate debug information
- `EXTISM_PROFILE=perf|jitdump|vtune`: enable Wasmtime profiling
- `EXTISM_CACHE_CONFIG=path/to/config.toml`: enable Wasmtime cache, details [here](#wasmtime-caching)
- `EXTISM_CACHE_CONFIG=path/to/config.toml`: enable Wasmtime cache, see [the docs](https://docs.wasmtime.dev/cli-cache.html) for details about configuration. Setting this to an empty string will disable caching.
> *Note*: The debug and coredump info will only be written if the plug-in has an error.
@@ -88,7 +88,7 @@ println!("{:?}", res);
### Plug-in State
Plug-ins may be stateful or stateless. Plug-ins can maintain state between calls by the use of variables. Our count vowels plug-in remembers the total number of vowels it's ever counted in the "total" key in the result. You can see this by making subsequent calls to the export:
Plug-ins may be stateful or stateless. Plug-ins can maintain state b/w calls by the use of variables. Our count vowels plug-in remembers the total number of vowels it's ever counted in the "total" key in the result. You can see this by making subsequent calls to the export:
```rust
let res = plugin.call::<&str, &str>("count_vowels", "Hello, world!").unwrap();
@@ -112,8 +112,7 @@ let mut plugin = Plugin::new(&manifest, [], true);
let res = plugin.call::<&str, &str>("count_vowels", "Yellow, world!").unwrap();
println!("{}", res);
# => {"count": 3, "total": 3, "vowels": "aeiouAEIOU"}
let manifest = Manifest::new([url]).with_config_key("vowels", "aeiouyAEIOUY");
let mut plugin = Plugin::new(&manifest, [], true).unwrap();
let mut plugin = Plugin::new(&manifest, [], true).with_config_key("vowels", "aeiouyAEIOUY");
let res = plugin.call::<&str, &str>("count_vowels", "Yellow, world!").unwrap();
println!("{}", res);
# => {"count": 4, "total": 4, "vowels": "aeiouyAEIOUY"}
@@ -220,52 +219,5 @@ println!("{}", res);
# => {"count": 3, "total": 6, "vowels": "aeiouAEIOU"}
```
### Logging
Plug-ins can't directly print anything to the console. They can however use Extism's built-in logging functionality, for example the [`log!` macro in the rust-pdk](https://docs.rs/extism-pdk/1.4.0/extism_pdk/macro.log.html) or the [`logInfo` function in the haskell-pdk](https://hackage.haskell.org/package/extism-pdk-1.2.0.0/docs/Extism-PDK.html#v:logInfo).
Inside your host application, the rust-sdk emits these as [tracing](https://github.com/tokio-rs/tracing) events. The simplest way to make the logged messages visible is by adding the `tracing_subscriber` dependency to your crate and then initializing a tracing subscriber at the top of your main function:
```rust
tracing_subscriber::fmt::init();
```
### Wasmtime Caching
To enable or disable caching for plugin compilation, you need to provide a configuration file that will be used by the [wasmtime crate](https://github.com/bytecodealliance/wasmtime).
For more information and values that can be used for configuring caching, take a look at [the docs](https://docs.wasmtime.dev/cli-cache.html).
> *Note*: As of now extism uses wasmtime [`version = ">= 27.0.0, < 31.0.0"`](https://github.com/extism/extism/blob/v1.11.1/runtime/Cargo.toml#L12), but the `enabled` key requirement [was removed](https://github.com/bytecodealliance/wasmtime/pull/10859) from `wasmtime` and its documentation, this could explain the `failed to parse config file` error you might encounter without it.
An example configuration for caching would be:
```toml
[cache]
enabled = true # This value is required
directory = "/some/path"
```
You can :
- [Create a global `wasmtime` configuration file](#using-a-configuration-file) in `$HOME/.config/wasmtime/config.toml`.
- [Set the `EXTISM_CACHE_CONFIG` environment variable](#using-an-environment-variable)
- [Set the configuration file path using `PluginBuilder`](#using-pluginbuilder)
#### Using a configuration file
The [wasmtime](https://github.com/bytecodealliance/wasmtime) crate, by default, will look for a configuration file in your systems' default configuration directory (for example on UNIX systems: `$HOME/.config/wasmtime/config.toml`),
for more [information on this behaviour](`https://docs.rs/wasmtime/31.0.0/wasmtime/struct.Config.html#method.cache_config_load_default`).
#### Using an environment variable
You can set the `EXTISM_CACHE_CONFIG=path/to/config.toml` environment variable to set the path of the configuration file used by [wasmtime](https://github.com/bytecodealliance/wasmtime).
Setting the variable to an empty string will disable caching (it won't load any configuration file).
> *Note*: If the environment variable is not set, `wasmtime` will still try to read from a configuration file that may exist in your system's default configuration folder (e.g. `$HOME/.config/wasmtime/config.toml`).
The environment variable does not override the path you might have set using `PluginBuilder`. will only be checked for if you did not specify a cache configuration path in `PluginBuilder`.
#### Using PluginBuilder
If you use a [PluginBuilder](https://docs.rs/extism/latest/extism/struct.PluginBuilder.html), you can set the `wasmtime` configuration path using the [with_cache_config](https://docs.rs/extism/latest/extism/struct.PluginBuilder.html#method.with_cache_config) method.
This will override the `EXTISM_CACHE_CONFIG` environment variable if it's set, so you could have a "global" and per plugin configuration if needed.

View File

@@ -16,7 +16,7 @@ fn main() {
let res = plugin.call::<&str, &str>("try_read", "").unwrap();
println!("{res:?}");
println!("{:?}", res);
println!("-----------------------------------------------------");
@@ -30,7 +30,7 @@ fn main() {
);
let res2 = plugin.call::<&str, &str>("try_write", &line).unwrap();
println!("{res2:?}");
println!("{:?}", res2);
println!("done!");
}

View File

@@ -30,6 +30,6 @@ fn main() {
let res = plugin
.call::<&str, &str>("reflect", "Hello, world!")
.unwrap();
println!("{res}");
println!("{}", res);
}
}

View File

@@ -25,6 +25,6 @@ fn main() {
println!("Dumping logs");
for line in LOGS.lock().unwrap().iter() {
print!("{line}");
print!("{}", line);
}
}

View File

@@ -52,6 +52,6 @@ fn main() {
let res = plugin
.call::<&str, &str>("count_vowels", "Hello, world!")
.unwrap();
println!("{res}");
println!("{}", res);
}
}

View File

@@ -156,7 +156,7 @@ void extism_current_plugin_memory_free(ExtismCurrentPlugin *plugin, ExtismMemory
* - `n_outputs`: number of return types
* - `func`: the function to call
* - `user_data`: a pointer that will be passed to the function when it's called
* this value should live as long as the function exists
* this value should live as long as the function exists
* - `free_user_data`: a callback to release the `user_data` value when the resulting
* `ExtismFunction` is freed.
*

View File

@@ -206,16 +206,15 @@ impl CurrentPlugin {
anyhow::bail!("expected extism_context to be an externref value",)
};
if let Some(d) = xs.data_mut(&mut *store)? {
match d.downcast_mut::<Box<dyn std::any::Any + Send + Sync>>() {
Some(xs) => match xs.downcast_mut::<T>() {
Some(xs) => Ok(xs),
None => anyhow::bail!("could not downcast extism_context inner value"),
},
None => anyhow::bail!("could not downcast extism_context"),
}
} else {
anyhow::bail!("extism_context not found")
match xs
.data_mut(&mut *store)?
.downcast_mut::<Box<dyn std::any::Any + Send + Sync>>()
{
Some(xs) => match xs.downcast_mut::<T>() {
Some(xs) => Ok(xs),
None => anyhow::bail!("could not downcast extism_context inner value"),
},
None => anyhow::bail!("could not downcast extism_context"),
}
}
@@ -477,7 +476,10 @@ impl CurrentPlugin {
offset: offs,
length,
});
s.ok()
match s {
Ok(s) => Some(s),
Err(_) => None,
}
}
#[doc(hidden)]

View File

@@ -284,8 +284,8 @@ impl Function {
/// A few things worth noting:
/// - The function always returns a `Result` that wraps the specified return type
/// - If a first parameter and type are passed (`_user_data` above) followed by a semicolon it will be
/// the name of the `UserData` parameter and can be used from inside the function
// definition.
/// the name of the `UserData` parameter and can be used from inside the function
// definition.
#[macro_export]
macro_rules! host_fn {
($pub:vis $name: ident ($($arg:ident : $argty:ty),*) $(-> $ret:ty)? $b:block) => {

View File

@@ -29,7 +29,6 @@ pub(crate) mod manifest;
pub(crate) mod pdk;
mod plugin;
mod plugin_builder;
mod pool;
mod readonly_dir;
mod timer;
@@ -44,7 +43,6 @@ pub use plugin::{
CancelHandle, CompiledPlugin, Plugin, WasmInput, EXTISM_ENV_MODULE, EXTISM_USER_MODULE,
};
pub use plugin_builder::{DebugOptions, PluginBuilder};
pub use pool::{Pool, PoolBuilder, PoolPlugin};
pub(crate) use internal::{Internal, Wasi};
pub(crate) use timer::{Timer, TimerAction};
@@ -98,7 +96,7 @@ pub fn set_log_callback<F: 'static + Clone + Fn(&str)>(
let x = tracing_subscriber::EnvFilter::builder()
.with_default_directive(tracing::Level::ERROR.into());
if is_level {
x.parse_lossy(format!("extism={filter}"))
x.parse_lossy(format!("extism={}", filter))
} else {
x.parse_lossy(filter)
}

View File

@@ -10,7 +10,7 @@ use crate::*;
fn hex(data: &[u8]) -> String {
let mut s = String::new();
for &byte in data {
write!(&mut s, "{byte:02x}").unwrap();
write!(&mut s, "{:02x}", byte).unwrap();
}
s
}
@@ -86,16 +86,14 @@ fn to_module(engine: &Engine, wasm: &extism_manifest::Wasm) -> Result<(String, M
#[cfg(feature = "register-http")]
{
// Setup request
let mut req = ureq::http::request::Builder::new()
.method(method.as_deref().unwrap_or("GET").to_uppercase().as_str())
.uri(url);
let mut req = ureq::request(method.as_deref().unwrap_or("GET"), url);
for (k, v) in headers.iter() {
req = req.header(k, v);
req = req.set(k, v);
}
// Fetch WASM code
let mut r = ureq::run(req.body(())?)?.into_body().into_reader();
let mut r = req.call()?.into_reader();
let mut data = Vec::new();
r.read_to_end(&mut data)?;

View File

@@ -233,22 +233,17 @@ pub(crate) fn http_request(
)));
}
let mut r = ureq::http::request::Builder::new()
.method(
req.method
.as_deref()
.unwrap_or("GET")
.to_uppercase()
.as_str(),
)
.uri(&req.url);
let mut r = ureq::request(req.method.as_deref().unwrap_or("GET"), &req.url);
for (k, v) in req.headers.iter() {
r = r.header(k, v);
r = r.set(k, v);
}
// Set HTTP timeout to respect the manifest timeout
let timeout = data.time_remaining();
if let Some(remaining) = data.time_remaining() {
r = r.timeout(remaining);
}
let res = if body_offset > 0 {
let handle = match data.memory_handle(body_offset) {
Some(h) => h,
@@ -257,15 +252,9 @@ pub(crate) fn http_request(
}
};
let buf: &[u8] = data.memory_bytes(handle)?;
let agent = ureq::agent();
let config = agent.configure_request(r.body(buf)?);
let req = config.timeout_global(timeout).build();
ureq::run(req)
r.send_bytes(buf)
} else {
let agent = ureq::agent();
let config = agent.configure_request(r.body(())?);
let req = config.timeout_global(timeout).build();
ureq::run(req)
r.call()
};
if let Some(handle) = data.memory_handle(body_offset) {
@@ -275,26 +264,26 @@ pub(crate) fn http_request(
let reader = match res {
Ok(res) => {
if let Some(headers) = &mut data.http_headers {
for (name, h) in res.headers() {
if let Ok(h) = h.to_str() {
headers.insert(name.as_str().to_string(), h.to_string());
for name in res.headers_names() {
if let Some(h) = res.header(&name) {
headers.insert(name, h.to_string());
}
}
}
data.http_status = res.status().as_u16();
Some(res.into_body().into_reader())
data.http_status = res.status();
Some(res.into_reader())
}
Err(e) => {
// Catch timeout and return
if let Some(d) = data.time_remaining() {
if matches!(e, ureq::Error::Timeout(_)) && d.as_nanos() == 0 {
if e.kind() == ureq::ErrorKind::Io && d.as_nanos() == 0 {
anyhow::bail!("timeout");
}
}
let msg = e.to_string();
if let ureq::Error::StatusCode(res) = e {
data.http_status = res;
None
if let Some(res) = e.into_response() {
data.http_status = res.status();
Some(res.into_reader())
} else {
return Err(Error::msg(msg));
}

View File

@@ -1,7 +1,6 @@
use std::{
any::Any,
collections::{BTreeMap, BTreeSet},
sync::TryLockError,
collections::{BTreeMap, BTreeSet, HashMap},
};
use anyhow::Context;
@@ -191,7 +190,6 @@ pub(crate) fn profiling_strategy() -> ProfilingStrategy {
/// Defines an input type for Wasm data.
///
/// Types that implement `Into<WasmInput>` can be passed directly into `Plugin::new`
#[derive(Clone)]
pub enum WasmInput<'a> {
/// Raw Wasm module
Data(std::borrow::Cow<'a, [u8]>),
@@ -244,7 +242,7 @@ impl<'a> From<&'a Vec<u8>> for WasmInput<'a> {
}
fn add_module<T: 'static>(
store: &mut Store<T>,
mut store: &mut Store<T>,
linker: &mut Linker<T>,
linked: &mut BTreeSet<String>,
modules: &BTreeMap<String, Module>,
@@ -255,6 +253,59 @@ fn add_module<T: 'static>(
return Ok(());
}
let mut imports: HashMap<&str, Vec<(&str, ExternType)>> = HashMap::new();
for module_import in module.imports() {
let import_module = module_import.module();
let import_name = module_import.name();
let import_type = module_import.ty();
if imports.contains_key(import_module) {
imports
.get_mut(import_module)
.unwrap()
.push((import_name, import_type));
} else {
imports.insert(import_module, vec![(import_name, import_type)]);
}
}
for (m, v) in imports.into_iter() {
if let Some(src) = modules.get(m) {
for (name, ty) in v {
match src
.get_export(name)
.or_else(|| linker.get(&mut store, m, name).map(|x| x.ty(&store)))
{
None => anyhow::bail!("missing import: {m}::{name}"),
Some(ex) => match (&ex, &ty) {
(ExternType::Func(a), ExternType::Func(b)) => {
if !a.matches(b) {
anyhow::bail!(
"function type mismatch {m}::{name}, got {ty:?} expected {ex:?}"
)
}
}
(ExternType::Global(a), ExternType::Global(b)) => {
if !a.content().matches(b.content()) {
anyhow::bail!(
"global type mismatch {m}::{name}, got {ty:?} expected {ex:?}"
)
}
}
(ExternType::Memory(_), ExternType::Memory(_)) => (),
(ExternType::Table(a), ExternType::Table(b)) => {
if !a.element().matches(b.element()) {
anyhow::bail!(
"table type mismatch {m}::{name}, got {ty:?} expected {ex:?}"
)
}
}
(a, b) => anyhow::bail!("import type mismatch: {a:?} != {b:?}"),
},
}
}
}
}
for import in module.imports() {
let module = import.module();
@@ -327,36 +378,6 @@ fn relink(
get_log_level() -> I32;
);
for (name, module) in modules.iter() {
if name == EXTISM_ENV_MODULE {
continue;
}
for import in module.imports() {
if import.module() == EXTISM_ENV_MODULE
&& modules[EXTISM_ENV_MODULE]
.get_export(import.name())
.is_none()
&& linker
.get(&mut store, EXTISM_ENV_MODULE, import.name())
.is_none()
{
let (kind, ty) = match import.ty() {
ExternType::Func(t) => ("function", t.to_string()),
ExternType::Global(t) => ("global", t.content().to_string()),
ExternType::Table(t) => ("table", t.element().to_string()),
ExternType::Memory(_) => ("memory", String::new()),
};
anyhow::bail!(
"Invalid {kind} import from extism:host/env: {} {ty}\n\n\
Note: This may indicate that the PDK that was used to build this plugin has additional features that aren't \
available in this version of the SDK, try updating the SDK to the latest version.",
import.name(),
)
}
}
}
let mut linked = BTreeSet::new();
linker.module(&mut store, EXTISM_ENV_MODULE, &modules[EXTISM_ENV_MODULE])?;
linked.insert(EXTISM_ENV_MODULE.to_string());
@@ -857,20 +878,15 @@ impl Plugin {
// Set host context
let r = if let Some(host_context) = host_context {
if let Some(inner) = self
let inner = self
.host_context
.data_mut(&mut self.store)
.map_err(|x| (x, -1))?
{
if let Some(inner) = inner.downcast_mut::<Box<dyn std::any::Any + Send + Sync>>() {
let x: Box<T> = Box::new(host_context);
*inner = x;
}
Some(self.host_context)
} else {
None
.map_err(|x| (x, -1))?;
if let Some(inner) = inner.downcast_mut::<Box<dyn std::any::Any + Send + Sync>>() {
let x: Box<T> = Box::new(host_context);
*inner = x;
}
Some(self.host_context)
} else {
None
};
@@ -913,7 +929,7 @@ impl Plugin {
let mut res = func.call(self.store_mut(), &[], results.as_mut_slice());
// Reset host context
if let Ok(Some(inner)) = self.host_context.data_mut(&mut self.store) {
if let Ok(inner) = self.host_context.data_mut(&mut self.store) {
if let Some(inner) = inner.downcast_mut::<Box<dyn std::any::Any + Send + Sync>>() {
let x: Box<dyn Any + Send + Sync> = Box::new(());
*inner = x;
@@ -963,7 +979,8 @@ impl Plugin {
}
Err(msg) => {
res = Err(Error::msg(format!(
"unable to load error message from memory: {msg}",
"unable to load error message from memory: {}",
msg,
)));
}
}
@@ -1089,12 +1106,7 @@ impl Plugin {
input: T,
) -> Result<U, Error> {
let lock = self.instance.clone();
let mut lock = lock.try_lock().map_err(|e| match e {
TryLockError::Poisoned(_) => anyhow::anyhow!(
"instance lock was poisoned; previous thread panicked while calling into wasm"
),
TryLockError::WouldBlock => anyhow::anyhow!("cannot make reentrant calls into plugin"),
})?;
let mut lock = lock.lock().unwrap();
let data = input.to_bytes()?;
self.raw_call(&mut lock, name, data, None::<()>)
.map_err(|e| e.0)
@@ -1119,12 +1131,7 @@ impl Plugin {
C: Any + Send + Sync + 'static,
{
let lock = self.instance.clone();
let mut lock = lock.try_lock().map_err(|e| match e {
TryLockError::Poisoned(_) => anyhow::anyhow!(
"instance lock was poisoned; previous thread panicked while calling into wasm"
),
TryLockError::WouldBlock => anyhow::anyhow!("cannot make reentrant calls into plugin"),
})?;
let mut lock = lock.lock().unwrap();
let data = input.to_bytes()?;
self.raw_call(&mut lock, name, data, Some(host_context))
.map_err(|e| e.0)
@@ -1143,18 +1150,7 @@ impl Plugin {
input: T,
) -> Result<U, (Error, i32)> {
let lock = self.instance.clone();
let mut lock = lock.try_lock().map_err(|e| match e {
TryLockError::Poisoned(_) => (
anyhow::anyhow!(
"instance lock was poisoned; previous thread panicked while calling into wasm"
),
-1,
),
TryLockError::WouldBlock => (
anyhow::anyhow!("cannot make reentrant calls into plugin"),
-1,
),
})?;
let mut lock = lock.lock().unwrap();
let data = input.to_bytes().map_err(|e| (e, -1))?;
self.raw_call(&mut lock, name, data, None::<()>)
.and_then(move |_| self.output().map_err(|e| (e, -1)))
@@ -1182,25 +1178,6 @@ impl Plugin {
anyhow::bail!("Plugin::clear_error failed, extism:host/env::error_set not found")
}
}
/// Returns the amount of fuel consumed by the plugin.
///
/// This function calculates the difference between the initial fuel and the remaining fuel.
/// If either the initial fuel or the remaining fuel is not set, it returns `None`.
///
/// # Returns
///
/// * `Some(u64)` - The amount of fuel consumed.
/// * `None` - If the initial fuel or remaining fuel is not set.
pub fn fuel_consumed(&self) -> Option<u64> {
self.fuel.map(|x| {
x.saturating_sub(
self.store
.get_fuel()
.expect("fuel support should be enabled to use fuel"),
)
})
}
}
// Enumerates the PDK languages that need some additional initialization

View File

@@ -33,7 +33,6 @@ impl Default for DebugOptions {
}
/// PluginBuilder is used to configure and create `Plugin` instances
#[derive(Clone)]
pub struct PluginBuilder<'a> {
pub(crate) source: WasmInput<'a>,
pub(crate) config: Option<wasmtime::Config>,

View File

@@ -1,174 +0,0 @@
use crate::{Error, FromBytesOwned, Plugin, ToBytes};
// `PoolBuilder` is used to configure and create `Pool`s
#[derive(Debug, Clone)]
pub struct PoolBuilder {
/// Max number of concurrent instances for a plugin - by default this is set to
/// the output of `std::thread::available_parallelism`
pub max_instances: usize,
}
impl PoolBuilder {
/// Create a `PoolBuilder` with default values
pub fn new() -> Self {
Self::default()
}
/// Set the max number of parallel instances
pub fn with_max_instances(mut self, n: usize) -> Self {
self.max_instances = n;
self
}
/// Create a new `Pool` with the given configuration
pub fn build<F: 'static + Fn() -> Result<Plugin, Error>>(self, source: F) -> Pool {
Pool::new_from_builder(source, self)
}
}
impl Default for PoolBuilder {
fn default() -> Self {
PoolBuilder {
max_instances: std::thread::available_parallelism()
.expect("available parallelism")
.into(),
}
}
}
/// `PoolPlugin` is used by the pool to track the number of live instances of a particular plugin
#[derive(Clone, Debug)]
pub struct PoolPlugin(std::rc::Rc<std::cell::RefCell<Plugin>>);
impl PoolPlugin {
fn new(plugin: Plugin) -> Self {
Self(std::rc::Rc::new(std::cell::RefCell::new(plugin)))
}
/// Access the underlying plugin
pub fn plugin(&self) -> std::cell::RefMut<Plugin> {
self.0.borrow_mut()
}
/// Helper to call a plugin function on the underlying plugin
pub fn call<'a, Input: ToBytes<'a>, Output: FromBytesOwned>(
&self,
name: impl AsRef<str>,
input: Input,
) -> Result<Output, Error> {
self.plugin().call(name.as_ref(), input)
}
/// Helper to get the underlying plugin's ID
pub fn id(&self) -> uuid::Uuid {
self.plugin().id
}
}
type PluginSource = dyn Fn() -> Result<Plugin, Error>;
struct PoolInner {
plugin_source: Box<PluginSource>,
instances: Vec<PoolPlugin>,
}
unsafe impl Send for PoolInner {}
unsafe impl Sync for PoolInner {}
/// `Pool` manages threadsafe access to a limited number of instances of multiple plugins
#[derive(Clone)]
pub struct Pool {
config: PoolBuilder,
inner: std::sync::Arc<std::sync::Mutex<PoolInner>>,
}
unsafe impl Send for Pool {}
unsafe impl Sync for Pool {}
impl Pool {
/// Create a new pool with the default configuration
pub fn new<F: 'static + Fn() -> Result<Plugin, Error>>(source: F) -> Self {
Pool {
config: Default::default(),
inner: std::sync::Arc::new(std::sync::Mutex::new(PoolInner {
plugin_source: Box::new(source),
instances: Default::default(),
})),
}
}
/// Create a new pool configured using a `PoolBuilder`
pub fn new_from_builder<F: 'static + Fn() -> Result<Plugin, Error>>(
source: F,
builder: PoolBuilder,
) -> Self {
Pool {
config: builder,
inner: std::sync::Arc::new(std::sync::Mutex::new(PoolInner {
plugin_source: Box::new(source),
instances: Default::default(),
})),
}
}
fn find_available(&self) -> Result<Option<PoolPlugin>, Error> {
let pool = self.inner.lock().unwrap();
for instance in pool.instances.iter() {
if std::rc::Rc::strong_count(&instance.0) == 1 {
return Ok(Some(instance.clone()));
}
}
Ok(None)
}
/// Get the number of live instances for a plugin
pub fn count(&self) -> usize {
self.inner.lock().unwrap().instances.len()
}
/// Get access to a plugin, this will create a new instance if needed (and allowed by the specified
/// max_instances). `Ok(None)` is returned if the timeout is reached before an available plugin could be
/// acquired
pub fn get(&self, timeout: std::time::Duration) -> Result<Option<PoolPlugin>, Error> {
let start = std::time::Instant::now();
let max = self.config.max_instances;
if let Some(avail) = self.find_available()? {
return Ok(Some(avail));
}
{
let mut pool = self.inner.lock().unwrap();
if pool.instances.len() < max {
let plugin = (*pool.plugin_source)()?;
let instance = PoolPlugin::new(plugin);
pool.instances.push(instance);
return Ok(Some(pool.instances.last().unwrap().clone()));
}
}
loop {
if let Ok(Some(x)) = self.find_available() {
return Ok(Some(x));
}
if std::time::Instant::now() - start > timeout {
return Ok(None);
}
std::thread::sleep(std::time::Duration::from_millis(100));
}
}
/// Access a plugin in a callback function. This calls `Pool::get` then the provided
/// callback. `Ok(None)` is returned if the timeout is reached before an available
/// plugin could be acquired
pub fn with_plugin<T>(
&self,
timeout: std::time::Duration,
f: impl FnOnce(&mut Plugin) -> Result<T, Error>,
) -> Result<Option<T>, Error> {
if let Some(plugin) = self.get(timeout)? {
return f(&mut plugin.plugin()).map(Some);
}
Ok(None)
}
}

View File

@@ -180,7 +180,7 @@ pub unsafe extern "C" fn extism_current_plugin_memory_free(
/// - `n_outputs`: number of return types
/// - `func`: the function to call
/// - `user_data`: a pointer that will be passed to the function when it's called
/// this value should live as long as the function exists
/// this value should live as long as the function exists
/// - `free_user_data`: a callback to release the `user_data` value when the resulting
/// `ExtismFunction` is freed.
///
@@ -352,11 +352,8 @@ pub unsafe extern "C" fn extism_compiled_plugin_new(
.map(|v| Box::into_raw(Box::new(v)))
.unwrap_or_else(|e| {
if !errmsg.is_null() {
let e = std::ffi::CString::new(format!(
"Unable to compile Extism plugin: {}",
e.root_cause(),
))
.unwrap();
let e = std::ffi::CString::new(format!("Unable to compile Extism plugin: {}", e))
.unwrap();
*errmsg = e.into_raw();
}
std::ptr::null_mut()
@@ -429,11 +426,8 @@ pub unsafe extern "C" fn extism_plugin_new(
.map(|v| Box::into_raw(Box::new(v)))
.unwrap_or_else(|e| {
if !errmsg.is_null() {
let e = std::ffi::CString::new(format!(
"Unable to compile Extism plugin: {}",
e.root_cause(),
))
.unwrap();
let e = std::ffi::CString::new(format!("Unable to compile Extism plugin: {}", e))
.unwrap();
*errmsg = e.into_raw();
}
std::ptr::null_mut()
@@ -450,11 +444,8 @@ pub unsafe extern "C" fn extism_plugin_new_from_compiled(
match plugin {
Err(e) => {
if !errmsg.is_null() {
let e = std::ffi::CString::new(format!(
"Unable to create Extism plugin: {}",
e.root_cause(),
))
.unwrap();
let e = std::ffi::CString::new(format!("Unable to create Extism plugin: {}", e))
.unwrap();
*errmsg = e.into_raw();
}
std::ptr::null_mut()
@@ -521,11 +512,8 @@ pub unsafe extern "C" fn extism_plugin_new_with_fuel_limit(
Ok(x) => x,
Err(e) => {
if !errmsg.is_null() {
let e = std::ffi::CString::new(format!(
"Unable to compile Extism plugin: {}",
e.root_cause(),
))
.unwrap();
let e = std::ffi::CString::new(format!("Unable to compile Extism plugin: {}", e))
.unwrap();
*errmsg = e.into_raw();
}
return std::ptr::null_mut();
@@ -537,11 +525,8 @@ pub unsafe extern "C" fn extism_plugin_new_with_fuel_limit(
match plugin {
Err(e) => {
if !errmsg.is_null() {
let e = std::ffi::CString::new(format!(
"Unable to create Extism plugin: {}",
e.root_cause(),
))
.unwrap();
let e = std::ffi::CString::new(format!("Unable to create Extism plugin: {}", e))
.unwrap();
*errmsg = e.into_raw();
}
std::ptr::null_mut()
@@ -871,7 +856,7 @@ fn set_log_file(log_file: impl Into<std::path::PathBuf>, filter: &str) -> Result
let x = tracing_subscriber::EnvFilter::builder()
.with_default_directive(tracing::Level::ERROR.into());
if is_level {
x.parse_lossy(format!("extism={filter}"))
x.parse_lossy(format!("extism={}", filter))
} else {
x.parse_lossy(filter)
}
@@ -926,7 +911,7 @@ unsafe fn set_log_buffer(filter: &str) -> Result<(), Error> {
let x = tracing_subscriber::EnvFilter::builder()
.with_default_directive(tracing::Level::ERROR.into());
if is_level {
x.parse_lossy(format!("extism={filter}"))
x.parse_lossy(format!("extism={}", filter))
} else {
x.parse_lossy(filter)
}

View File

@@ -16,7 +16,7 @@ fn test_issue_620() {
// Call test method, this does not work
let p = plugin.call::<(), String>("test", ()).unwrap();
println!("{p}");
println!("{}", p);
}
// https://github.com/extism/extism/issues/619
@@ -53,5 +53,5 @@ fn test_issue_775() {
Ok(code) => Err(code),
}
.unwrap();
println!("{p}");
println!("{}", p);
}

View File

@@ -412,7 +412,7 @@ quickcheck! {
quickcheck! {
fn check_alloc_with_load_and_store(amounts: Vec<u16>) -> bool {
use rand::Rng;
let mut rng = rand::rng();
let mut rng = rand::thread_rng();
let (mut store, mut instance) = init_kernel_test();
let instance = &mut instance;
for a in amounts {
@@ -425,7 +425,7 @@ quickcheck! {
}
for _ in 0..16 {
let i = rng.random_range(ptr..ptr+a as u64);
let i = rng.gen_range(ptr..ptr+a as u64);
extism_store_u8(&mut store, instance, i, i as u8);
if extism_load_u8(&mut store, instance, i as u64) != i as u8 {
return false

View File

@@ -1,4 +1,3 @@
mod issues;
mod kernel;
mod pool;
mod runtime;

View File

@@ -1,48 +0,0 @@
use crate::*;
fn run_thread(p: Pool, i: u64) -> std::thread::JoinHandle<()> {
std::thread::spawn(move || {
std::thread::sleep(std::time::Duration::from_millis(i));
let s: String = p
.get(std::time::Duration::from_secs(1))
.unwrap()
.unwrap()
.call("count_vowels", "abc")
.unwrap();
println!("{s}");
})
}
#[test]
fn test_threads() {
for i in 1..=3 {
let data = include_bytes!("../../../wasm/code.wasm");
let plugin_builder =
extism::PluginBuilder::new(extism::Manifest::new([extism::Wasm::data(data)]))
.with_wasi(true);
let pool: Pool = PoolBuilder::new()
.with_max_instances(i)
.build(move || plugin_builder.clone().build());
let threads = vec![
run_thread(pool.clone(), 1000),
run_thread(pool.clone(), 1000),
run_thread(pool.clone(), 1000),
run_thread(pool.clone(), 1000),
run_thread(pool.clone(), 1000),
run_thread(pool.clone(), 1000),
run_thread(pool.clone(), 500),
run_thread(pool.clone(), 500),
run_thread(pool.clone(), 500),
run_thread(pool.clone(), 500),
run_thread(pool.clone(), 500),
run_thread(pool.clone(), 0),
];
for t in threads {
t.join().unwrap();
}
assert!(pool.count() <= i);
}
}

View File

@@ -134,7 +134,8 @@ fn it_works() {
let native_avg: std::time::Duration = native_sum / native_num_tests as u32;
println!(
"native function call (avg, N = {native_num_tests}): {native_avg:?}"
"native function call (avg, N = {}): {:?}",
native_num_tests, native_avg
);
let num_tests = test_times.len();
@@ -144,7 +145,7 @@ fn it_works() {
.unwrap();
let avg: std::time::Duration = sum / num_tests as u32;
println!("wasm function call (avg, N = {num_tests}): {avg:?}");
println!("wasm function call (avg, N = {}): {:?}", num_tests, avg);
// Check that log file was written to
if log {
@@ -211,7 +212,7 @@ fn test_cancel() {
let _output: Result<&[u8], Error> = plugin.call("loop_forever", "abc123");
let end = std::time::Instant::now();
let time = end - start;
println!("Cancelled plugin ran for {time:?}");
println!("Cancelled plugin ran for {:?}", time);
}
}
@@ -257,23 +258,6 @@ fn test_fuel() {
}
}
#[test]
fn test_fuel_consumption() {
let manifest = Manifest::new([extism_manifest::Wasm::data(WASM_LOOP)]);
let mut plugin = PluginBuilder::new(manifest)
.with_wasi(true)
.with_fuel_limit(10000)
.build()
.unwrap();
let output: Result<&[u8], Error> = plugin.call("loop_forever", "abc123");
assert!(output.is_err());
let fuel_consumed = plugin.fuel_consumed().unwrap();
println!("Fuel consumed: {fuel_consumed}");
assert!(fuel_consumed > 0);
}
#[test]
#[cfg(feature = "http")]
fn test_http_timeout() {
@@ -439,7 +423,7 @@ fn test_memory_max() {
assert!(output.is_err());
let err = output.unwrap_err().root_cause().to_string();
println!("{err:?}");
println!("{:?}", err);
assert_eq!(err, "oom");
// Should pass with memory.max set to a large enough number
@@ -502,7 +486,7 @@ fn test_extism_error() {
let mut plugin = Plugin::new(&manifest, [f], true).unwrap();
let output: Result<String, Error> = plugin.call("count_vowels", "a".repeat(1024));
assert!(output.is_err());
println!("{output:?}");
println!("{:?}", output);
assert_eq!(output.unwrap_err().root_cause().to_string(), "TEST");
}
@@ -822,7 +806,7 @@ fn test_http_response_headers() {
.unwrap();
let req = HttpRequest::new("https://extism.org");
let Json(res): Json<HashMap<String, String>> = plugin.call("http_get", Json(req)).unwrap();
println!("{res:?}");
println!("{:?}", res);
assert_eq!(res["content-type"], "text/html; charset=utf-8");
}
@@ -837,6 +821,6 @@ fn test_http_response_headers_disabled() {
.unwrap();
let req = HttpRequest::new("https://extism.org");
let Json(res): Json<HashMap<String, String>> = plugin.call("http_get", Json(req)).unwrap();
println!("{res:?}");
println!("{:?}", res);
assert!(res.is_empty());
}

View File

@@ -92,39 +92,25 @@ impl Timer {
loop {
if plugins.is_empty() {
if let Ok(x) = rx.recv() {
handle!(x);
handle!(x)
}
}
let mut timeout: Option<std::time::Duration> = None;
plugins.retain(|_k, (engine, end)| {
if let Some(end) = end {
let now = std::time::Instant::now();
if *end <= now {
engine.increment_epoch();
return false;
} else {
let time_left =
(*end - now).saturating_sub(std::time::Duration::from_millis(1));
if let Some(t) = &timeout {
if time_left < *t {
timeout = Some(time_left);
}
} else {
timeout = Some(time_left);
plugins = plugins
.into_iter()
.filter(|(_k, (engine, end))| {
if let Some(end) = end {
let now = std::time::Instant::now();
if end <= &now {
engine.increment_epoch();
return false;
}
}
}
true
})
.collect();
true
});
if let Some(timeout) = timeout {
if let Ok(x) = rx.recv_timeout(timeout) {
handle!(x)
}
} else if let Ok(x) = rx.recv() {
for x in rx.try_iter() {
handle!(x)
}
}