Compare commits

...

16 Commits

Author SHA1 Message Date
zach
e7127759a6 fix: private fields 2024-05-31 10:50:07 -07:00
zach
ed5bad922a cleanup: add ways to get output from WasiOutput 2024-05-31 10:43:48 -07:00
zach
5af226219e cleanup: require store reset on error 2024-05-31 10:23:28 -07:00
zach
b0e6387eef cleanup: enable wasi whenever run_wasi_command is used 2024-05-31 10:23:28 -07:00
zach
6a8e65eeb3 chore: clippy 2024-05-31 10:23:27 -07:00
zach
e8041dc5be doc: add comment 2024-05-31 10:22:41 -07:00
zach
433276274b cleanup: run_command 2024-05-31 10:22:40 -07:00
zach
a531e337fe feat: add Plugin::run_command 2024-05-31 10:20:44 -07:00
zach
ea29e080a6 cleanup: allow blocking thread 2024-05-31 10:17:55 -07:00
zach
c4312fe324 chore: updates for ExternRef changes upstream 2024-05-31 10:17:55 -07:00
zach
3ab6137ff1 chore: allow wastime 21 2024-05-31 10:17:55 -07:00
zach
fde222a6a0 cleanup: enable wasm gc 2024-05-31 10:17:55 -07:00
zach
ec4d45fb88 chore: use wasmtime 20 2024-05-31 10:17:55 -07:00
zach
345d2d9f3b fix: don't silently ignore non-directory paths 2024-05-31 10:17:55 -07:00
zach
d8460d6aa2 chore: clippy 2024-05-31 10:17:55 -07:00
zach
ecc6463e6c chore: update to wasmtime 19 2024-05-31 10:17:53 -07:00
8 changed files with 239 additions and 88 deletions

View File

@@ -59,6 +59,10 @@ pub trait ToBytes<'a> {
/// `to_bytes` converts a value into `Self::Bytes`
fn to_bytes(&self) -> Result<Self::Bytes, Error>;
fn to_vec(&self) -> Result<Vec<u8>, Error> {
self.to_bytes().map(|x| x.as_ref().to_vec())
}
}
impl<'a> ToBytes<'a> for () {
@@ -73,6 +77,10 @@ impl<'a> ToBytes<'a> for Vec<u8> {
fn to_bytes(&self) -> Result<Self::Bytes, Error> {
Ok(self.clone())
}
fn to_vec(&self) -> Result<Vec<u8>, Error> {
Ok(self.clone())
}
}
impl<'a> ToBytes<'a> for String {
@@ -80,6 +88,10 @@ impl<'a> ToBytes<'a> for String {
fn to_bytes(&self) -> Result<Self::Bytes, Error> {
Ok(self.clone())
}
fn to_vec(&self) -> Result<Vec<u8>, Error> {
self.to_bytes().map(|x| x.into_bytes())
}
}
impl<'a> ToBytes<'a> for &'a [u8] {
@@ -150,6 +162,10 @@ impl<'a, T: ToBytes<'a>> ToBytes<'a> for &'a T {
fn to_bytes(&self) -> Result<Self::Bytes, Error> {
<T as ToBytes>::to_bytes(self)
}
fn to_vec(&self) -> Result<Vec<u8>, Error> {
<T as ToBytes>::to_vec(self)
}
}
impl<'a, T: ToBytes<'a>> ToBytes<'a> for Option<T> {
@@ -161,6 +177,13 @@ impl<'a, T: ToBytes<'a>> ToBytes<'a> for Option<T> {
None => Ok(vec![]),
}
}
fn to_vec(&self) -> Result<Vec<u8>, Error> {
match self {
Some(x) => <T as ToBytes>::to_vec(x),
None => Ok(vec![]),
}
}
}
#[test]

View File

@@ -9,8 +9,8 @@ repository.workspace = true
version.workspace = true
[dependencies]
wasmtime = ">= 14.0.0, < 18.0.0"
wasmtime-wasi = ">= 14.0.0, < 18.0.0"
wasmtime = ">= 20.0.0, < 22.0.0"
wasmtime-wasi = ">= 20.0.0, < 22.0.0"
anyhow = "1"
serde = {version = "1", features = ["derive"]}
serde_json = "1"

View File

@@ -1,5 +1,88 @@
use wasmtime_wasi::pipe::{MemoryInputPipe, MemoryOutputPipe};
use crate::*;
#[derive(Default, Clone)]
pub struct WasiConfig {
pub args: Vec<String>,
pub stdin: Option<wasmtime_wasi::pipe::MemoryInputPipe>,
pub stdout: Option<wasmtime_wasi::pipe::MemoryOutputPipe>,
pub stderr: Option<wasmtime_wasi::pipe::MemoryOutputPipe>,
}
impl WasiConfig {
pub fn new() -> WasiConfig {
WasiConfig::default()
}
pub fn with_arg(mut self, x: impl Into<String>) -> Self {
self.args.push(x.into());
self
}
pub fn with_args(mut self, x: impl IntoIterator<Item = impl Into<String>>) -> Self {
let x: Vec<String> = x.into_iter().map(|x| x.into()).collect();
self.args.extend(x);
self
}
pub fn with_stdin<'a, T: ToBytes<'a>>(mut self, x: T) -> Result<Self, Error> {
self.stdin = Some(MemoryInputPipe::new(x.to_vec()?));
Ok(self)
}
pub fn with_stdout(mut self, size: usize) -> Self {
self.stdout = Some(MemoryOutputPipe::new(size));
self
}
pub fn with_stderr(mut self, size: usize) -> Self {
self.stderr = Some(MemoryOutputPipe::new(size));
self
}
}
#[derive(Clone)]
pub struct WasiOutput {
pub(crate) return_code: i32,
pub(crate) stdout: Option<wasmtime_wasi::pipe::MemoryOutputPipe>,
pub(crate) stderr: Option<wasmtime_wasi::pipe::MemoryOutputPipe>,
}
impl WasiOutput {
pub fn return_code(&self) -> i32 {
self.return_code
}
pub fn stdout(&self) -> Vec<u8> {
match &self.stdout {
Some(x) => x.contents().to_vec(),
None => vec![],
}
}
pub fn stderr(&self) -> Vec<u8> {
match &self.stderr {
Some(x) => x.contents().to_vec(),
None => vec![],
}
}
pub fn convert_stdout<T: FromBytesOwned>(&self) -> Result<T, Error> {
match &self.stdout {
Some(x) => T::from_bytes(x.contents().as_ref()),
None => anyhow::bail!("stdout not enabled"),
}
}
pub fn convert_stderr<T: FromBytesOwned>(&self) -> Result<T, Error> {
match &self.stderr {
Some(x) => T::from_bytes(x.contents().as_ref()),
None => anyhow::bail!("stdout not enabled"),
}
}
}
/// CurrentPlugin stores data that is available to the caller in PDK functions, this should
/// only be accessed from inside a host function
pub struct CurrentPlugin {
@@ -194,11 +277,11 @@ impl CurrentPlugin {
anyhow::bail!("unable to locate an extism kernel global: extism_context",)
};
let Val::ExternRef(Some(xs)) = xs.get(store) else {
let Val::ExternRef(Some(xs)) = xs.get(&mut store) else {
anyhow::bail!("expected extism_context to be an externref value",)
};
match xs.data().downcast_ref::<T>().cloned() {
match xs.data(&mut store)?.downcast_ref::<T>().cloned() {
Some(xs) => Ok(xs.clone()),
None => anyhow::bail!("could not downcast extism_context",),
}
@@ -307,39 +390,73 @@ impl CurrentPlugin {
pub(crate) fn new(
manifest: extism_manifest::Manifest,
wasi: bool,
wasi_args: Option<WasiConfig>,
available_pages: Option<u32>,
id: uuid::Uuid,
) -> Result<Self, Error> {
let wasi = if wasi {
let auth = wasmtime_wasi::ambient_authority();
let mut ctx = wasmtime_wasi::WasiCtxBuilder::new();
for (k, v) in manifest.config.iter() {
ctx.env(k, v)?;
ctx.allow_ip_name_lookup(true)
.allow_tcp(true)
.allow_udp(true)
.allow_blocking_current_thread(true);
if let Some(wasi_args) = wasi_args {
ctx.args(&wasi_args.args);
if let Some(stdin) = wasi_args.stdin {
ctx.stdin(stdin);
}
if let Some(stdout) = wasi_args.stdout {
ctx.stdout(stdout);
}
if let Some(stderr) = wasi_args.stderr {
ctx.stderr(stderr);
}
}
if let Some(a) = &manifest.allowed_paths {
for (k, v) in a.iter() {
let d = wasmtime_wasi::Dir::open_ambient_dir(k, auth).map_err(|err| {
Error::msg(format!(
"Unable to preopen directory \"{}\": {}",
k.display(),
err.kind()
))
})?;
ctx.preopened_dir(d, v)?;
ctx.preopened_dir(
k,
v.to_string_lossy(),
wasmtime_wasi::DirPerms::READ | wasmtime_wasi::DirPerms::MUTATE,
wasmtime_wasi::FilePerms::READ | wasmtime_wasi::FilePerms::WRITE,
)?;
}
}
if let Some(h) = &manifest.allowed_hosts {
let h = h.clone();
ctx.socket_addr_check(move |addr, _kind| {
for host in h.iter() {
let addrs = std::net::ToSocketAddrs::to_socket_addrs(&host);
if let Ok(addrs) = addrs {
for a in addrs.into_iter() {
if addr == &a {
return true;
}
}
}
}
false
});
}
// Enable WASI output, typically used for debugging purposes
if std::env::var("EXTISM_ENABLE_WASI_OUTPUT").is_ok() {
ctx.inherit_stdout().inherit_stderr();
}
Some(Wasi { ctx: ctx.build() })
Some(Wasi {
ctx: ctx.build_p1(),
})
} else {
None
};
let memory_limiter = if let Some(pgs) = available_pages {
let n = pgs as usize * 65536;
Some(crate::current_plugin::MemoryLimiter {

View File

@@ -38,8 +38,13 @@ impl From<wasmtime::ValType> for ValType {
F32 => ValType::F32,
F64 => ValType::F64,
V128 => ValType::V128,
FuncRef => ValType::FuncRef,
ExternRef => ValType::ExternRef,
Ref(t) => {
if t.heap_type().is_func() {
ValType::FuncRef
} else {
ValType::ExternRef
}
}
}
}
}
@@ -53,8 +58,8 @@ impl From<ValType> for wasmtime::ValType {
F32 => wasmtime::ValType::F32,
F64 => wasmtime::ValType::F64,
V128 => wasmtime::ValType::V128,
FuncRef => wasmtime::ValType::FuncRef,
ExternRef => wasmtime::ValType::ExternRef,
FuncRef => wasmtime::ValType::FUNCREF,
ExternRef => wasmtime::ValType::EXTERNREF,
}
}
}
@@ -171,8 +176,8 @@ pub struct Function {
/// Module name
pub(crate) namespace: Option<String>,
/// Function type
pub(crate) ty: wasmtime::FuncType,
pub(crate) params: Vec<wasmtime::ValType>,
pub(crate) results: Vec<wasmtime::ValType>,
/// Function handle
pub(crate) f: Arc<FunctionInner>,
@@ -185,8 +190,8 @@ impl Function {
/// Create a new host function
pub fn new<T: 'static, F>(
name: impl Into<String>,
args: impl IntoIterator<Item = ValType>,
returns: impl IntoIterator<Item = ValType>,
params: impl IntoIterator<Item = ValType>,
results: impl IntoIterator<Item = ValType>,
user_data: UserData<T>,
f: F,
) -> Function
@@ -198,13 +203,13 @@ impl Function {
{
let data = user_data.clone();
let name = name.into();
let args = args.into_iter().map(wasmtime::ValType::from);
let returns = returns.into_iter().map(wasmtime::ValType::from);
let ty = wasmtime::FuncType::new(args, returns);
trace!("Creating function {name}: type={ty:?}");
let params = params.into_iter().map(wasmtime::ValType::from).collect();
let results = results.into_iter().map(wasmtime::ValType::from).collect();
trace!("Creating function {name}: params={params:?}, results={results:?}");
Function {
name,
ty,
params,
results,
f: Arc::new(
move |mut caller: Caller<_>, inp: &[Val], outp: &mut [Val]| {
let x = data.clone();
@@ -219,6 +224,10 @@ impl Function {
}
}
pub(crate) fn ty(&self, engine: &wasmtime::Engine) -> wasmtime::FuncType {
wasmtime::FuncType::new(engine, self.params.clone(), self.results.clone())
}
/// Host function name
pub fn name(&self) -> &str {
&self.name
@@ -241,11 +250,6 @@ impl Function {
self.set_namespace(namespace);
self
}
/// Get function type
pub fn ty(&self) -> &wasmtime::FuncType {
&self.ty
}
}
/// The `host_fn` macro is used to define typed host functions

View File

@@ -3,7 +3,7 @@ use crate::*;
/// WASI context
pub struct Wasi {
/// wasi
pub ctx: wasmtime_wasi::WasiCtx,
pub ctx: wasmtime_wasi::preview1::WasiP1Ctx,
}
/// InternalExt provides a unified way of acessing `memory`, `store` and `internal` values

View File

@@ -23,7 +23,7 @@ mod timer;
/// Extism C API
pub mod sdk;
pub use current_plugin::CurrentPlugin;
pub use current_plugin::{CurrentPlugin, WasiConfig, WasiOutput};
pub use extism_convert::{FromBytes, FromBytesOwned, ToBytes};
pub use extism_manifest::{Manifest, Wasm, WasmMetadata};
pub use function::{Function, UserData, Val, ValType, PTR};

View File

@@ -224,7 +224,7 @@ fn relink(
macro_rules! add_funcs {
($($name:ident($($args:expr),*) $(-> $($r:expr),*)?);* $(;)?) => {
$(
let t = FuncType::new([$($args),*], [$($($r),*)?]);
let t = FuncType::new(&engine, [$($args),*], [$($($r),*)?]);
linker.func_new(EXTISM_ENV_MODULE, stringify!($name), t, pdk::$name)?;
)*
};
@@ -250,16 +250,16 @@ fn relink(
// If wasi is enabled then add it to the linker
if with_wasi {
wasmtime_wasi::add_to_linker(&mut linker, |x: &mut CurrentPlugin| {
wasmtime_wasi::preview1::add_to_linker_sync(&mut linker, |x: &mut CurrentPlugin| {
&mut x.wasi.as_mut().unwrap().ctx
})?;
}
for f in imports {
for f in imports.iter() {
let name = f.name();
let ns = f.namespace().unwrap_or(EXTISM_USER_MODULE);
unsafe {
linker.func_new(ns, name, f.ty().clone(), &*(f.f.as_ref() as *const _))?;
linker.func_new(ns, name, f.ty(engine).clone(), &*(f.f.as_ref() as *const _))?;
}
}
@@ -305,7 +305,8 @@ impl Plugin {
.coredump_on_trap(debug_options.coredump.is_some())
.profiler(debug_options.profiling_strategy)
.wasm_tail_call(true)
.wasm_function_references(true);
.wasm_function_references(true)
.wasm_gc(true);
match cache_dir {
Some(None) => (),
@@ -337,13 +338,12 @@ impl Plugin {
let id = uuid::Uuid::new_v4();
let mut store = Store::new(
&engine,
CurrentPlugin::new(manifest, with_wasi, available_pages, id)?,
CurrentPlugin::new(manifest, with_wasi, None, available_pages, id)?,
);
store.set_epoch_deadline(1);
let imports: Vec<Function> = imports.into_iter().collect();
let (instance_pre, linker) = relink(&engine, &mut store, &imports, &modules, with_wasi)?;
let timer_tx = Timer::tx();
let mut plugin = Plugin {
modules,
@@ -377,22 +377,24 @@ impl Plugin {
pub(crate) fn reset_store(
&mut self,
instance_lock: &mut std::sync::MutexGuard<Option<Instance>>,
wasi_args: Option<WasiConfig>,
) -> Result<(), Error> {
if self.store_needs_reset {
if self.store_needs_reset || wasi_args.is_some() {
let engine = self.store.engine().clone();
let internal = self.current_plugin_mut();
let with_wasi = internal.wasi.is_some();
let with_wasi = internal.wasi.is_some() || wasi_args.is_some();
self.store = Store::new(
&engine,
CurrentPlugin::new(
internal.manifest.clone(),
internal.wasi.is_some(),
wasi_args,
internal.available_pages,
self.id,
)?,
);
self.linker = Linker::new(&engine);
self.store.set_epoch_deadline(1);
let (instance_pre, linker) = relink(
&engine,
&mut self.store,
@@ -402,6 +404,7 @@ impl Plugin {
)?;
self.linker = linker;
self.instance_pre = instance_pre;
let store = &mut self.store as *mut _;
let linker = &mut self.linker as *mut _;
let current_plugin = self.current_plugin_mut();
@@ -465,7 +468,7 @@ impl Plugin {
if let Some(f) = x.func() {
let (params, mut results) = (f.params(), f.results());
match (params.len(), results.len()) {
(0, 1) => results.next() == Some(wasmtime::ValType::I32),
(0, 1) => matches!(results.next(), Some(wasmtime::ValType::I32)),
(0, 0) => true,
_ => false,
}
@@ -481,7 +484,7 @@ impl Plugin {
&mut self,
input: *const u8,
mut len: usize,
host_context: Option<ExternRef>,
host_context: Option<Rooted<ExternRef>>,
) -> Result<(), Error> {
self.output = Output::default();
self.clear_error()?;
@@ -615,7 +618,7 @@ impl Plugin {
if let Some(reactor_init) = reactor_init {
reactor_init.call(&mut store, &[], &mut [])?;
}
let mut results = vec![Val::null(); init.ty(&store).results().len()];
let mut results = vec![Val::I32(0); init.ty(&store).results().len()];
init.call(
&mut store,
&[Val::I32(0), Val::I32(0)],
@@ -700,17 +703,19 @@ impl Plugin {
lock: &mut std::sync::MutexGuard<Option<Instance>>,
name: impl AsRef<str>,
input: impl AsRef<[u8]>,
host_context: Option<ExternRef>,
host_context: Option<Rooted<ExternRef>>,
wasi_args: Option<WasiConfig>,
) -> Result<i32, (Error, i32)> {
let name = name.as_ref();
let input = input.as_ref();
if let Err(e) = self.reset_store(lock) {
if let Err(e) = self.reset_store(lock, wasi_args) {
error!(
plugin = self.id.to_string(),
"call to Plugin::reset_store failed: {e:?}"
);
}
self.store_needs_reset = name == "_start";
self.instantiate(lock).map_err(|e| (e, -1))?;
@@ -748,14 +753,13 @@ impl Plugin {
self.current_plugin_mut().start_time = std::time::Instant::now();
// Call the function
let mut results = vec![wasmtime::Val::null(); n_results];
let mut results = vec![wasmtime::Val::I32(0); n_results];
let mut res = func.call(self.store_mut(), &[], results.as_mut_slice());
// Stop timer
self.store
.epoch_deadline_callback(|_| Ok(UpdateDeadline::Continue(1)));
let _ = self.timer_tx.send(TimerAction::Stop { id: self.id });
self.store_needs_reset = name == "_start";
// Get extism error
self.get_output_after_call().map_err(|x| (x, -1))?;
@@ -833,13 +837,7 @@ impl Plugin {
}
}
let wasi_exit_code = e
.downcast_ref::<wasmtime_wasi::I32Exit>()
.map(|e| e.0)
.or_else(|| {
e.downcast_ref::<wasmtime_wasi::preview2::I32Exit>()
.map(|e| e.0)
});
let wasi_exit_code = e.downcast_ref::<wasmtime_wasi::I32Exit>().map(|e| e.0);
if let Some(exit_code) = wasi_exit_code {
debug!(
plugin = self.id.to_string(),
@@ -902,7 +900,7 @@ impl Plugin {
let lock = self.instance.clone();
let mut lock = lock.lock().unwrap();
let data = input.to_bytes()?;
self.raw_call(&mut lock, name, data, None)
self.raw_call(&mut lock, name, data, None, None)
.map_err(|e| e.0)
.and_then(move |_| self.output())
}
@@ -921,11 +919,32 @@ impl Plugin {
let lock = self.instance.clone();
let mut lock = lock.lock().unwrap();
let data = input.to_bytes()?;
self.raw_call(&mut lock, name, data, Some(ExternRef::new(host_context)))
let r = ExternRef::new(&mut self.store, host_context)?;
self.raw_call(&mut lock, name, data, Some(r), None)
.map_err(|e| e.0)
.and_then(move |_| self.output())
}
/// Execute the `_start` function of a WASI command module, providing input/output and command-line arguments
/// via `WasiConfig`
pub fn run_wasi_command(&mut self, wasi_args: WasiConfig) -> Result<WasiOutput, Error> {
let mut output = WasiOutput {
return_code: 0,
stdout: wasi_args.stdout.clone(),
stderr: wasi_args.stderr.clone(),
};
let lock = self.instance.clone();
let mut lock = lock.lock().unwrap();
let res = self.raw_call(&mut lock, "_start", b"", None, Some(wasi_args));
output.return_code = match res {
Ok(rc) => rc,
Err((_, rc)) => rc,
};
Ok(output)
}
/// Similar to `Plugin::call`, but returns the Extism error code along with the
/// `Error`. It is assumed if `Ok(_)` is returned that the error code was `0`.
///
@@ -940,7 +959,7 @@ impl Plugin {
let lock = self.instance.clone();
let mut lock = lock.lock().unwrap();
let data = input.to_bytes().map_err(|e| (e, -1))?;
self.raw_call(&mut lock, name, data, None)
self.raw_call(&mut lock, name, data, None, None)
.and_then(move |_| self.output().map_err(|e| (e, -1)))
}

View File

@@ -41,9 +41,9 @@ pub type ExtismFunctionType = extern "C" fn(
/// Log drain callback
pub type ExtismLogDrainFunctionType = extern "C" fn(data: *const std::ffi::c_char, size: Size);
impl From<&wasmtime::Val> for ExtismVal {
fn from(value: &wasmtime::Val) -> Self {
match value.ty() {
impl ExtismVal {
fn from_val(value: &wasmtime::Val, ctx: impl AsContext) -> Self {
match value.ty(ctx) {
wasmtime::ValType::I32 => ExtismVal {
t: ValType::I32,
v: ValUnion {
@@ -218,7 +218,10 @@ pub unsafe extern "C" fn extism_function_new(
output_types.clone(),
user_data,
move |plugin, inputs, outputs, user_data| {
let inputs: Vec<_> = inputs.iter().map(ExtismVal::from).collect();
let inputs: Vec<_> = inputs
.iter()
.map(|x| ExtismVal::from_val(x, unsafe { &*plugin.store }))
.collect();
let mut output_tmp: Vec<_> = output_types
.iter()
.map(|t| ExtismVal {
@@ -407,20 +410,6 @@ pub unsafe extern "C" fn extism_plugin_config(
}
};
let wasi = &mut plugin.current_plugin_mut().wasi;
if let Some(Wasi { ctx, .. }) = wasi {
for (k, v) in json.iter() {
match v {
Some(v) => {
let _ = ctx.push_env(k, v);
}
None => {
let _ = ctx.push_env(k, "");
}
}
}
}
let id = plugin.id;
let config = &mut plugin.current_plugin_mut().manifest.config;
for (k, v) in json.into_iter() {
@@ -528,12 +517,11 @@ pub unsafe extern "C" fn extism_plugin_call_with_host_context(
name
);
let input = std::slice::from_raw_parts(data, data_len as usize);
let res = plugin.raw_call(
&mut lock,
name,
input,
Some(ExternRef::new(CVoidContainer(host_context))),
);
let r = match ExternRef::new(&mut plugin.store, CVoidContainer(host_context)) {
Err(e) => return plugin.return_error(&mut lock, e, -1),
Ok(x) => x,
};
let res = plugin.raw_call(&mut lock, name, input, Some(r), None);
match res {
Err((e, rc)) => plugin.return_error(&mut lock, e, rc),