mirror of
https://github.com/extism/extism.git
synced 2026-01-09 22:07:57 -05:00
feat: add ability to dump extism kernel memory and generate coredumps, cleanup kernel memory layout (#539)
Fixes https://github.com/extism/extism/issues/537 - Requires wasmtime 14.0.0 or greater for coredump serialization - Adds `EXTISM_COREDUMP` environment variable to write wasmtime coredump to a file when an error occurs - This will create a coredump from the main wasm module, which means it doesn't give us too much insight into the kernel - Adds `EXTISM_MEMDUMP` environment variable to write extism linear memory to a file when an error occurs - This gives us access to the kernel memory - Adds some missing profiling options - Converts timeouts to a Trap instead of a plain error, this helps us get better information about where the timeout occured - Some small improvements to error handling after a plugin call - Adds a test for coredump and memdump generation - Adds the ability to configure debug options using `PluginBuilder` - Fixes memory layout and a wasted page of memory in the kernel, found while debugging a memory dump --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: zshipko <zshipko@users.noreply.github.com>
This commit is contained in:
@@ -17,13 +17,13 @@
|
|||||||
//! ## Input/Output
|
//! ## Input/Output
|
||||||
//!
|
//!
|
||||||
//! Input and output are just allocated blocks of memory that are marked as either input or output using
|
//! Input and output are just allocated blocks of memory that are marked as either input or output using
|
||||||
//! the `extism_input_set` or `extism_output_set` functions. The global variables `INPUT_OFFSET` contains
|
//! the `extism_input_set` or `extism_output_set` functions. The MemoryRoot field `input_offset` contains
|
||||||
//! the offset in memory to the input data and `INPUT_LENGTH` contains the size of the input data. `OUTPUT_OFFSET`
|
//! the offset in memory to the input data and `input_length` contains the size of the input data. `output_offset`
|
||||||
//! and `OUTPUT_LENGTH` are used for the output data.
|
//! and `output_length` are used for the output data.
|
||||||
//!
|
//!
|
||||||
//! ## Error handling
|
//! ## Error handling
|
||||||
//!
|
//!
|
||||||
//! The `ERROR` global is used to track the current error message. If it is set to `0` then there is no error.
|
//! The `error` field is used to track the current error message. If it is set to `0` then there is no error.
|
||||||
//! The length of the error message can be retreived using `extism_length`.
|
//! The length of the error message can be retreived using `extism_length`.
|
||||||
//!
|
//!
|
||||||
//! ## Memory offsets
|
//! ## Memory offsets
|
||||||
@@ -45,31 +45,6 @@ pub type Length = u64;
|
|||||||
/// WebAssembly page size
|
/// WebAssembly page size
|
||||||
const PAGE_SIZE: usize = 65536;
|
const PAGE_SIZE: usize = 65536;
|
||||||
|
|
||||||
/// Determines the amount of bytes that can be wasted by re-using a block. If more than this number is wasted by re-using
|
|
||||||
/// a block then it will be split into two smaller blocks.
|
|
||||||
const BLOCK_SPLIT_SIZE: usize = 128;
|
|
||||||
|
|
||||||
/// Offset to the input data
|
|
||||||
static mut INPUT_OFFSET: Pointer = 0;
|
|
||||||
|
|
||||||
/// Length of the input data
|
|
||||||
static mut INPUT_LENGTH: Length = 0;
|
|
||||||
|
|
||||||
/// Offset to the output data
|
|
||||||
static mut OUTPUT_OFFSET: Pointer = 0;
|
|
||||||
|
|
||||||
/// Offset to the input data
|
|
||||||
static mut OUTPUT_LENGTH: Length = 0;
|
|
||||||
|
|
||||||
/// Current error message
|
|
||||||
static mut ERROR: AtomicU64 = AtomicU64::new(0);
|
|
||||||
|
|
||||||
/// Determines if the kernel has been initialized already
|
|
||||||
static mut INITIALIZED: AtomicBool = AtomicBool::new(false);
|
|
||||||
|
|
||||||
/// A pointer to the first page that will be managed by Extism, this is set during initialization
|
|
||||||
static mut START_PAGE: usize = 0;
|
|
||||||
|
|
||||||
/// Provides information about the usage status of a `MemoryBlock`
|
/// Provides information about the usage status of a `MemoryBlock`
|
||||||
#[repr(u8)]
|
#[repr(u8)]
|
||||||
#[derive(PartialEq)]
|
#[derive(PartialEq)]
|
||||||
@@ -98,10 +73,22 @@ pub enum MemoryStatus {
|
|||||||
/// including their data.
|
/// including their data.
|
||||||
#[repr(C)]
|
#[repr(C)]
|
||||||
pub struct MemoryRoot {
|
pub struct MemoryRoot {
|
||||||
|
/// Set to true after initialization
|
||||||
|
pub initialized: AtomicBool,
|
||||||
/// Position of the bump allocator, relative to `blocks` field
|
/// Position of the bump allocator, relative to `blocks` field
|
||||||
pub position: AtomicU64,
|
pub position: AtomicU64,
|
||||||
/// The total size of all data allocated using this allocator
|
/// The total size of all data allocated using this allocator
|
||||||
pub length: AtomicU64,
|
pub length: AtomicU64,
|
||||||
|
/// Offset of error block
|
||||||
|
pub error: AtomicU64,
|
||||||
|
/// Input position in memory
|
||||||
|
pub input_offset: Pointer,
|
||||||
|
/// Input length
|
||||||
|
pub input_length: Length,
|
||||||
|
/// Output position in memory
|
||||||
|
pub output_offset: Pointer,
|
||||||
|
/// Output length
|
||||||
|
pub output_length: Length,
|
||||||
/// A pointer to the start of the first block
|
/// A pointer to the start of the first block
|
||||||
pub blocks: [MemoryBlock; 0],
|
pub blocks: [MemoryBlock; 0],
|
||||||
}
|
}
|
||||||
@@ -131,32 +118,41 @@ pub fn num_pages(nbytes: u64) -> usize {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get the `MemoryRoot` at the correct offset in memory
|
// Get the `MemoryRoot`, this is always stored at offset 1 in memory
|
||||||
#[inline]
|
#[inline]
|
||||||
unsafe fn memory_root() -> &'static mut MemoryRoot {
|
unsafe fn memory_root() -> &'static mut MemoryRoot {
|
||||||
&mut *((START_PAGE * PAGE_SIZE) as *mut MemoryRoot)
|
&mut *(1 as *mut MemoryRoot)
|
||||||
}
|
}
|
||||||
|
|
||||||
impl MemoryRoot {
|
impl MemoryRoot {
|
||||||
/// Initialize or load the `MemoryRoot` from the correct position in memory
|
/// Initialize or load the `MemoryRoot` from the correct position in memory
|
||||||
pub unsafe fn new() -> &'static mut MemoryRoot {
|
pub unsafe fn new() -> &'static mut MemoryRoot {
|
||||||
|
let root = memory_root();
|
||||||
|
|
||||||
// If this fails then `INITIALIZED` is already `true` and we can just return the
|
// If this fails then `INITIALIZED` is already `true` and we can just return the
|
||||||
// already initialized `MemoryRoot`
|
// already initialized `MemoryRoot`
|
||||||
if INITIALIZED
|
if root
|
||||||
|
.initialized
|
||||||
.compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
|
.compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
|
||||||
.is_err()
|
.is_err()
|
||||||
{
|
{
|
||||||
return memory_root();
|
return root;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Ensure that at least one page is allocated to store the `MemoryRoot` data
|
// Ensure that at least one page is allocated to store the `MemoryRoot` data
|
||||||
START_PAGE = core::arch::wasm32::memory_grow(0, 1);
|
if core::arch::wasm32::memory_size(0) == 0 {
|
||||||
if START_PAGE == usize::MAX {
|
if core::arch::wasm32::memory_grow(0, 1) == usize::MAX {
|
||||||
panic!("Out of memory");
|
core::arch::wasm32::unreachable()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
root.input_offset = 0;
|
||||||
|
root.input_length = 0;
|
||||||
|
root.output_offset = 0;
|
||||||
|
root.output_length = 0;
|
||||||
|
root.error.store(0, Ordering::Release);
|
||||||
|
|
||||||
// Initialize the `MemoryRoot` length, position and data
|
// Initialize the `MemoryRoot` length, position and data
|
||||||
let root = memory_root();
|
|
||||||
root.length.store(
|
root.length.store(
|
||||||
PAGE_SIZE as u64 - core::mem::size_of::<MemoryRoot>() as u64,
|
PAGE_SIZE as u64 - core::mem::size_of::<MemoryRoot>() as u64,
|
||||||
Ordering::Release,
|
Ordering::Release,
|
||||||
@@ -181,6 +177,11 @@ impl MemoryRoot {
|
|||||||
self.length.load(Ordering::Acquire) as usize,
|
self.length.load(Ordering::Acquire) as usize,
|
||||||
);
|
);
|
||||||
self.position.store(0, Ordering::Release);
|
self.position.store(0, Ordering::Release);
|
||||||
|
self.error.store(0, Ordering::Release);
|
||||||
|
self.input_offset = 0;
|
||||||
|
self.input_length = 0;
|
||||||
|
self.output_offset = 0;
|
||||||
|
self.output_length = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
#[inline(always)]
|
#[inline(always)]
|
||||||
@@ -224,7 +225,7 @@ impl MemoryRoot {
|
|||||||
// Re-use freed blocks when they're large enough
|
// Re-use freed blocks when they're large enough
|
||||||
if status == MemoryStatus::Free as u8 && b.size >= length as usize {
|
if status == MemoryStatus::Free as u8 && b.size >= length as usize {
|
||||||
// Split block if there is too much excess
|
// Split block if there is too much excess
|
||||||
if b.size - length as usize >= BLOCK_SPLIT_SIZE {
|
if b.size - length as usize >= 128 {
|
||||||
b.size -= length as usize;
|
b.size -= length as usize;
|
||||||
b.used = 0;
|
b.used = 0;
|
||||||
|
|
||||||
@@ -348,14 +349,15 @@ pub unsafe fn extism_free(p: Pointer) {
|
|||||||
if p == 0 {
|
if p == 0 {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
let block = MemoryRoot::new().find_block(p);
|
let root = MemoryRoot::new();
|
||||||
|
let block = root.find_block(p);
|
||||||
if let Some(block) = block {
|
if let Some(block) = block {
|
||||||
block.free();
|
block.free();
|
||||||
|
|
||||||
// If the input pointer is freed for some reason, make sure the input length to 0
|
// If the input pointer is freed for some reason, make sure the input length to 0
|
||||||
// since the original data is gone
|
// since the original data is gone
|
||||||
if p == INPUT_OFFSET {
|
if p == root.input_offset {
|
||||||
INPUT_LENGTH = 0;
|
root.input_length = 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -396,21 +398,23 @@ pub unsafe fn extism_load_u64(p: Pointer) -> u64 {
|
|||||||
/// Load a byte from the input data
|
/// Load a byte from the input data
|
||||||
#[no_mangle]
|
#[no_mangle]
|
||||||
pub unsafe fn extism_input_load_u8(p: Pointer) -> u8 {
|
pub unsafe fn extism_input_load_u8(p: Pointer) -> u8 {
|
||||||
|
let root = MemoryRoot::new();
|
||||||
#[cfg(feature = "bounds-checking")]
|
#[cfg(feature = "bounds-checking")]
|
||||||
if p >= INPUT_LENGTH {
|
if p >= root.input_length {
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
*((INPUT_OFFSET + p) as *mut u8)
|
*((root.input_offset + p) as *mut u8)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Load a u64 from the input data
|
/// Load a u64 from the input data
|
||||||
#[no_mangle]
|
#[no_mangle]
|
||||||
pub unsafe fn extism_input_load_u64(p: Pointer) -> u64 {
|
pub unsafe fn extism_input_load_u64(p: Pointer) -> u64 {
|
||||||
|
let root = MemoryRoot::new();
|
||||||
#[cfg(feature = "bounds-checking")]
|
#[cfg(feature = "bounds-checking")]
|
||||||
if p + core::mem::size_of::<u64>() as Pointer > INPUT_LENGTH {
|
if p + core::mem::size_of::<u64>() as Pointer > root.input_length {
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
*((INPUT_OFFSET + p) as *mut u64)
|
*((root.input_offset + p) as *mut u64)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Write a byte in Extism-managed memory
|
/// Write a byte in Extism-managed memory
|
||||||
@@ -436,82 +440,83 @@ pub unsafe fn extism_store_u64(p: Pointer, x: u64) {
|
|||||||
/// Set the range of the input data in memory
|
/// Set the range of the input data in memory
|
||||||
#[no_mangle]
|
#[no_mangle]
|
||||||
pub unsafe fn extism_input_set(p: Pointer, len: Length) {
|
pub unsafe fn extism_input_set(p: Pointer, len: Length) {
|
||||||
|
let root = MemoryRoot::new();
|
||||||
#[cfg(feature = "bounds-checking")]
|
#[cfg(feature = "bounds-checking")]
|
||||||
{
|
{
|
||||||
let root = MemoryRoot::new();
|
|
||||||
if !root.pointer_in_bounds(p) || !root.pointer_in_bounds(p + len - 1) {
|
if !root.pointer_in_bounds(p) || !root.pointer_in_bounds(p + len - 1) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
INPUT_OFFSET = p;
|
root.input_offset = p;
|
||||||
INPUT_LENGTH = len;
|
root.input_length = len;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Set the range of the output data in memory
|
/// Set the range of the output data in memory
|
||||||
#[no_mangle]
|
#[no_mangle]
|
||||||
pub unsafe fn extism_output_set(p: Pointer, len: Length) {
|
pub unsafe fn extism_output_set(p: Pointer, len: Length) {
|
||||||
|
let root = MemoryRoot::new();
|
||||||
#[cfg(feature = "bounds-checking")]
|
#[cfg(feature = "bounds-checking")]
|
||||||
{
|
{
|
||||||
let root = MemoryRoot::new();
|
|
||||||
if !root.pointer_in_bounds(p) || !root.pointer_in_bounds(p + len - 1) {
|
if !root.pointer_in_bounds(p) || !root.pointer_in_bounds(p + len - 1) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
OUTPUT_OFFSET = p;
|
root.output_offset = p;
|
||||||
OUTPUT_LENGTH = len;
|
root.output_length = len;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get the input length
|
/// Get the input length
|
||||||
#[no_mangle]
|
#[no_mangle]
|
||||||
pub fn extism_input_length() -> Length {
|
pub fn extism_input_length() -> Length {
|
||||||
unsafe { INPUT_LENGTH }
|
unsafe { MemoryRoot::new().input_length }
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get the input offset in Exitsm-managed memory
|
/// Get the input offset in Exitsm-managed memory
|
||||||
#[no_mangle]
|
#[no_mangle]
|
||||||
pub fn extism_input_offset() -> Length {
|
pub fn extism_input_offset() -> Length {
|
||||||
unsafe { INPUT_OFFSET }
|
unsafe { MemoryRoot::new().input_offset }
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get the output length
|
/// Get the output length
|
||||||
#[no_mangle]
|
#[no_mangle]
|
||||||
pub fn extism_output_length() -> Length {
|
pub unsafe fn extism_output_length() -> Length {
|
||||||
unsafe { OUTPUT_LENGTH }
|
unsafe { MemoryRoot::new().output_length }
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get the output offset in Extism-managed memory
|
/// Get the output offset in Extism-managed memory
|
||||||
#[no_mangle]
|
#[no_mangle]
|
||||||
pub fn extism_output_offset() -> Length {
|
pub unsafe fn extism_output_offset() -> Length {
|
||||||
unsafe { OUTPUT_OFFSET }
|
MemoryRoot::new().output_offset
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Reset the allocator
|
/// Reset the allocator
|
||||||
#[no_mangle]
|
#[no_mangle]
|
||||||
pub unsafe fn extism_reset() {
|
pub unsafe fn extism_reset() {
|
||||||
ERROR.store(0, Ordering::SeqCst);
|
|
||||||
MemoryRoot::new().reset()
|
MemoryRoot::new().reset()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Set the error message offset
|
/// Set the error message offset
|
||||||
#[no_mangle]
|
#[no_mangle]
|
||||||
pub unsafe fn extism_error_set(ptr: Pointer) {
|
pub unsafe fn extism_error_set(ptr: Pointer) {
|
||||||
|
let root = MemoryRoot::new();
|
||||||
|
|
||||||
// Allow ERROR to be set to 0
|
// Allow ERROR to be set to 0
|
||||||
if ptr == 0 {
|
if ptr == 0 {
|
||||||
ERROR.store(ptr, Ordering::SeqCst);
|
root.error.store(ptr, Ordering::SeqCst);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(feature = "bounds-checking")]
|
#[cfg(feature = "bounds-checking")]
|
||||||
if !MemoryRoot::new().pointer_in_bounds(ptr) {
|
if !root.pointer_in_bounds(ptr) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
ERROR.store(ptr, Ordering::SeqCst);
|
root.error.store(ptr, Ordering::SeqCst);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get the error message offset, if it's `0` then no error has been set
|
/// Get the error message offset, if it's `0` then no error has been set
|
||||||
#[no_mangle]
|
#[no_mangle]
|
||||||
pub unsafe fn extism_error_get() -> Pointer {
|
pub unsafe fn extism_error_get() -> Pointer {
|
||||||
ERROR.load(Ordering::SeqCst)
|
MemoryRoot::new().error.load(Ordering::SeqCst)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get the position of the allocator, this can be used as an indication of how many bytes are currently in-use
|
/// Get the position of the allocator, this can be used as an indication of how many bytes are currently in-use
|
||||||
|
|||||||
@@ -9,8 +9,8 @@ repository = "https://github.com/extism/extism"
|
|||||||
description = "Extism runtime and Rust SDK"
|
description = "Extism runtime and Rust SDK"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
wasmtime = ">= 13.0.0, < 15.0.0"
|
wasmtime = ">= 14.0.0, < 15.0.0"
|
||||||
wasmtime-wasi = ">= 13.0.0, < 15.0.0"
|
wasmtime-wasi = ">= 14.0.0, < 15.0.0"
|
||||||
anyhow = "1"
|
anyhow = "1"
|
||||||
serde = {version = "1", features = ["derive"]}
|
serde = {version = "1", features = ["derive"]}
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
|
|||||||
@@ -15,6 +15,16 @@ To use the `extism` crate, you can add it to your Cargo file:
|
|||||||
extism = "*"
|
extism = "*"
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Environment variables
|
||||||
|
|
||||||
|
There are a few environment variables that can be used for debugging purposes:
|
||||||
|
|
||||||
|
- `EXTISM_ENABLE_WASI_OUTPUT=1`: show WASI stdout/stderr
|
||||||
|
- `EXTISM_MEMDUMP=extism.mem`: dump Extism linear memory to a file
|
||||||
|
- `EXTISM_COREDUMP=extism.core`: write [coredump](https://github.com/WebAssembly/tool-conventions/blob/main/Coredump.md) to a file when a WebAssembly function traps
|
||||||
|
- `EXTISM_DEBUG=1`: generate debug information
|
||||||
|
- `EXTISM_PROFILE=perf|jitdump|vtune`: enable Wasmtime profiling
|
||||||
|
|
||||||
## Getting Started
|
## Getting Started
|
||||||
|
|
||||||
This guide should walk you through some of the concepts in Extism and the `extism` crate.
|
This guide should walk you through some of the concepts in Extism and the `extism` crate.
|
||||||
@@ -39,7 +49,7 @@ fn main() {
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
> **Note**: See [the Manifest docs](https://docs.rs/extism-manifest/0.5.0/extism_manifest/) as it has a rich schema and a lot of options.
|
> **Note**: See [the Manifest docs](https://docs.rs/extism-manifest/extism_manifest/) as it has a rich schema and a lot of options.
|
||||||
|
|
||||||
### Calling A Plug-in's Exports
|
### Calling A Plug-in's Exports
|
||||||
|
|
||||||
|
|||||||
@@ -258,16 +258,20 @@ impl CurrentPlugin {
|
|||||||
|
|
||||||
/// Get a pointer to the plugin memory
|
/// Get a pointer to the plugin memory
|
||||||
pub(crate) fn memory_ptr(&mut self) -> *mut u8 {
|
pub(crate) fn memory_ptr(&mut self) -> *mut u8 {
|
||||||
let (linker, mut store) = self.linker_and_store();
|
if let Some(mem) = self.memory() {
|
||||||
if let Some(mem) = linker.get(&mut store, "env", "memory") {
|
let (_, mut store) = self.linker_and_store();
|
||||||
if let Some(mem) = mem.into_memory() {
|
return mem.data_ptr(&mut store);
|
||||||
return mem.data_ptr(&mut store);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
std::ptr::null_mut()
|
std::ptr::null_mut()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Get extism memory
|
||||||
|
pub(crate) fn memory(&mut self) -> Option<wasmtime::Memory> {
|
||||||
|
let (linker, mut store) = self.linker_and_store();
|
||||||
|
linker.get(&mut store, "env", "memory")?.into_memory()
|
||||||
|
}
|
||||||
|
|
||||||
/// Get a `MemoryHandle` from a `Val` reference - this can be used to convert a host function's
|
/// Get a `MemoryHandle` from a `Val` reference - this can be used to convert a host function's
|
||||||
/// argument directly to `MemoryHandle`
|
/// argument directly to `MemoryHandle`
|
||||||
pub fn memory_from_val(&mut self, offs: &Val) -> Option<MemoryHandle> {
|
pub fn memory_from_val(&mut self, offs: &Val) -> Option<MemoryHandle> {
|
||||||
|
|||||||
Binary file not shown.
@@ -27,6 +27,7 @@ pub use plugin_builder::PluginBuilder;
|
|||||||
|
|
||||||
pub(crate) use internal::{Internal, Wasi};
|
pub(crate) use internal::{Internal, Wasi};
|
||||||
pub(crate) use log::{debug, error, trace};
|
pub(crate) use log::{debug, error, trace};
|
||||||
|
pub(crate) use plugin_builder::DebugOptions;
|
||||||
pub(crate) use timer::{Timer, TimerAction};
|
pub(crate) use timer::{Timer, TimerAction};
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@@ -70,6 +70,8 @@ pub struct Plugin {
|
|||||||
/// Set to `true` when de-initializarion may have occured (i.e.a call to `_start`),
|
/// Set to `true` when de-initializarion may have occured (i.e.a call to `_start`),
|
||||||
/// in this case we need to re-initialize the entire module.
|
/// in this case we need to re-initialize the entire module.
|
||||||
pub(crate) needs_reset: bool,
|
pub(crate) needs_reset: bool,
|
||||||
|
|
||||||
|
pub(crate) debug_options: DebugOptions,
|
||||||
}
|
}
|
||||||
|
|
||||||
unsafe impl Send for Plugin {}
|
unsafe impl Send for Plugin {}
|
||||||
@@ -108,6 +110,8 @@ const EXPORT_MODULE_NAME: &str = "env";
|
|||||||
fn profiling_strategy() -> ProfilingStrategy {
|
fn profiling_strategy() -> ProfilingStrategy {
|
||||||
match std::env::var("EXTISM_PROFILE").as_deref() {
|
match std::env::var("EXTISM_PROFILE").as_deref() {
|
||||||
Ok("perf") => ProfilingStrategy::PerfMap,
|
Ok("perf") => ProfilingStrategy::PerfMap,
|
||||||
|
Ok("jitdump") => ProfilingStrategy::JitDump,
|
||||||
|
Ok("vtune") => ProfilingStrategy::VTune,
|
||||||
Ok(x) => {
|
Ok(x) => {
|
||||||
log::warn!("Invalid value for EXTISM_PROFILE: {x}");
|
log::warn!("Invalid value for EXTISM_PROFILE: {x}");
|
||||||
ProfilingStrategy::None
|
ProfilingStrategy::None
|
||||||
@@ -116,12 +120,6 @@ fn profiling_strategy() -> ProfilingStrategy {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Raise an error when the epoch deadline is encountered - this is used for timeout/cancellation
|
|
||||||
// to stop a plugin that is executing
|
|
||||||
fn deadline_callback(_: StoreContextMut<CurrentPlugin>) -> Result<UpdateDeadline, Error> {
|
|
||||||
Err(Error::msg("timeout"))
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Plugin {
|
impl Plugin {
|
||||||
/// Create a new plugin from the given manifest, and host functions. The `with_wasi` parameter determines
|
/// Create a new plugin from the given manifest, and host functions. The `with_wasi` parameter determines
|
||||||
/// whether or not the module should be executed with WASI enabled.
|
/// whether or not the module should be executed with WASI enabled.
|
||||||
@@ -141,13 +139,36 @@ impl Plugin {
|
|||||||
imports: impl IntoIterator<Item = Function>,
|
imports: impl IntoIterator<Item = Function>,
|
||||||
with_wasi: bool,
|
with_wasi: bool,
|
||||||
) -> Result<Plugin, Error> {
|
) -> Result<Plugin, Error> {
|
||||||
// Create a new engine, if the `EXTISM_DEBUG` environment variable is set
|
Self::build_new(wasm, imports, with_wasi, Default::default())
|
||||||
// then we enable debug info
|
}
|
||||||
|
|
||||||
|
pub(crate) fn build_new(
|
||||||
|
wasm: impl AsRef<[u8]>,
|
||||||
|
imports: impl IntoIterator<Item = Function>,
|
||||||
|
with_wasi: bool,
|
||||||
|
mut debug_options: DebugOptions,
|
||||||
|
) -> Result<Plugin, Error> {
|
||||||
|
// Configure debug options
|
||||||
|
debug_options.debug_info =
|
||||||
|
debug_options.debug_info || std::env::var("EXTISM_DEBUG").is_ok();
|
||||||
|
if let Ok(x) = std::env::var("EXTISM_COREDUMP") {
|
||||||
|
debug_options.coredump = Some(std::path::PathBuf::from(x));
|
||||||
|
};
|
||||||
|
if let Ok(x) = std::env::var("EXTISM_MEMDUMP") {
|
||||||
|
debug_options.memdump = Some(std::path::PathBuf::from(x));
|
||||||
|
};
|
||||||
|
let profiling_strategy = debug_options
|
||||||
|
.profiling_strategy
|
||||||
|
.map_or(ProfilingStrategy::None, |_| profiling_strategy());
|
||||||
|
debug_options.profiling_strategy = Some(profiling_strategy.clone());
|
||||||
|
|
||||||
|
// Setup wasmtime types
|
||||||
let engine = Engine::new(
|
let engine = Engine::new(
|
||||||
Config::new()
|
Config::new()
|
||||||
.epoch_interruption(true)
|
.epoch_interruption(true)
|
||||||
.debug_info(std::env::var("EXTISM_DEBUG").is_ok())
|
.debug_info(debug_options.debug_info)
|
||||||
.profiler(profiling_strategy()),
|
.coredump_on_trap(debug_options.coredump.is_some())
|
||||||
|
.profiler(profiling_strategy),
|
||||||
)?;
|
)?;
|
||||||
let mut imports = imports.into_iter();
|
let mut imports = imports.into_iter();
|
||||||
let (manifest, modules) = manifest::load(&engine, wasm.as_ref())?;
|
let (manifest, modules) = manifest::load(&engine, wasm.as_ref())?;
|
||||||
@@ -245,6 +266,7 @@ impl Plugin {
|
|||||||
output: Output::default(),
|
output: Output::default(),
|
||||||
_functions: imports.collect(),
|
_functions: imports.collect(),
|
||||||
needs_reset: false,
|
needs_reset: false,
|
||||||
|
debug_options,
|
||||||
};
|
};
|
||||||
|
|
||||||
plugin.current_plugin_mut().store = &mut plugin.store;
|
plugin.current_plugin_mut().store = &mut plugin.store;
|
||||||
@@ -576,11 +598,10 @@ impl Plugin {
|
|||||||
.map(std::time::Duration::from_millis),
|
.map(std::time::Duration::from_millis),
|
||||||
})
|
})
|
||||||
.unwrap();
|
.unwrap();
|
||||||
self.store.epoch_deadline_callback(deadline_callback);
|
|
||||||
|
|
||||||
// Call the function
|
// Call the function
|
||||||
let mut results = vec![wasmtime::Val::null(); n_results];
|
let mut results = vec![wasmtime::Val::null(); n_results];
|
||||||
let res = func.call(self.store_mut(), &[], results.as_mut_slice());
|
let mut res = func.call(self.store_mut(), &[], results.as_mut_slice());
|
||||||
|
|
||||||
// Stop timer
|
// Stop timer
|
||||||
self.timer_tx
|
self.timer_tx
|
||||||
@@ -592,13 +613,24 @@ impl Plugin {
|
|||||||
self.get_output_after_call();
|
self.get_output_after_call();
|
||||||
self.needs_reset = name == "_start";
|
self.needs_reset = name == "_start";
|
||||||
|
|
||||||
|
let mut msg = None;
|
||||||
|
|
||||||
if self.output.error_offset != 0 && self.output.error_length != 0 {
|
if self.output.error_offset != 0 && self.output.error_length != 0 {
|
||||||
let handle = MemoryHandle {
|
let handle = MemoryHandle {
|
||||||
offset: self.output.error_offset,
|
offset: self.output.error_offset,
|
||||||
length: self.output.error_length,
|
length: self.output.error_length,
|
||||||
};
|
};
|
||||||
if let Ok(e) = self.current_plugin_mut().memory_str(handle) {
|
if let Ok(e) = self.current_plugin_mut().memory_str(handle) {
|
||||||
return Err((Error::msg(e.to_string()), -1));
|
let x = e.to_string();
|
||||||
|
error!("Call to {name} returned with error message: {}", x);
|
||||||
|
|
||||||
|
// If `res` is `Ok` and there is an error message set, then convert the response
|
||||||
|
// to an `Error`
|
||||||
|
if res.is_ok() {
|
||||||
|
res = Err(Error::msg(x));
|
||||||
|
} else {
|
||||||
|
msg = Some(x);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -613,24 +645,69 @@ impl Plugin {
|
|||||||
// Return result to caller
|
// Return result to caller
|
||||||
Ok(0)
|
Ok(0)
|
||||||
}
|
}
|
||||||
Err(e) => match e.downcast::<wasmtime_wasi::I32Exit>() {
|
Err(e) => {
|
||||||
Ok(exit) => {
|
if let Some(coredump) = e.downcast_ref::<wasmtime::WasmCoreDump>() {
|
||||||
trace!("WASI return code: {}", exit.0);
|
if let Some(file) = self.debug_options.coredump.clone() {
|
||||||
if exit.0 != 0 {
|
debug!("Saving coredump to {}", file.display());
|
||||||
return Err((Error::msg("WASI return code"), exit.0));
|
|
||||||
|
if let Err(e) =
|
||||||
|
std::fs::write(file, coredump.serialize(self.store_mut(), "extism"))
|
||||||
|
{
|
||||||
|
error!("Unable to write coredump: {:?}", e);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return Ok(0);
|
|
||||||
}
|
}
|
||||||
Err(e) => {
|
|
||||||
let cause = e.root_cause().to_string();
|
if let Some(file) = &self.debug_options.memdump.clone() {
|
||||||
if cause == "timeout" || cause == "oom" {
|
trace!("Memory dump enabled");
|
||||||
return Err((Error::msg(cause), -1));
|
if let Some(memory) = self.current_plugin_mut().memory() {
|
||||||
|
debug!("Dumping memory to {}", file.display());
|
||||||
|
let data = memory.data(&mut self.store);
|
||||||
|
if let Err(e) = std::fs::write(file, &data) {
|
||||||
|
error!("Unable to write memory dump: {:?}", e);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
error!("Unable to get extism memory for writing to disk");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let wasi_exit_code = e
|
||||||
|
.downcast_ref::<wasmtime_wasi::I32Exit>()
|
||||||
|
.map(|e| e.0)
|
||||||
|
.or_else(|| {
|
||||||
|
e.downcast_ref::<wasmtime_wasi::preview2::I32Exit>()
|
||||||
|
.map(|e| e.0)
|
||||||
|
});
|
||||||
|
if let Some(exit_code) = wasi_exit_code {
|
||||||
|
trace!("WASI exit code: {}", exit_code);
|
||||||
|
if exit_code == 0 && msg.is_none() {
|
||||||
|
return Ok(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
error!("Call: {e:?}");
|
return Err((
|
||||||
return Err((e.context("Call failed"), -1));
|
Error::msg(msg.unwrap_or_else(|| "WASI exit code".to_string())),
|
||||||
|
exit_code,
|
||||||
|
));
|
||||||
}
|
}
|
||||||
},
|
|
||||||
|
// Handle timeout interrupts
|
||||||
|
if let Some(wasmtime::Trap::Interrupt) = e.downcast_ref::<wasmtime::Trap>() {
|
||||||
|
trace!("Call to {name} timed out");
|
||||||
|
return Err((Error::msg("timeout"), -1));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle out-of-memory error from `MemoryLimiter`
|
||||||
|
let cause = e.root_cause().to_string();
|
||||||
|
if cause == "oom" {
|
||||||
|
return Err((Error::msg(cause), -1));
|
||||||
|
}
|
||||||
|
|
||||||
|
error!("Call to {name} encountered an error: {e:?}");
|
||||||
|
return Err((
|
||||||
|
e.context(msg.unwrap_or_else(|| "Error in Extism plugin call".to_string())),
|
||||||
|
-1,
|
||||||
|
));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -718,7 +795,7 @@ pub(crate) enum GuestRuntime {
|
|||||||
///
|
///
|
||||||
/// # const WASM: &[u8] = include_bytes!("../../wasm/code.wasm");
|
/// # const WASM: &[u8] = include_bytes!("../../wasm/code.wasm");
|
||||||
/// // Convert from `Plugin` to `MyPlugin`
|
/// // Convert from `Plugin` to `MyPlugin`
|
||||||
/// let mut plugin: MyPlugin = extism::Plugin::new(WASM, [], true).unwrap().into();
|
/// let mut plugin: MyPlugin = extism::Plugin::new(WASM, [], true).unwrap().try_into().unwrap();
|
||||||
/// // and call the `count_vowels` function
|
/// // and call the `count_vowels` function
|
||||||
/// let count = plugin.count_vowels("this is a test").unwrap();
|
/// let count = plugin.count_vowels("this is a test").unwrap();
|
||||||
/// ```
|
/// ```
|
||||||
@@ -730,9 +807,15 @@ macro_rules! typed_plugin {
|
|||||||
unsafe impl Send for $name {}
|
unsafe impl Send for $name {}
|
||||||
unsafe impl Sync for $name {}
|
unsafe impl Sync for $name {}
|
||||||
|
|
||||||
impl From<$crate::Plugin> for $name {
|
impl TryFrom<$crate::Plugin> for $name {
|
||||||
fn from(x: $crate::Plugin) -> Self {
|
type Error = $crate::Error;
|
||||||
$name(x)
|
fn try_from(mut x: $crate::Plugin) -> Result<Self, Self::Error> {
|
||||||
|
$(
|
||||||
|
if !x.function_exists(stringify!($f)) {
|
||||||
|
return Err($crate::Error::msg(format!("Invalid function: {}", stringify!($f))));
|
||||||
|
}
|
||||||
|
)*
|
||||||
|
Ok($name(x))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -5,11 +5,20 @@ enum Source {
|
|||||||
Data(Vec<u8>),
|
Data(Vec<u8>),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Default, Clone)]
|
||||||
|
pub(crate) struct DebugOptions {
|
||||||
|
pub(crate) profiling_strategy: Option<wasmtime::ProfilingStrategy>,
|
||||||
|
pub(crate) coredump: Option<std::path::PathBuf>,
|
||||||
|
pub(crate) memdump: Option<std::path::PathBuf>,
|
||||||
|
pub(crate) debug_info: bool,
|
||||||
|
}
|
||||||
|
|
||||||
/// PluginBuilder is used to configure and create `Plugin` instances
|
/// PluginBuilder is used to configure and create `Plugin` instances
|
||||||
pub struct PluginBuilder {
|
pub struct PluginBuilder {
|
||||||
source: Source,
|
source: Source,
|
||||||
wasi: bool,
|
wasi: bool,
|
||||||
functions: Vec<Function>,
|
functions: Vec<Function>,
|
||||||
|
debug_options: DebugOptions,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl PluginBuilder {
|
impl PluginBuilder {
|
||||||
@@ -19,6 +28,7 @@ impl PluginBuilder {
|
|||||||
source: Source::Data(data.into()),
|
source: Source::Data(data.into()),
|
||||||
wasi: false,
|
wasi: false,
|
||||||
functions: vec![],
|
functions: vec![],
|
||||||
|
debug_options: DebugOptions::default(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -28,6 +38,7 @@ impl PluginBuilder {
|
|||||||
source: Source::Manifest(manifest),
|
source: Source::Manifest(manifest),
|
||||||
wasi: false,
|
wasi: false,
|
||||||
functions: vec![],
|
functions: vec![],
|
||||||
|
debug_options: DebugOptions::default(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -68,11 +79,34 @@ impl PluginBuilder {
|
|||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn with_profiling_strategy(mut self, p: wasmtime::ProfilingStrategy) -> Self {
|
||||||
|
self.debug_options.profiling_strategy = Some(p);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn with_coredump(mut self, path: impl AsRef<std::path::Path>) -> Self {
|
||||||
|
self.debug_options.coredump = Some(path.as_ref().to_path_buf());
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn with_memdump(mut self, path: impl AsRef<std::path::Path>) -> Self {
|
||||||
|
self.debug_options.memdump = Some(path.as_ref().to_path_buf());
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn with_debug_info(mut self) -> Self {
|
||||||
|
self.debug_options.debug_info = true;
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
/// Generate a new plugin with the configured settings
|
/// Generate a new plugin with the configured settings
|
||||||
pub fn build(self) -> Result<Plugin, Error> {
|
pub fn build(self) -> Result<Plugin, Error> {
|
||||||
match self.source {
|
match self.source {
|
||||||
Source::Manifest(m) => Plugin::new_with_manifest(&m, self.functions, self.wasi),
|
Source::Manifest(m) => {
|
||||||
Source::Data(d) => Plugin::new(d, self.functions, self.wasi),
|
let data = serde_json::to_vec(&m)?;
|
||||||
|
Plugin::build_new(&data, self.functions, self.wasi, self.debug_options)
|
||||||
|
}
|
||||||
|
Source::Data(d) => Plugin::build_new(d, self.functions, self.wasi, self.debug_options),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -214,7 +214,9 @@ fn test_timeout() {
|
|||||||
let end = std::time::Instant::now();
|
let end = std::time::Instant::now();
|
||||||
let time = end - start;
|
let time = end - start;
|
||||||
println!("Timed out plugin ran for {:?}", time);
|
println!("Timed out plugin ran for {:?}", time);
|
||||||
assert!(output.unwrap_err().root_cause().to_string() == "timeout");
|
let s = output.unwrap_err().root_cause().to_string();
|
||||||
|
println!("{}", s);
|
||||||
|
assert!(s == "timeout");
|
||||||
// std::io::stdout().write_all(output).unwrap();
|
// std::io::stdout().write_all(output).unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -236,7 +238,7 @@ fn test_typed_plugin_macro() {
|
|||||||
hello_world,
|
hello_world,
|
||||||
);
|
);
|
||||||
|
|
||||||
let mut plugin: CountVowelsPlugin = Plugin::new(WASM, [f], true).unwrap().into();
|
let mut plugin: CountVowelsPlugin = Plugin::new(WASM, [f], true).unwrap().try_into().unwrap();
|
||||||
|
|
||||||
let Json(output0): Json<Count> = plugin.count_vowels("abc123").unwrap();
|
let Json(output0): Json<Count> = plugin.count_vowels("abc123").unwrap();
|
||||||
let Json(output1): Json<Count> = plugin.0.call("count_vowels", "abc123").unwrap();
|
let Json(output1): Json<Count> = plugin.0.call("count_vowels", "abc123").unwrap();
|
||||||
@@ -254,7 +256,7 @@ fn test_multiple_instantiations() {
|
|||||||
hello_world,
|
hello_world,
|
||||||
);
|
);
|
||||||
|
|
||||||
let mut plugin: CountVowelsPlugin = Plugin::new(WASM, [f], true).unwrap().into();
|
let mut plugin: CountVowelsPlugin = Plugin::new(WASM, [f], true).unwrap().try_into().unwrap();
|
||||||
|
|
||||||
// This is 10,001 because the wasmtime store limit is 10,000 - we want to test
|
// This is 10,001 because the wasmtime store limit is 10,000 - we want to test
|
||||||
// that our reinstantiation process is working and that limit is never hit.
|
// that our reinstantiation process is working and that limit is never hit.
|
||||||
@@ -314,7 +316,10 @@ fn test_memory_max() {
|
|||||||
let mut plugin = Plugin::new_with_manifest(&manifest, [], true).unwrap();
|
let mut plugin = Plugin::new_with_manifest(&manifest, [], true).unwrap();
|
||||||
let output: Result<String, Error> = plugin.call("count_vowels", "a".repeat(65536 * 2));
|
let output: Result<String, Error> = plugin.call("count_vowels", "a".repeat(65536 * 2));
|
||||||
assert!(output.is_err());
|
assert!(output.is_err());
|
||||||
assert!(output.unwrap_err().root_cause().to_string() == "oom");
|
|
||||||
|
let err = output.unwrap_err().root_cause().to_string();
|
||||||
|
println!("{:?}", err);
|
||||||
|
assert_eq!(err, "oom");
|
||||||
|
|
||||||
// Should pass with memory.max set to a large enough number
|
// Should pass with memory.max set to a large enough number
|
||||||
let manifest =
|
let manifest =
|
||||||
@@ -356,3 +361,47 @@ fn test_extism_error() {
|
|||||||
assert!(output.is_err());
|
assert!(output.is_err());
|
||||||
assert_eq!(output.unwrap_err().root_cause().to_string(), "TEST");
|
assert_eq!(output.unwrap_err().root_cause().to_string(), "TEST");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_extism_memdump() {
|
||||||
|
let f = Function::new(
|
||||||
|
"hello_world",
|
||||||
|
[ValType::I64],
|
||||||
|
[ValType::I64],
|
||||||
|
None,
|
||||||
|
hello_world_set_error,
|
||||||
|
);
|
||||||
|
let mut plugin = PluginBuilder::new_with_module(WASM)
|
||||||
|
.with_wasi(true)
|
||||||
|
.with_functions([f])
|
||||||
|
.with_memdump("extism.mem")
|
||||||
|
.build()
|
||||||
|
.unwrap();
|
||||||
|
let output: Result<String, Error> = plugin.call("count_vowels", "a".repeat(1024));
|
||||||
|
assert!(output.is_err());
|
||||||
|
assert!(std::path::PathBuf::from("extism.mem").exists());
|
||||||
|
let _ = std::fs::remove_file("extism.mem");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_extism_coredump() {
|
||||||
|
let f = Function::new(
|
||||||
|
"hello_world",
|
||||||
|
[ValType::I64],
|
||||||
|
[ValType::I64],
|
||||||
|
None,
|
||||||
|
hello_world_set_error,
|
||||||
|
);
|
||||||
|
let manifest = Manifest::new([extism_manifest::Wasm::data(WASM_LOOP)])
|
||||||
|
.with_timeout(std::time::Duration::from_secs(1));
|
||||||
|
let mut plugin = PluginBuilder::new(manifest)
|
||||||
|
.with_wasi(true)
|
||||||
|
.with_functions([f])
|
||||||
|
.with_coredump("extism.core")
|
||||||
|
.build()
|
||||||
|
.unwrap();
|
||||||
|
let output: Result<&[u8], Error> = plugin.call("infinite_loop", "abc123");
|
||||||
|
assert!(output.is_err());
|
||||||
|
assert!(std::path::PathBuf::from("extism.core").exists());
|
||||||
|
let _ = std::fs::remove_file("extism.core");
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user