//! # Extism kernel //! //! - Isolated memory from both host and plugin //! - An allocator for managing that memory //! - Input/output handling //! - Error message handling //! //! ## Allocator //! //! The Extism allocator is a bump allocator that tracks the `length` of the total number of bytes //! available to the allocator and `position` to track how much of the data has been used. Things like memory //! have not really been optimized at all. When a new allocation that is larger than the remaning size is made, //! the allocator attempts to call `memory.grow` if that fails a `0` offset is returned, which should be interpreted //! as a failed allocation. //! //! ## Input/Output //! //! Input and output are just allocated blocks of memory that are marked as either input or output using //! the `input_set` or `output_set` functions. The MemoryRoot field `input_offset` contains //! the offset in memory to the input data and `input_length` contains the size of the input data. `output_offset` //! and `output_length` are used for the output data. //! //! ## Error handling //! //! The `error` field is used to track the current error message. If it is set to `0` then there is no error. //! The length of the error message can be retreived using `length`. //! //! ## Memory offsets //! An offset of `0` is similar to a `NULL` pointer in C - it implies an allocation failure or memory error //! of some kind //! //! ## Extism functions //! //! These functions are backward compatible with the pre-kernel runtime, but a few new functions are added to //! give runtimes more access to the internals necesarry to load data in and out of a plugin. #![no_std] #![allow(clippy::missing_safety_doc)] use core::sync::atomic::*; pub type Pointer = u64; pub type Handle = u64; /// WebAssembly page size const PAGE_SIZE: usize = 65536; /// Provides information about the usage status of a `MemoryBlock` #[repr(u8)] #[derive(PartialEq)] pub enum MemoryStatus { /// Unused memory that is available b Unused = 0, /// In-use memory Active = 1, /// Free memory that is available for re-use Free = 2, } /// A single `MemoryRoot` exists at the start of the memory to track information about the total /// size of the allocated memory and the position of the bump allocator. /// /// The overall layout of the Extism-manged memory is organized like this: /// |------|-------+---------|-------+--------------| /// | Root | Block + Data | Block + Data | ... /// |------|-------+---------|-------+--------------| /// /// Where `Root` and `Block` are fixed to the size of the `MemoryRoot` and `MemoryBlock` structs. But /// the size of `Data` is dependent on the allocation size. /// /// This means that the offset of a `Block` is the size of `Root` plus the size of all existing `Blocks` /// including their data. #[repr(C)] pub struct MemoryRoot { /// Set to true after initialization pub initialized: AtomicBool, /// Position of the bump allocator, relative to `blocks` field pub position: AtomicU64, /// The total size of all data allocated using this allocator pub length: AtomicU64, /// Offset of error block pub error: AtomicU64, /// Input position in memory pub input_offset: Handle, /// Input length pub input_length: u64, /// Output position in memory pub output_offset: Pointer, /// Output length pub output_length: u64, /// A pointer to the start of the first block pub blocks: [MemoryBlock; 0], } /// A `MemoryBlock` contains some metadata about a single allocation #[repr(C)] pub struct MemoryBlock { /// The usage status of the block, `Unused` or `Free` blocks can be re-used. pub status: AtomicU8, /// The total size of the allocation pub size: usize, /// The number of bytes currently being used. If this block is a fresh allocation then `size` and `used` will /// always be the same. If a block is re-used then these numbers may differ. pub used: usize, /// A pointer to the block data pub data: [u8; 0], } /// Returns the number of pages needed for the given number of bytes pub fn num_pages(nbytes: u64) -> usize { let npages = nbytes / PAGE_SIZE as u64; let remainder = nbytes % PAGE_SIZE as u64; if remainder != 0 { (npages + 1) as usize } else { npages as usize } } // Get the `MemoryRoot`, this is always stored at offset 1 in memory #[inline] unsafe fn memory_root() -> &'static mut MemoryRoot { &mut *(1 as *mut MemoryRoot) } impl MemoryRoot { /// Initialize or load the `MemoryRoot` from the correct position in memory pub unsafe fn new() -> &'static mut MemoryRoot { let root = memory_root(); // If this fails then `INITIALIZED` is already `true` and we can just return the // already initialized `MemoryRoot` if root .initialized .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) .is_err() { return root; } // Ensure that at least one page is allocated to store the `MemoryRoot` data if core::arch::wasm32::memory_size(0) == 0 && core::arch::wasm32::memory_grow(0, 1) == usize::MAX { core::arch::wasm32::unreachable() } root.input_offset = 0; root.input_length = 0; root.output_offset = 0; root.output_length = 0; root.error.store(0, Ordering::Release); // Initialize the `MemoryRoot` length, position and data root.length.store( PAGE_SIZE as u64 - core::mem::size_of::() as u64, Ordering::Release, ); root.position.store(0, Ordering::Release); // Ensure the first block is marked as `Unused` #[allow(clippy::size_of_in_element_count)] core::ptr::write_bytes( root.blocks.as_mut_ptr() as *mut _, MemoryStatus::Unused as u8, core::mem::size_of::(), ); root } /// Resets the position of the allocator and zeroes out all allocations pub unsafe fn reset(&mut self) { // Clear allocated data let self_position = self.position.fetch_and(0, Ordering::SeqCst); core::ptr::write_bytes( self.blocks.as_mut_ptr() as *mut u8, MemoryStatus::Unused as u8, self_position as usize, ); // Clear extism runtime metadata self.error.store(0, Ordering::Release); self.input_offset = 0; self.input_length = 0; self.output_offset = 0; self.output_length = 0; } #[inline(always)] #[allow(unused)] fn pointer_in_bounds(&self, p: Pointer) -> bool { let start_ptr = self.blocks.as_ptr() as Pointer; p >= start_ptr && p < start_ptr + self.length.load(Ordering::Acquire) as Pointer } #[inline(always)] #[allow(unused)] fn pointer_in_bounds_fast(p: Pointer) -> bool { // Similar to `pointer_in_bounds` but less accurate on the upper bound. This uses the total memory size, // instead of checking `MemoryRoot::length` let end = (core::arch::wasm32::memory_size(0) as u64) << 16; p >= core::mem::size_of::() as Pointer && p <= end as u64 } // Find a block that is free to use, this can be a new block or an existing freed block. The `self_position` argument // is used to avoid loading the allocators position more than once when performing an allocation. unsafe fn find_free_block( &mut self, length: u64, self_position: u64, ) -> Option<&'static mut MemoryBlock> { // Get the first block let mut block = self.blocks.as_mut_ptr(); // Only loop while the block pointer is less then the current position while (block as u64) < self.blocks.as_ptr() as u64 + self_position { let b = &mut *block; // Get the block status, this lets us know if we are able to re-use it let status = b.status.load(Ordering::Acquire); // An unused block is safe to use if status == MemoryStatus::Unused as u8 { return Some(b); } // Re-use freed blocks when they're large enough if status == MemoryStatus::Free as u8 && b.size >= length as usize { // Split block if there is too much excess if b.size - length as usize >= 128 { b.size -= length as usize + core::mem::size_of::(); b.used = 0; let block1 = b.data.as_mut_ptr().add(b.size) as *mut MemoryBlock; let b1 = &mut *block1; b1.size = length as usize; b1.used = 0; b1.status.store(MemoryStatus::Free as u8, Ordering::Release); return Some(b1); } // Otherwise return the whole block return Some(b); } // Get the next block block = b.next_ptr(); } None } /// Create a new `MemoryBlock`, when `Some(block)` is returned, `block` will contain at least enough room for `length` bytes /// but may be as large as `length` + `BLOCK_SPLIT_SIZE` bytes. When `None` is returned the allocation has failed. pub unsafe fn alloc(&mut self, length: u64) -> Option<&'static mut MemoryBlock> { let self_position = self.position.load(Ordering::Acquire); let self_length = self.length.load(Ordering::Acquire); // Get the current index for a new block let curr = self.blocks.as_ptr() as u64 + self_position; // Get the number of bytes available let mem_left = self_length - self_position - core::mem::size_of::() as u64; let length_with_block = length + core::mem::size_of::() as u64; // When the allocation is larger than the number of bytes available // we will need to try to grow the memory if length_with_block >= mem_left { // If the current position is large enough to hold the length of the block being // allocated then check for existing free blocks that can be re-used before // growing memory if length_with_block <= self_position { let b = self.find_free_block(length, self_position); // If there's a free block then re-use it if let Some(b) = b { b.used = length as usize; b.status .store(MemoryStatus::Active as u8, Ordering::Release); return Some(b); } } // Calculate the number of pages needed to cover the remaining bytes let npages = num_pages(length_with_block - mem_left); let x = core::arch::wasm32::memory_grow(0, npages); if x == usize::MAX { return None; } self.length .fetch_add(npages as u64 * PAGE_SIZE as u64, Ordering::SeqCst); } // Bump the position by the size of the actual data + the size of the MemoryBlock structure self.position.fetch_add( length + core::mem::size_of::() as u64, Ordering::SeqCst, ); // Initialize a new block at the current position let ptr = curr as *mut MemoryBlock; let block = &mut *ptr; block .status .store(MemoryStatus::Active as u8, Ordering::Release); block.size = length as usize; block.used = length as usize; Some(block) } /// Finds the block at an offset in memory pub unsafe fn find_block(&mut self, offs: Pointer) -> Option<&mut MemoryBlock> { if !Self::pointer_in_bounds_fast(offs) { return None; } // Get the first block let mut block = self.blocks.as_mut_ptr(); // Only loop while the block pointer is less then the current position while (block as u64) < self.blocks.as_ptr() as u64 + offs { let b = &mut *block; // Get the block status, this lets us know if we are able to re-use it let status = b.status.load(Ordering::Acquire); if status == MemoryStatus::Active as u8 && b.data.as_ptr() as Pointer == offs { return Some(b); } // Get the next block block = b.next_ptr(); } None } } impl MemoryBlock { /// Get a pointer to the next block /// /// NOTE: This does no checking to ensure the resulting pointer is valid, the offset /// is calculated based on metadata provided by the current block #[inline] pub unsafe fn next_ptr(&mut self) -> *mut MemoryBlock { self.data.as_mut_ptr().add(self.size) as *mut MemoryBlock } /// Mark a block as free pub fn free(&mut self) { self.status .store(MemoryStatus::Free as u8, Ordering::Release); } } // Extism functions /// Allocate a block of memory and return the offset #[no_mangle] pub unsafe fn alloc(n: u64) -> Handle { if n == 0 { return 0; } let region = MemoryRoot::new(); let block = region.alloc(n); match block { Some(block) => block.data.as_mut_ptr() as Handle, None => 0, } } /// Free allocated memory #[no_mangle] pub unsafe fn free(p: Handle) { if p == 0 { return; } let root = MemoryRoot::new(); let block = root.find_block(p); if let Some(block) = block { block.free(); // If the input pointer is freed for some reason, make sure the input length to 0 // since the original data is gone if p == root.input_offset { root.input_length = 0; } } } /// Get the length of an allocated memory block /// /// Note: this should only be called on memory handles returned /// by a call to `alloc` - it will return garbage on invalid offsets #[no_mangle] pub unsafe fn length_unsafe(p: Handle) -> u64 { if p == 0 { return 0; } if !MemoryRoot::pointer_in_bounds_fast(p) { return 0; } let ptr = p - core::mem::size_of::() as u64; let block = &mut *(ptr as *mut MemoryBlock); // Simplest sanity check to verify the pointer is a block if block.status.load(Ordering::Acquire) != MemoryStatus::Active as u8 { return 0; } block.used as u64 } /// Get the length but returns 0 if the offset is not a valid handle. /// /// Note: this function walks each node in the allocations list, which ensures correctness, but is also /// slow #[no_mangle] pub unsafe fn length(p: Pointer) -> u64 { if p == 0 { return 0; } if let Some(block) = MemoryRoot::new().find_block(p) { block.used as u64 } else { 0 } } /// Load a byte from Extism-managed memory #[no_mangle] pub unsafe fn load_u8(p: Pointer) -> u8 { #[cfg(feature = "bounds-checking")] if !MemoryRoot::pointer_in_bounds_fast(p) { return 0; } *(p as *mut u8) } /// Load a u64 from Extism-managed memory #[no_mangle] pub unsafe fn load_u64(p: Pointer) -> u64 { #[cfg(feature = "bounds-checking")] if !MemoryRoot::pointer_in_bounds_fast(p + core::mem::size_of::() as u64 - 1) { return 0; } *(p as *mut u64) } /// Load a byte from the input data #[no_mangle] pub unsafe fn input_load_u8(offset: u64) -> u8 { let root = MemoryRoot::new(); #[cfg(feature = "bounds-checking")] if offset >= root.input_length { return 0; } *((root.input_offset + offset) as *mut u8) } /// Load a u64 from the input data #[no_mangle] pub unsafe fn input_load_u64(offset: u64) -> u64 { let root = MemoryRoot::new(); #[cfg(feature = "bounds-checking")] if offset + core::mem::size_of::() as u64 > root.input_length { return 0; } *((root.input_offset + offset) as *mut u64) } /// Write a byte in Extism-managed memory #[no_mangle] pub unsafe fn store_u8(p: Pointer, x: u8) { #[cfg(feature = "bounds-checking")] if !MemoryRoot::pointer_in_bounds_fast(p) { return; } *(p as *mut u8) = x; } /// Write a u64 in Extism-managed memory #[no_mangle] pub unsafe fn store_u64(p: Pointer, x: u64) { #[cfg(feature = "bounds-checking")] if !MemoryRoot::pointer_in_bounds_fast(p + core::mem::size_of::() as u64 - 1) { return; } *(p as *mut u64) = x; } /// Set the range of the input data in memory /// h must always be a handle so that length works on it /// len must match length(handle) /// **Note**: this function takes ownership of the handle passed in /// the caller should not `free` this value #[no_mangle] pub unsafe fn input_set(h: Handle, len: u64) { let root = MemoryRoot::new(); #[cfg(feature = "bounds-checking")] { if !root.pointer_in_bounds(h) || !root.pointer_in_bounds(h + len - 1) { return; } } root.input_offset = h; root.input_length = len; } /// Set the range of the output data in memory /// **Note**: this function takes ownership of the handle passed in /// the caller should not `free` this value #[no_mangle] pub unsafe fn output_set(p: Pointer, len: u64) { let root = MemoryRoot::new(); #[cfg(feature = "bounds-checking")] { if !root.pointer_in_bounds(p) || !root.pointer_in_bounds(p + len - 1) { return; } } root.output_offset = p; root.output_length = len; } /// Get the input length #[no_mangle] pub fn input_length() -> u64 { unsafe { MemoryRoot::new().input_length } } /// Get the input offset in Exitsm-managed memory #[no_mangle] pub fn input_offset() -> Handle { unsafe { MemoryRoot::new().input_offset } } /// Get the output length #[no_mangle] pub fn output_length() -> u64 { unsafe { MemoryRoot::new().output_length } } /// Get the output offset in Extism-managed memory #[no_mangle] pub unsafe fn output_offset() -> Pointer { MemoryRoot::new().output_offset } /// Reset the allocator #[no_mangle] pub unsafe fn reset() { MemoryRoot::new().reset() } /// Set the error message offset, the handle passed to this /// function should not be freed after this call /// **Note**: this function takes ownership of the handle passed in /// the caller should not `free` this value #[no_mangle] pub unsafe fn error_set(h: Handle) { let root = MemoryRoot::new(); // Allow ERROR to be set to 0 if h == 0 { root.error.store(h, Ordering::SeqCst); return; } #[cfg(feature = "bounds-checking")] if !root.pointer_in_bounds(h) { return; } root.error.store(h, Ordering::SeqCst); } /// Get the error message offset, if it's `0` then no error has been set #[no_mangle] pub unsafe fn error_get() -> Handle { MemoryRoot::new().error.load(Ordering::SeqCst) } /// Get the position of the allocator, this can be used as an indication of how many bytes are currently in-use #[no_mangle] pub unsafe fn memory_bytes() -> u64 { MemoryRoot::new().length.load(Ordering::Acquire) } #[cfg(test)] mod test { use crate::*; use wasm_bindgen_test::*; #[wasm_bindgen_test] fn test_oom() { let size = 1024 * 1024 * 5; let mut last = 0; for _ in 0..1024 { unsafe { let ptr = alloc(size); last = ptr; if ptr == 0 { break; } assert_eq!(length(ptr), size); } } assert_eq!(last, 0); } }