Compare commits

..

4 Commits

Author SHA1 Message Date
zach
3300fed506 chore: fmt 2025-06-26 13:35:48 -07:00
zach
b026d30865 chore: clippy 2025-06-26 13:34:04 -07:00
zach
e8d0397754 test(kernel): add block re-use quickcheck test 2025-06-26 13:26:50 -07:00
zach
39685a9038 cleanup(kernel): improve re-use of freed blocks 2025-06-25 10:32:29 -07:00
17 changed files with 93 additions and 172 deletions

View File

@@ -263,24 +263,24 @@ impl MemoryRoot {
let mem_left = self_length - self_position - core::mem::size_of::<MemoryRoot>() as u64;
let length_with_block = length + core::mem::size_of::<MemoryBlock>() as u64;
// If the current position is large enough to hold the length of the block being
// allocated then check for existing free blocks that can be re-used before
// growing memory
if length_with_block <= self_position {
let b = self.find_free_block(length, self_position);
// If there's a free block then re-use it
if let Some(b) = b {
b.used = length as usize;
b.status
.store(MemoryStatus::Active as u8, Ordering::Release);
return Some(b);
}
}
// When the allocation is larger than the number of bytes available
// we will need to try to grow the memory
if length_with_block >= mem_left {
// If the current position is large enough to hold the length of the block being
// allocated then check for existing free blocks that can be re-used before
// growing memory
if length_with_block <= self_position {
let b = self.find_free_block(length, self_position);
// If there's a free block then re-use it
if let Some(b) = b {
b.used = length as usize;
b.status
.store(MemoryStatus::Active as u8, Ordering::Release);
return Some(b);
}
}
// Calculate the number of pages needed to cover the remaining bytes
let npages = num_pages(length_with_block - mem_left);
let x = core::arch::wasm32::memory_grow(0, npages);

View File

@@ -1,10 +1,6 @@
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
mod local_path;
pub use local_path::LocalPath;
#[deprecated]
pub type ManifestMemory = MemoryOptions;
@@ -283,7 +279,7 @@ pub struct Manifest {
/// the path on disk to the path it should be available inside the plugin.
/// For example, `".": "/tmp"` would mount the current directory as `/tmp` inside the module
#[serde(default)]
pub allowed_paths: Option<BTreeMap<PathBuf, LocalPath>>,
pub allowed_paths: Option<BTreeMap<String, PathBuf>>,
/// The plugin timeout in milliseconds
#[serde(default)]
@@ -341,15 +337,15 @@ impl Manifest {
}
/// Add a path to `allowed_paths`
pub fn with_allowed_path(mut self, src: impl Into<LocalPath>, dest: impl AsRef<Path>) -> Self {
pub fn with_allowed_path(mut self, src: String, dest: impl AsRef<Path>) -> Self {
let dest = dest.as_ref().to_path_buf();
match &mut self.allowed_paths {
Some(p) => {
p.insert(dest, src.into());
p.insert(src, dest);
}
None => {
let mut p = BTreeMap::new();
p.insert(dest, src.into());
p.insert(src, dest);
self.allowed_paths = Some(p);
}
}
@@ -358,8 +354,8 @@ impl Manifest {
}
/// Set `allowed_paths`
pub fn with_allowed_paths(mut self, paths: impl Iterator<Item = (LocalPath, PathBuf)>) -> Self {
self.allowed_paths = Some(paths.map(|(local, wasm)| (wasm, local)).collect());
pub fn with_allowed_paths(mut self, paths: impl Iterator<Item = (String, PathBuf)>) -> Self {
self.allowed_paths = Some(paths.collect());
self
}

View File

@@ -1,119 +0,0 @@
use std::path::{Path, PathBuf};
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
#[cfg_attr(feature = "json_schema", derive(schemars::JsonSchema))]
pub enum LocalPath {
ReadOnly(PathBuf),
ReadWrite(PathBuf),
}
impl LocalPath {
pub fn as_path(&self) -> &Path {
match self {
LocalPath::ReadOnly(p) => p.as_path(),
LocalPath::ReadWrite(p) => p.as_path(),
}
}
}
impl From<&str> for LocalPath {
fn from(value: &str) -> Self {
if let Some(s) = value.strip_prefix("ro:") {
LocalPath::ReadOnly(PathBuf::from(s))
} else {
LocalPath::ReadWrite(PathBuf::from(value))
}
}
}
impl From<String> for LocalPath {
fn from(value: String) -> Self {
LocalPath::from(value.as_str())
}
}
impl From<PathBuf> for LocalPath {
fn from(value: PathBuf) -> Self {
LocalPath::ReadWrite(value)
}
}
impl From<&Path> for LocalPath {
fn from(value: &Path) -> Self {
LocalPath::ReadWrite(value.to_path_buf())
}
}
impl serde::Serialize for LocalPath {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
match self {
LocalPath::ReadOnly(path) => {
let s = match path.to_str() {
Some(s) => s,
None => {
return Err(serde::ser::Error::custom(
"Path contains invalid UTF-8 characters",
))
}
};
format!("ro:{s}").serialize(serializer)
}
LocalPath::ReadWrite(path) => path.serialize(serializer),
}
}
}
struct LocalPathVisitor;
impl serde::de::Visitor<'_> for LocalPathVisitor {
type Value = LocalPath;
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
formatter.write_str("path string")
}
fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
Ok(From::from(v))
}
fn visit_string<E>(self, v: String) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
Ok(From::from(v))
}
fn visit_bytes<E>(self, v: &[u8]) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
std::str::from_utf8(v)
.map(From::from)
.map_err(|_| serde::de::Error::invalid_value(serde::de::Unexpected::Bytes(v), &self))
}
fn visit_byte_buf<E>(self, v: Vec<u8>) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
String::from_utf8(v).map(From::from).map_err(|e| {
serde::de::Error::invalid_value(serde::de::Unexpected::Bytes(&e.into_bytes()), &self)
})
}
}
impl<'de> serde::Deserialize<'de> for LocalPath {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::de::Deserializer<'de>,
{
deserializer.deserialize_string(LocalPathVisitor)
}
}

View File

@@ -16,7 +16,7 @@ fn main() {
let res = plugin.call::<&str, &str>("try_read", "").unwrap();
println!("{:?}", res);
println!("{res:?}");
println!("-----------------------------------------------------");
@@ -30,7 +30,7 @@ fn main() {
);
let res2 = plugin.call::<&str, &str>("try_write", &line).unwrap();
println!("{:?}", res2);
println!("{res2:?}");
println!("done!");
}

View File

@@ -30,6 +30,6 @@ fn main() {
let res = plugin
.call::<&str, &str>("reflect", "Hello, world!")
.unwrap();
println!("{}", res);
println!("{res}");
}
}

View File

@@ -25,6 +25,6 @@ fn main() {
println!("Dumping logs");
for line in LOGS.lock().unwrap().iter() {
print!("{}", line);
print!("{line}");
}
}

View File

@@ -52,6 +52,6 @@ fn main() {
let res = plugin
.call::<&str, &str>("count_vowels", "Hello, world!")
.unwrap();
println!("{}", res);
println!("{res}");
}
}

View File

@@ -352,9 +352,9 @@ impl CurrentPlugin {
if let Some(a) = &manifest.allowed_paths {
for (k, v) in a.iter() {
let readonly = matches!(v, extism_manifest::LocalPath::ReadOnly(_));
let readonly = k.starts_with("ro:");
let dir_path = v.as_path();
let dir_path = if readonly { &k[3..] } else { k };
let dir = wasi_common::sync::dir::Dir::from_cap_std(
wasi_common::sync::Dir::open_ambient_dir(dir_path, auth)?,
@@ -366,7 +366,7 @@ impl CurrentPlugin {
Box::new(dir)
};
ctx.push_preopened_dir(file, k)?;
ctx.push_preopened_dir(file, v)?;
}
}

Binary file not shown.

View File

@@ -98,7 +98,7 @@ pub fn set_log_callback<F: 'static + Clone + Fn(&str)>(
let x = tracing_subscriber::EnvFilter::builder()
.with_default_directive(tracing::Level::ERROR.into());
if is_level {
x.parse_lossy(format!("extism={}", filter))
x.parse_lossy(format!("extism={filter}"))
} else {
x.parse_lossy(filter)
}

View File

@@ -10,7 +10,7 @@ use crate::*;
fn hex(data: &[u8]) -> String {
let mut s = String::new();
for &byte in data {
write!(&mut s, "{:02x}", byte).unwrap();
write!(&mut s, "{byte:02x}").unwrap();
}
s
}

View File

@@ -963,8 +963,7 @@ impl Plugin {
}
Err(msg) => {
res = Err(Error::msg(format!(
"unable to load error message from memory: {}",
msg,
"unable to load error message from memory: {msg}",
)));
}
}

View File

@@ -871,7 +871,7 @@ fn set_log_file(log_file: impl Into<std::path::PathBuf>, filter: &str) -> Result
let x = tracing_subscriber::EnvFilter::builder()
.with_default_directive(tracing::Level::ERROR.into());
if is_level {
x.parse_lossy(format!("extism={}", filter))
x.parse_lossy(format!("extism={filter}"))
} else {
x.parse_lossy(filter)
}
@@ -926,7 +926,7 @@ unsafe fn set_log_buffer(filter: &str) -> Result<(), Error> {
let x = tracing_subscriber::EnvFilter::builder()
.with_default_directive(tracing::Level::ERROR.into());
if is_level {
x.parse_lossy(format!("extism={}", filter))
x.parse_lossy(format!("extism={filter}"))
} else {
x.parse_lossy(filter)
}

View File

@@ -16,7 +16,7 @@ fn test_issue_620() {
// Call test method, this does not work
let p = plugin.call::<(), String>("test", ()).unwrap();
println!("{}", p);
println!("{p}");
}
// https://github.com/extism/extism/issues/619
@@ -53,5 +53,5 @@ fn test_issue_775() {
Ok(code) => Err(code),
}
.unwrap();
println!("{}", p);
println!("{p}");
}

View File

@@ -225,6 +225,26 @@ fn test_kernel_allocations() {
extism_free(&mut store, instance, q);
}
#[test]
fn test_kernel_page_allocations() {
let (mut store, mut instance) = init_kernel_test();
let instance = &mut instance;
let a = extism_alloc(&mut store, instance, 65500 * 4);
let a_size = extism_length(&mut store, instance, a);
let b = extism_alloc(&mut store, instance, 65539);
extism_free(&mut store, instance, a);
let c = extism_alloc(&mut store, instance, 65500 * 2);
let c_size = extism_length(&mut store, instance, c);
let d = extism_alloc(&mut store, instance, 65500);
assert_eq!(a + (a_size - c_size), c);
assert!(c < b);
assert!(d < b);
}
#[test]
fn test_kernel_error() {
let (mut store, mut instance) = init_kernel_test();
@@ -436,3 +456,31 @@ quickcheck! {
true
}
}
quickcheck! {
fn check_block_reuse(allocs: Vec<u16>) -> bool {
let (mut store, mut instance) = init_kernel_test();
let instance = &mut instance;
let init = extism_alloc(&mut store, instance, allocs.iter().map(|x| *x as u64).sum::<u64>() + allocs.len() as u64 * 64);
let bounds = init + extism_length(&mut store, instance, init);
extism_free(&mut store, instance, init);
for a in allocs {
let ptr = extism_alloc(&mut store, instance, a as u64);
if ptr == 0 {
continue
}
if extism_length(&mut store, instance, ptr) != a as u64 {
return false
}
extism_free(&mut store, instance , ptr);
if ptr > bounds {
println!("ptr={ptr}, bounds={bounds}");
return false
}
}
true
}
}

View File

@@ -9,7 +9,7 @@ fn run_thread(p: Pool, i: u64) -> std::thread::JoinHandle<()> {
.unwrap()
.call("count_vowels", "abc")
.unwrap();
println!("{}", s);
println!("{s}");
})
}

View File

@@ -133,10 +133,7 @@ fn it_works() {
.unwrap();
let native_avg: std::time::Duration = native_sum / native_num_tests as u32;
println!(
"native function call (avg, N = {}): {:?}",
native_num_tests, native_avg
);
println!("native function call (avg, N = {native_num_tests}): {native_avg:?}");
let num_tests = test_times.len();
let sum: std::time::Duration = test_times
@@ -145,7 +142,7 @@ fn it_works() {
.unwrap();
let avg: std::time::Duration = sum / num_tests as u32;
println!("wasm function call (avg, N = {}): {:?}", num_tests, avg);
println!("wasm function call (avg, N = {num_tests}): {avg:?}");
// Check that log file was written to
if log {
@@ -212,7 +209,7 @@ fn test_cancel() {
let _output: Result<&[u8], Error> = plugin.call("loop_forever", "abc123");
let end = std::time::Instant::now();
let time = end - start;
println!("Cancelled plugin ran for {:?}", time);
println!("Cancelled plugin ran for {time:?}");
}
}
@@ -271,7 +268,7 @@ fn test_fuel_consumption() {
assert!(output.is_err());
let fuel_consumed = plugin.fuel_consumed().unwrap();
println!("Fuel consumed: {}", fuel_consumed);
println!("Fuel consumed: {fuel_consumed}");
assert!(fuel_consumed > 0);
}
@@ -440,7 +437,7 @@ fn test_memory_max() {
assert!(output.is_err());
let err = output.unwrap_err().root_cause().to_string();
println!("{:?}", err);
println!("{err:?}");
assert_eq!(err, "oom");
// Should pass with memory.max set to a large enough number
@@ -503,7 +500,7 @@ fn test_extism_error() {
let mut plugin = Plugin::new(&manifest, [f], true).unwrap();
let output: Result<String, Error> = plugin.call("count_vowels", "a".repeat(1024));
assert!(output.is_err());
println!("{:?}", output);
println!("{output:?}");
assert_eq!(output.unwrap_err().root_cause().to_string(), "TEST");
}
@@ -823,7 +820,7 @@ fn test_http_response_headers() {
.unwrap();
let req = HttpRequest::new("https://extism.org");
let Json(res): Json<HashMap<String, String>> = plugin.call("http_get", Json(req)).unwrap();
println!("{:?}", res);
println!("{res:?}");
assert_eq!(res["content-type"], "text/html; charset=utf-8");
}
@@ -838,6 +835,6 @@ fn test_http_response_headers_disabled() {
.unwrap();
let req = HttpRequest::new("https://extism.org");
let Json(res): Json<HashMap<String, String>> = plugin.call("http_get", Json(req)).unwrap();
println!("{:?}", res);
println!("{res:?}");
assert!(res.is_empty());
}