wip

See https://github.com/bytecodealliance/wasmtime/issues/8799

I will make a 1.4.1 release after this is merged

Makefile

@@ -4,10 +4,12 @@ AEXT=a
 FEATURES?=default
 DEFAULT_FEATURES?=yes
 RUST_TARGET?=
+EXTRA_LIBS=
 
 UNAME := $(shell uname -s)
 ifeq ($(UNAME),Darwin)
 	SOEXT=dylib
+	EXTRA_LIBS=-framework Security
 endif
 
 ifeq ($(DEFAULT_FEATURES),no)
@@ -29,7 +31,8 @@ endif
 build:
 	cargo build --release $(FEATURE_FLAGS) --manifest-path libextism/Cargo.toml $(TARGET_FLAGS)
 	sed -e "s%@CMAKE_INSTALL_PREFIX@%$(DEST)%" libextism/extism.pc.in > libextism/extism.pc
-	sed -e "s%@CMAKE_INSTALL_PREFIX@%$(DEST)%" libextism/extism-static.pc.in > libextism/extism-static.pc
+	sed -e "s%@CMAKE_INSTALL_PREFIX@%$(DEST)%" \
+	    -e "s%Libs: %Libs: $(EXTRA_LIBS) %" libextism/extism-static.pc.in > libextism/extism-static.pc
 
 bench:
 	@(cargo criterion $(TARGET_FLAGS) || echo 'For nicer output use cargo-criterion: `cargo install cargo-criterion` - using `cargo bench`') && cargo bench $(TARGET_FLAGS)

README.md

@@ -51,7 +51,7 @@ started:
 | Java SDK    | <img alt="Java SDK" src="https://extism.org/img/sdk-languages/java-android.svg" width="50px"/> | https://github.com/extism/java-sdk                                      | [Sonatype](https://central.sonatype.com/artifact/org.extism.sdk/extism) |
 | .NET SDK    | <img alt=".NET SDK" src="https://extism.org/img/sdk-languages/dotnet.svg" width="50px"/>       | https://github.com/extism/dotnet-sdk <br/>(supports C# & F#!)           | [Nuget](https://www.nuget.org/packages/Extism.Sdk)                      |
 | OCaml SDK   | <img alt="OCaml SDK" src="https://extism.org/img/sdk-languages/ocaml.svg" width="50px"/>       | https://github.com/extism/ocaml-sdk                                     | [opam](https://opam.ocaml.org/packages/extism/)                         |
-| Perl SDK    | <img alt="Perl SDK" src="https://extism.org/img/sdk-languages/perl.svg" width="50px"/>         | https://github.com/extism/perl-sdk                                      | N/A                                                                     |
+| Perl SDK    | <img alt="Perl SDK" src="https://extism.org/img/sdk-languages/perl.svg" width="50px"/>         | https://github.com/extism/perl-sdk                                      | [CPAN](https://metacpan.org/pod/Extism)                                 |
 | PHP SDK     | <img alt="PHP SDK" src="https://extism.org/img/sdk-languages/php.svg" width="50px"/>           | https://github.com/extism/php-sdk                                       | [Packagist](https://packagist.org/packages/extism/extism)               |
 | Python SDK  | <img alt="Python SDK" src="https://extism.org/img/sdk-languages/python.svg" width="50px"/>     | https://github.com/extism/python-sdk                                    | [PyPi](https://pypi.org/project/extism/)                                |
 | Ruby SDK    | <img alt="Ruby SDK" src="https://extism.org/img/sdk-languages/ruby.svg" width="50px"/>         | https://github.com/extism/ruby-sdk                                      | [RubyGems](https://rubygems.org/gems/extism)                            |

kernel/.gitignore

@@ -0,0 +1,3 @@
+owi-out
+*.wat
+proofs

kernel/Cargo.toml

@@ -7,6 +7,7 @@ edition = "2021"
 
 [dev-dependencies]
 wasm-bindgen-test = "0.3.39"
+owi = {git = "https://github.com/dylibso/owi-rs"}
 
 [features]
 default = ["bounds-checking"]
@@ -15,4 +16,4 @@ bounds-checking = []
 [workspace]
 members = [
   "."
-]
+]

kernel/build.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -e
+
 export CARGO_FLAGS=""
 
 while getopts d flag

kernel/examples/alloc_length.rs

@@ -0,0 +1,21 @@
+#![no_main]
+#![no_std]
+use extism_runtime_kernel::*;
+use owi::*;
+
+main!({
+    let n = alloc(1024);
+
+    let x = u64();
+    assume(x > 0);
+
+    let m = alloc(x);
+
+    // 1. Length should equal `x` while active
+    assert(length(m) == x);
+
+    // 2. Length should equal `0` after free
+    free(m); // Free the block
+    assert(length(m) == 0);
+    free(n);
+});

kernel/examples/error.rs

@@ -0,0 +1,24 @@
+#![no_main]
+#![no_std]
+use extism_runtime_kernel::*;
+use owi::*;
+
+main!({
+    let x = u64();
+    let n = u64();
+    let m = u64();
+    assume(x > 0);
+    assume(n > 0);
+    assume(m > 0);
+
+    alloc(n);
+    alloc(m);
+    let n = alloc(x);
+    assert(error_get() == 0);
+
+    error_set(n);
+    assert(error_get() == n);
+    alloc(m);
+    error_set(0);
+    assert(error_get() == 0);
+});

kernel/examples/load_store.rs

@@ -0,0 +1,17 @@
+#![no_main]
+#![no_std]
+use extism_runtime_kernel::*;
+use owi::*;
+
+main!({
+    let x = u64();
+    assume(x > 0);
+
+    let m = alloc(x);
+    assert(length(m) == x);
+    for i in 0..x {
+        store_u8(m + i, i as u8);
+        assert(load_u8(m + i) == i as u8);
+    }
+    free(m);
+});

kernel/examples/reuse.rs

@@ -0,0 +1,39 @@
+#![no_main]
+#![no_std]
+use extism_runtime_kernel::*;
+use owi::*;
+
+main!({
+    let x = u64();
+    assume(x > 0);
+
+    let y = u64();
+    assume(y > 0);
+
+    let mut tmp = 0;
+
+    for _ in 0..y {
+        let m = alloc(x);
+        if tmp == 0 {
+            tmp = m;
+        } else {
+            // Check that the existing block is being re-used
+            assert(m == tmp);
+        }
+
+        assert(length(m) == x);
+        free(m); // Free the block
+    }
+
+    let y = u64();
+    assume(y == x + 1);
+    let n = alloc(y);
+    assert(n > tmp);
+
+    let z = u64();
+    assume(z <= x);
+    assume(x - z < 32);
+    assume(z > 0);
+    let p = alloc(z);
+    assert(p == tmp);
+});

kernel/src/bin/extism-runtime.rs

@@ -1,5 +1,5 @@
-#![no_main]
 #![no_std]
+#![no_main]
 
 pub use extism_runtime_kernel::*;
 

kernel/src/lib.rs

@@ -175,7 +175,7 @@ impl MemoryRoot {
         core::ptr::write_bytes(
             self.blocks.as_mut_ptr() as *mut u8,
             MemoryStatus::Unused as u8,
-            self_position as usize,
+            self_position as usize - core::mem::size_of::<MemoryRoot>(),
         );
 
         // Clear extism runtime metadata
@@ -638,4 +638,15 @@ mod test {
 
         assert_eq!(last, 0);
     }
+
+    #[wasm_bindgen_test]
+    fn test_sym() {
+        unsafe {
+            reset();
+            let a = alloc(47);
+            let b = alloc(20);
+            assert_eq!(length(a), 47);
+            assert_eq!(length(b), 20);
+        }
+    }
 }

kernel/verify.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -e
+
+OUT_DIR=./target/wasm32-unknown-unknown/release/examples/
+get_proof() {
+   cp "$OUT_DIR/$1.wasm" "./proofs/$1.wasm"
+}
+
+cargo build --examples --release --target wasm32-unknown-unknown --no-default-features
+
+mkdir -p proofs
+get_proof alloc_length
+get_proof load_store
+get_proof reuse
+get_proof error
+
+for proof in $(ls proofs/*.wasm); do
+  echo "Checking $proof"
+  owi  conc "$proof" $@
+  echo
+  echo "---"
+done

libextism/.clang-format

@@ -0,0 +1,2 @@
+BasedOnStyle: LLVM
+IndentWidth: 2

libextism/Makefile

@@ -4,7 +4,7 @@ build:
 
 .PHONY: static
 static:
-	$(CC) -g -o example example.c -l:libextism.a -lm
+	$(CC) -g -o example example.c -l:libextism.a -lm -lpthread
 
 # if needed, set PKG_CONFIG_PATH= to the directory with extism*.pc installed
 LDFLAGS=`pkg-config --libs extism`

libextism/README.md

@@ -56,7 +56,6 @@ Since you may not have an Extism plug-in on hand to test, let's load a demo plug
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <unistd.h>
 
 void print_plugin_output(ExtismPlugin *plugin, int32_t rc){
   if (rc != EXTISM_SUCCESS) {
@@ -64,9 +63,9 @@ void print_plugin_output(ExtismPlugin *plugin, int32_t rc){
     return;
   }
 
-  size_t outlen = extism_plugin_output_length(plugin);
+  ExtismSize outlen = extism_plugin_output_length(plugin);
   const uint8_t *out = extism_plugin_output_data(plugin);
-  write(STDOUT_FILENO, out, outlen);
+  fwrite(out, 1, outlen, stdout);
 }
 
 int main(void) {
@@ -106,9 +105,9 @@ if (rc != EXTISM_SUCCESS) {
   exit(2);
 }
 
-size_t outlen = extism_plugin_output_length(plugin);
+ExtismSize outlen = extism_plugin_output_length(plugin);
 const uint8_t *out = extism_plugin_output_data(plugin);
-write(STDOUT_FILENO, out, outlen);
+fwrite(out, 1, outlen, stdout);
 ```
 
 Will print
@@ -171,7 +170,6 @@ We want to expose two functions to our plugin, `kv_write(key: String, value: Byt
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <unistd.h>
 
 // A stubbed out KV store
 typedef struct KVStore KVStore;
@@ -184,14 +182,14 @@ extern const uint32_t fake_kv_store_get(KVStore *kv, const char *key,
 
 // Our host functions to access the fake KV store
 void kv_get(ExtismCurrentPlugin *plugin, const ExtismVal *inputs,
-            size_t ninputs, ExtismVal *outputs, size_t noutputs,
+            ExtismSize ninputs, ExtismVal *outputs, ExtismSize noutputs,
             void *userdata) {
   // Cast the userdata pointer
   KVStore *kv = (KVStore *)userdata;
 
   // Get the offset to the key in the plugin memory
   uint64_t offs = inputs[0].v.i64;
-  size_t keylen = extism_current_plugin_memory_length(plugin, offs);
+  ExtismSize keylen = extism_current_plugin_memory_length(plugin, offs);
 
   // Allocate a new block to return
   uint64_t outoffs =
@@ -205,23 +203,23 @@ void kv_get(ExtismCurrentPlugin *plugin, const ExtismVal *inputs,
   *(uint64_t *)(extism_current_plugin_memory(plugin) + outoffs) = value;
 
   // Return the offset to our allocated block
-  outputs[0].t = PTR;
+  outputs[0].t = EXTISM_PTR;
   outputs[0].v.i64 = outoffs;
 }
 
 void kv_set(ExtismCurrentPlugin *plugin, const ExtismVal *inputs,
-            size_t ninputs, ExtismVal *outputs, size_t noutputs,
+            ExtismSize ninputs, ExtismVal *outputs, ExtismSize noutputs,
             void *userdata) {
   // Cast the userdata pointer
   KVStore *kv = (KVStore *)userdata;
 
   // Get the offset to the key in the plugin memory
   uint64_t keyoffs = inputs[0].v.i64;
-  size_t keylen = extism_current_plugin_memory_length(plugin, keyoffs);
+  ExtismSize keylen = extism_current_plugin_memory_length(plugin, keyoffs);
 
   // Get the offset to the value in the plugin memory
   uint64_t valueoffs = inputs[1].v.i64;
-  size_t valuelen = extism_current_plugin_memory_length(plugin, valueoffs);
+  ExtismSize valuelen = extism_current_plugin_memory_length(plugin, valueoffs);
 
   // Set key => value
   fake_kv_store_set(
@@ -234,13 +232,13 @@ int main(void) {
   const char *manifest = "{\"wasm\": [{\"url\": "
                          "\"https://github.com/extism/plugins/releases/latest/"
                          "download/count_vowels_kvstore.wasm\"}]}";
-  const ExtismValType kv_get_inputs[] = {PTR};
-  const ExtismValType kv_get_outputs[] = {PTR};
+  const ExtismValType kv_get_inputs[] = {EXTISM_PTR};
+  const ExtismValType kv_get_outputs[] = {EXTISM_PTR};
   ExtismFunction *kv_get_fn = extism_function_new(
       "kv_get", kv_get_inputs, 1, kv_get_outputs, 1, kv_get, kv, NULL);
 
-  const ExtismValType kv_set_inputs[] = {PTR};
-  const ExtismValType kv_set_outputs[] = {PTR};
+  const ExtismValType kv_set_inputs[] = {EXTISM_PTR};
+  const ExtismValType kv_set_outputs[] = {EXTISM_PTR};
   ExtismFunction *kv_set_fn = extism_function_new(
       "kv_set", kv_set_inputs, 1, kv_set_outputs, 1, kv_set, kv, NULL);
   const ExtismFunction *functions[] = {kv_get_fn};
@@ -288,5 +286,3 @@ print_plugin_output(plugin, extism_plugin_call(plugin, "count_vowels",
 # => Writing value=6 from key=count-vowels"
 # => {"count": 3, "total": 6, "vowels": "aeiouAEIOU"}
 ```
-
-

libextism/example.c

@@ -9,7 +9,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
-void log_handler(const char *line, uintptr_t length) {
+void log_handler(const char *line, ExtismSize length) {
   fwrite(line, length, 1, stderr);
 }
 
@@ -63,8 +63,8 @@ int main(int argc, char *argv[]) {
 
   size_t len = 0;
   uint8_t *data = read_file("../wasm/code-functions.wasm", &len);
-  ExtismValType inputs[] = {PTR};
-  ExtismValType outputs[] = {PTR};
+  ExtismValType inputs[] = {EXTISM_PTR};
+  ExtismValType outputs[] = {EXTISM_PTR};
   ExtismFunction *f =
       extism_function_new("hello_world", inputs, 1, outputs, 1, hello_world,
                           "Hello, again!", free_data);

libextism/extism-static.pc.in

@@ -5,6 +5,6 @@ includedir=${prefix}/include
 Version: 1.0.0
 Name: Extism
 Description: The framework for building with WebAssembly (wasm).
-Libs: -L${libdir} -l:libextism.a
-Libs.private: -lm
+Libs: ${libdir}/libextism.a
+Libs.private: -lm -lpthread
 Cflags: -I${includedir}

libextism/extism.pc.in

@@ -6,5 +6,5 @@ Version: 1.0.0
 Name: Extism
 Description: The framework for building with WebAssembly (wasm).
 Libs: -L${libdir} -lextism
-Libs.private: -lm
+Libs.private: -lm -lpthread
 Cflags: -I${includedir}

runtime/Cargo.toml

@@ -10,14 +10,14 @@ version.workspace = true
 
 [dependencies]
 wasmtime = ">= 20.0.0, < 22.0.0"
-wasmtime-wasi = ">= 20.0.0, < 22.0.0"
+wasi-common = ">= 20.0.0, < 22.0.0"
 anyhow = "1"
 serde = {version = "1", features = ["derive"]}
 serde_json = "1"
 toml = "0.8"
 sha2 = "0.10"
 tracing = "0.1"
-tracing-subscriber = {version = "0.3", features = ["std", "env-filter", "fmt"]}
+tracing-subscriber = {version = "0.3.18", features = ["std", "env-filter", "fmt"]}
 url = "2"
 glob = "0.3"
 ureq = {version = "2.5", optional=true}

runtime/src/current_plugin.rs

@@ -311,51 +311,29 @@ impl CurrentPlugin {
         id: uuid::Uuid,
     ) -> Result<Self, Error> {
         let wasi = if wasi {
-            let mut ctx = wasmtime_wasi::WasiCtxBuilder::new();
-
-            // Disable sockets/DNS lookup
-            ctx.allow_ip_name_lookup(true)
-                .allow_tcp(true)
-                .allow_udp(true)
-                .allow_blocking_current_thread(true);
+            let auth = wasi_common::sync::ambient_authority();
+            let random = wasi_common::sync::random_ctx();
+            let clocks = wasi_common::sync::clocks_ctx();
+            let sched = wasi_common::sync::sched_ctx();
+            let table = wasi_common::Table::new();
+            let ctx = wasi_common::WasiCtx::new(random, clocks, sched, table);
 
             if let Some(a) = &manifest.allowed_paths {
                 for (k, v) in a.iter() {
-                    ctx.preopened_dir(
-                        k,
-                        v.to_string_lossy(),
-                        wasmtime_wasi::DirPerms::READ | wasmtime_wasi::DirPerms::MUTATE,
-                        wasmtime_wasi::FilePerms::READ | wasmtime_wasi::FilePerms::WRITE,
-                    )?;
+                    let file = Box::new(wasi_common::sync::dir::Dir::from_cap_std(
+                        wasi_common::sync::Dir::open_ambient_dir(k, auth)?,
+                    ));
+                    ctx.push_preopened_dir(file, v)?;
                 }
             }
 
-            if let Some(h) = &manifest.allowed_hosts {
-                let h = h.clone();
-                ctx.socket_addr_check(move |addr, _kind| {
-                    for host in h.iter() {
-                        let addrs = std::net::ToSocketAddrs::to_socket_addrs(&host);
-                        if let Ok(addrs) = addrs {
-                            for a in addrs.into_iter() {
-                                if addr == &a {
-                                    return true;
-                                }
-                            }
-                        }
-                    }
-
-                    false
-                });
-            }
-
             // Enable WASI output, typically used for debugging purposes
             if std::env::var("EXTISM_ENABLE_WASI_OUTPUT").is_ok() {
-                ctx.inherit_stdout().inherit_stderr();
+                ctx.set_stderr(Box::new(wasi_common::sync::stdio::stderr()));
+                ctx.set_stdout(Box::new(wasi_common::sync::stdio::stdout()));
             }
 
-            Some(Wasi {
-                ctx: ctx.build_p1(),
-            })
+            Some(Wasi { ctx })
         } else {
             None
         };

runtime/src/internal.rs

@@ -3,7 +3,7 @@ use crate::*;
 /// WASI context
 pub struct Wasi {
     /// wasi
-    pub ctx: wasmtime_wasi::preview1::WasiP1Ctx,
+    pub ctx: wasi_common::WasiCtx,
 }
 
 /// InternalExt provides a unified way of acessing `memory`, `store` and `internal` values

runtime/src/plugin.rs

@@ -250,7 +250,7 @@ fn relink(
 
     // If wasi is enabled then add it to the linker
     if with_wasi {
-        wasmtime_wasi::preview1::add_to_linker_sync(&mut linker, |x: &mut CurrentPlugin| {
+        wasi_common::sync::add_to_linker(&mut linker, |x: &mut CurrentPlugin| {
             &mut x.wasi.as_mut().unwrap().ctx
         })?;
     }
@@ -834,7 +834,7 @@ impl Plugin {
                     }
                 }
 
-                let wasi_exit_code = e.downcast_ref::<wasmtime_wasi::I32Exit>().map(|e| e.0);
+                let wasi_exit_code = e.downcast_ref::<wasi_common::I32Exit>().map(|e| e.0);
                 if let Some(exit_code) = wasi_exit_code {
                     debug!(
                         plugin = self.id.to_string(),
@@ -899,7 +899,13 @@ impl Plugin {
         let data = input.to_bytes()?;
         self.raw_call(&mut lock, name, data, None)
             .map_err(|e| e.0)
-            .and_then(move |_| self.output())
+            .and_then(move |rc| {
+                if rc != 0 {
+                    Err(Error::msg(format!("Returned non-zero exit code: {rc}")))
+                } else {
+                    self.output()
+                }
+            })
     }
 
     pub fn call_with_host_context<'a, 'b, T, U, C>(