Make Rust SDK depend directly on extism-runtime (#65)

.github/workflows/ci.yml

@@ -3,8 +3,8 @@ on: [pull_request, workflow_dispatch]
 name: CI
 
 env:
-  RUNTIME_MANIFEST: runtime/Cargo.toml
   RUNTIME_CRATE: extism-runtime
+  LIBEXTISM_CRATE: libextism
   RUST_SDK_CRATE: extism
 
 jobs:
@@ -41,7 +41,7 @@ jobs:
       - name: Build
         if: steps.cache-libextism.outputs.cache-hit != 'true'
         shell: bash
-        run: cargo build --release -p ${{ env.RUNTIME_CRATE }}
+        run: cargo build --release -p ${{ env.LIBEXTISM_CRATE }}
       - name: Upload artifact
         uses: actions/upload-artifact@v3
         with:

Cargo.toml

@@ -3,8 +3,8 @@ members = [
   "manifest",
   "runtime",
   "rust",
+  "libextism"
 ]
 exclude = [
   "elixir/native/extism_nif"
 ]
-

Makefile

@@ -25,7 +25,7 @@ lint:
 	cargo clippy --release --no-deps --manifest-path runtime/Cargo.toml
 
 build:
-	cargo build --release $(FEATURE_FLAGS) --manifest-path runtime/Cargo.toml
+	cargo build --release $(FEATURE_FLAGS) --manifest-path libextism/Cargo.toml
 
 install:
 	install runtime/extism.h $(DEST)/include

libextism/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "libextism"
+version = "0.0.0"
+edition = "2021"
+authors = ["The Extism Authors", "oss@extism.org"]
+license = "BSD-3-Clause"
+homepage = "https://extism.org"
+repository = "https://github.com/extism/extism"
+description = "libextism"
+
+[lib]
+name = "extism"
+crate-type = ["cdylib"]
+
+[dependencies]
+extism-runtime = {path = "../runtime"}
+
+[features]
+default = ["http", "register-http", "register-filesystem"]
+nn = ["extism-runtime/nn"]
+register-http = ["extism-runtime/register-http"] # enables wasm to be downloaded using http
+register-filesystem = ["extism-runtime/register-filesystem"] # enables wasm to be loaded from disk
+http = ["extism-runtime/http"] # enables extism_http_request

libextism/src/lib.rs

@@ -0,0 +1,10 @@
+//! This crate is used to generate `libextism` using `extism-runtime`
+
+pub use extism_runtime::sdk::*;
+
+#[cfg(test)]
+#[test]
+fn test_version() {
+    let s = unsafe { std::ffi::CStr::from_ptr(extism_version()) };
+    assert!(s.to_bytes() != b"0.0.0");
+}

manifest/Cargo.toml

@@ -9,7 +9,7 @@ repository = "https://github.com/extism/extism"
 description = "Extism plug-in manifest crate"
 
 [dependencies]
-serde = {version = "1", features=["derive"]}
+serde = {version = "1", features = ["derive"]}
 base64 = "0.20.0-alpha"
 schemars = {version = "0.8", optional=true}
 

runtime/Cargo.toml

@@ -8,24 +8,18 @@ homepage = "https://extism.org"
 repository = "https://github.com/extism/extism"
 description = "Extism runtime component"
 
-[lib]
-name = "extism"
-crate-type = ["cdylib"]
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
 [dependencies]
 wasmtime = "2.0.1"
 wasmtime-wasi = "2.0.1"
 wasmtime-wasi-nn = {version = "2.0.1", optional=true}
 anyhow = "1"
-serde = { version = "1", features=["derive"] }
-toml = "0.5"
+serde = {version = "1", features = ["derive"]}
 serde_json = "1"
+toml = "0.5"
 sha2 = "0.10"
 log = "0.4"
 log4rs = "1.1"
-url = "2.3"
+url = "2"
 glob = "0.3"
 ureq = {version = "2.5", optional=true}
 extism-manifest = { version = "0.0.1-rc.6", path = "../manifest" }

runtime/build.rs

@@ -1,10 +1,6 @@
-use std::env;
-
 fn main() {
-    let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
-
     if let Ok(bindings) = cbindgen::Builder::new()
-        .with_crate(crate_dir)
+        .with_crate(".")
         .with_language(cbindgen::Language::C)
         .with_no_includes()
         .with_sys_include("stdint.h")

runtime/src/context.rs

@@ -29,7 +29,7 @@ impl Context {
 
     /// Get the next valid plugin ID
     pub fn next_id(&mut self) -> Result<PluginIndex, Error> {
-        // Make sure we haven't exhausted all plugin IDs, it reach this it would require the machine
+        // Make sure we haven't exhausted all plugin IDs, to reach this it would require the machine
         // running this code to have a lot of memory - no computer I tested on was able to allocate
         // the max number of plugins.
         //
@@ -37,7 +37,7 @@ impl Context {
         // try to use one of those before returning an error
         let exhausted = self.next_id.load(std::sync::atomic::Ordering::SeqCst) == PluginIndex::MAX;
 
-        // If there is a significant number of old IDs we can start to re-use them
+        // If there are a significant number of old IDs we can start to re-use them
         if self.reclaimed_ids.len() >= START_REUSING_IDS || exhausted {
             if let Some(x) = self.reclaimed_ids.pop_front() {
                 return Ok(x);
@@ -55,6 +55,33 @@ impl Context {
             .fetch_add(1, std::sync::atomic::Ordering::SeqCst))
     }
 
+    pub fn insert(&mut self, plugin: Plugin) -> PluginIndex {
+        // Generate a new plugin ID
+        let id: i32 = match self.next_id() {
+            Ok(id) => id,
+            Err(e) => {
+                error!("Error creating Plugin: {:?}", e);
+                self.set_error(e);
+                return -1;
+            }
+        };
+        self.plugins.insert(id, plugin);
+        id
+    }
+
+    pub fn new_plugin(&mut self, data: impl AsRef<[u8]>, with_wasi: bool) -> PluginIndex {
+        let plugin = match Plugin::new(data, with_wasi) {
+            Ok(x) => x,
+            Err(e) => {
+                error!("Error creating Plugin: {:?}", e);
+                self.set_error(e);
+                return -1;
+            }
+        };
+        let id = self.insert(plugin);
+        id
+    }
+
     /// Set the context error
     pub fn set_error(&mut self, e: impl std::fmt::Debug) {
         trace!("Set context error: {:?}", e);
@@ -78,9 +105,9 @@ impl Context {
 
     /// Remove a plugin from the context
     pub fn remove(&mut self, id: PluginIndex) {
-        self.plugins.remove(&id);
-
-        // Collect old IDs in case we need to re-use them
-        self.reclaimed_ids.push_back(id);
+        if self.plugins.remove(&id).is_some() {
+            // Collect old IDs in case we need to re-use them
+            self.reclaimed_ids.push_back(id);
+        }
     }
 }

runtime/src/sdk.rs

@@ -36,29 +36,8 @@ pub unsafe extern "C" fn extism_plugin_new(
 ) -> PluginIndex {
     trace!("Call to extism_plugin_new with wasm pointer {:?}", wasm);
     let ctx = &mut *ctx;
-
     let data = std::slice::from_raw_parts(wasm, wasm_size as usize);
-    let plugin = match Plugin::new(data, with_wasi) {
-        Ok(x) => x,
-        Err(e) => {
-            error!("Error creating Plugin: {:?}", e);
-            ctx.set_error(e);
-            return -1;
-        }
-    };
-
-    // Allocate a new plugin ID
-    let id: i32 = match ctx.next_id() {
-        Ok(id) => id,
-        Err(e) => {
-            error!("Error creating Plugin: {:?}", e);
-            ctx.set_error(e);
-            return -1;
-        }
-    };
-    ctx.plugins.insert(id, plugin);
-    info!("New plugin added: {id}");
-    id
+    ctx.new_plugin(data, with_wasi)
 }
 
 /// Update a plugin, keeping the existing ID

rust/Cargo.toml

@@ -4,12 +4,13 @@ version = "0.0.1-rc.6"
 edition = "2021"
 authors = ["The Extism Authors", "oss@extism.org"]
 license = "BSD-3-Clause"
-links = "extism"
 homepage = "https://extism.org"
 repository = "https://github.com/extism/extism"
 description = "Extism Host SDK for Rust"
 
 [dependencies]
 extism-manifest = { version = "0.0.1-rc.6", path = "../manifest" }
+extism-runtime = { version = "0.0.1-rc.6", path = "../runtime"}
 serde_json = "1"
-log = "0.4"
+log = "0.4"
+thiserror = "1"

rust/build.rs

@@ -1,10 +0,0 @@
-fn main() {
-    println!("cargo:rustc-link-search=/usr/local/lib");
-
-    if let Ok(home) = std::env::var("HOME") {
-        let path = std::path::PathBuf::from(home).join(".local").join("lib");
-        println!("cargo:rustc-link-search={}", path.display());
-    }
-
-    println!("cargo:rustc-link-lib=extism");
-}

rust/src/context.rs

@@ -1,8 +1,6 @@
 use crate::*;
 
-pub struct Context {
-    pub(crate) pointer: *mut bindings::ExtismContext,
-}
+pub struct Context(pub(crate) std::sync::Arc<std::sync::Mutex<extism_runtime::Context>>);
 
 impl Default for Context {
     fn default() -> Context {
@@ -13,25 +11,20 @@ impl Default for Context {
 impl Context {
     /// Create a new context
     pub fn new() -> Context {
-        let pointer = unsafe { bindings::extism_context_new() };
-        Context { pointer }
+        Context(std::sync::Arc::new(std::sync::Mutex::new(
+            extism_runtime::Context::new(),
+        )))
     }
 
     /// Remove all registered plugins
     pub fn reset(&mut self) {
-        unsafe { bindings::extism_context_reset(self.pointer) }
+        unsafe { bindings::extism_context_reset(&mut *self.lock()) }
     }
-}
 
-unsafe impl Send for Context {}
-unsafe impl Sync for Context {}
-
-impl Drop for Context {
-    fn drop(&mut self) {
-        if self.pointer.is_null() {
-            return;
+    pub(crate) fn lock<'a>(&'a self) -> std::sync::MutexGuard<'a, extism_runtime::Context> {
+        match self.0.lock() {
+            Ok(x) => x,
+            Err(x) => x.into_inner(),
         }
-        unsafe { bindings::extism_context_free(self.pointer) }
-        self.pointer = std::ptr::null_mut();
     }
 }

rust/src/lib.rs

@@ -1,9 +1,5 @@
-use std::collections::BTreeMap;
-
-use extism_manifest::Manifest;
-
-#[allow(non_camel_case_types)]
-pub(crate) mod bindings;
+pub use extism_manifest::{self as manifest, Manifest};
+pub use extism_runtime::sdk as bindings;
 
 mod context;
 mod plugin;
@@ -11,17 +7,16 @@ mod plugin;
 pub use context::Context;
 pub use plugin::Plugin;
 
-#[derive(Debug)]
+#[derive(Debug, thiserror::Error)]
 pub enum Error {
+    #[error("Unable to load plugin: {0}")]
     UnableToLoadPlugin(String),
+    #[error("{0}")]
     Message(String),
-    Json(serde_json::Error),
-}
-
-impl From<serde_json::Error> for Error {
-    fn from(e: serde_json::Error) -> Self {
-        Error::Json(e)
-    }
+    #[error("JSON: {0}")]
+    Json(#[from] serde_json::Error),
+    #[error("Runtime: {0}")]
+    Runtime(#[from] extism_runtime::Error),
 }
 
 /// Gets the version of Extism
@@ -54,7 +49,7 @@ mod tests {
         let wasm_start = Instant::now();
         set_log_file("test.log", Some(log::Level::Info));
         let context = Context::new();
-        let plugin = Plugin::new(&context, WASM, false).unwrap();
+        let mut plugin = Plugin::new(&context, WASM, false).unwrap();
         println!("register loaded plugin: {:?}", wasm_start.elapsed());
 
         let repeat = 1182;
@@ -146,20 +141,20 @@ mod tests {
         use std::io::Write;
         std::thread::spawn(|| {
             let context = Context::new();
-            let plugin = Plugin::new(&context, WASM, false).unwrap();
+            let mut plugin = Plugin::new(&context, WASM, false).unwrap();
             let output = plugin.call("count_vowels", "this is a test").unwrap();
             std::io::stdout().write_all(output).unwrap();
         });
 
         std::thread::spawn(|| {
             let context = Context::new();
-            let plugin = Plugin::new(&context, WASM, false).unwrap();
+            let mut plugin = Plugin::new(&context, WASM, false).unwrap();
             let output = plugin.call("count_vowels", "this is a test aaa").unwrap();
             std::io::stdout().write_all(output).unwrap();
         });
 
         let context = Context::new();
-        let plugin = Plugin::new(&context, WASM, false).unwrap();
+        let mut plugin = Plugin::new(&context, WASM, false).unwrap();
         let output = plugin.call("count_vowels", "abc123").unwrap();
         std::io::stdout().write_all(output).unwrap();
     }

rust/src/plugin.rs

@@ -1,16 +1,14 @@
 use crate::*;
+use std::collections::BTreeMap;
 
 pub struct Plugin<'a> {
-    id: bindings::ExtismPlugin,
+    id: extism_runtime::PluginIndex,
     context: &'a Context,
 }
 
 impl<'a> Plugin<'a> {
     pub unsafe fn from_id(id: i32, context: &'a Context) -> Plugin<'a> {
-        Plugin {
-            id,
-            context: context,
-        }
+        Plugin { id, context }
     }
 
     pub fn as_i32(&self) -> i32 {
@@ -19,7 +17,7 @@ impl<'a> Plugin<'a> {
 
     /// Create a new plugin from the given manifest
     pub fn new_with_manifest(
-        ctx: &'a Context,
+        ctx: &'a mut Context,
         manifest: &Manifest,
         wasi: bool,
     ) -> Result<Plugin<'a>, Error> {
@@ -29,17 +27,10 @@ impl<'a> Plugin<'a> {
 
     /// Create a new plugin from a WASM module
     pub fn new(ctx: &'a Context, data: impl AsRef<[u8]>, wasi: bool) -> Result<Plugin, Error> {
-        let plugin = unsafe {
-            bindings::extism_plugin_new(
-                ctx.pointer,
-                data.as_ref().as_ptr(),
-                data.as_ref().len() as u64,
-                wasi,
-            )
-        };
+        let plugin = ctx.lock().new_plugin(data, wasi);
 
         if plugin < 0 {
-            let err = unsafe { bindings::extism_error(ctx.pointer, -1) };
+            let err = unsafe { bindings::extism_error(&mut *ctx.lock(), -1) };
             let buf = unsafe { std::ffi::CStr::from_ptr(err) };
             let buf = buf.to_str().unwrap().to_string();
             return Err(Error::UnableToLoadPlugin(buf));
@@ -55,7 +46,7 @@ impl<'a> Plugin<'a> {
     pub fn update(&mut self, data: impl AsRef<[u8]>, wasi: bool) -> Result<(), Error> {
         let b = unsafe {
             bindings::extism_plugin_update(
-                self.context.pointer,
+                &mut *self.context.lock(),
                 self.id,
                 data.as_ref().as_ptr(),
                 data.as_ref().len() as u64,
@@ -66,7 +57,7 @@ impl<'a> Plugin<'a> {
             return Ok(());
         }
 
-        let err = unsafe { bindings::extism_error(self.context.pointer, -1) };
+        let err = unsafe { bindings::extism_error(&mut *self.context.lock(), -1) };
         if !err.is_null() {
             let s = unsafe { std::ffi::CStr::from_ptr(err) };
             return Err(Error::Message(s.to_str().unwrap().to_string()));
@@ -82,11 +73,11 @@ impl<'a> Plugin<'a> {
     }
 
     /// Set configuration values
-    pub fn set_config(&self, config: &BTreeMap<String, Option<String>>) -> Result<(), Error> {
+    pub fn set_config(&mut self, config: &BTreeMap<String, Option<String>>) -> Result<(), Error> {
         let encoded = serde_json::to_vec(config)?;
         unsafe {
             bindings::extism_plugin_config(
-                self.context.pointer,
+                &mut *self.context.lock(),
                 self.id,
                 encoded.as_ptr() as *const _,
                 encoded.len() as u64,
@@ -96,7 +87,7 @@ impl<'a> Plugin<'a> {
     }
 
     /// Set configuration values, builder-style
-    pub fn with_config(self, config: &BTreeMap<String, Option<String>>) -> Result<Self, Error> {
+    pub fn with_config(mut self, config: &BTreeMap<String, Option<String>>) -> Result<Self, Error> {
         self.set_config(config)?;
         Ok(self)
     }
@@ -106,7 +97,7 @@ impl<'a> Plugin<'a> {
         let name = std::ffi::CString::new(name.as_ref()).expect("Invalid function name");
         unsafe {
             bindings::extism_plugin_function_exists(
-                self.context.pointer,
+                &mut *self.context.lock(),
                 self.id,
                 name.as_ptr() as *const _,
             )
@@ -114,11 +105,11 @@ impl<'a> Plugin<'a> {
     }
 
     /// Call a function with the given input
-    pub fn call(&self, name: impl AsRef<str>, input: impl AsRef<[u8]>) -> Result<&[u8], Error> {
+    pub fn call(&mut self, name: impl AsRef<str>, input: impl AsRef<[u8]>) -> Result<&[u8], Error> {
         let name = std::ffi::CString::new(name.as_ref()).expect("Invalid function name");
         let rc = unsafe {
             bindings::extism_plugin_call(
-                self.context.pointer,
+                &mut *self.context.lock(),
                 self.id,
                 name.as_ptr() as *const _,
                 input.as_ref().as_ptr() as *const _,
@@ -127,7 +118,7 @@ impl<'a> Plugin<'a> {
         };
 
         if rc != 0 {
-            let err = unsafe { bindings::extism_error(self.context.pointer, self.id) };
+            let err = unsafe { bindings::extism_error(&mut *self.context.lock(), self.id) };
             if !err.is_null() {
                 let s = unsafe { std::ffi::CStr::from_ptr(err) };
                 return Err(Error::Message(s.to_str().unwrap().to_string()));
@@ -137,9 +128,9 @@ impl<'a> Plugin<'a> {
         }
 
         let out_len =
-            unsafe { bindings::extism_plugin_output_length(self.context.pointer, self.id) };
+            unsafe { bindings::extism_plugin_output_length(&mut *self.context.lock(), self.id) };
         unsafe {
-            let ptr = bindings::extism_plugin_output_data(self.context.pointer, self.id);
+            let ptr = bindings::extism_plugin_output_data(&mut *self.context.lock(), self.id);
             Ok(std::slice::from_raw_parts(ptr, out_len as usize))
         }
     }
@@ -147,9 +138,6 @@ impl<'a> Plugin<'a> {
 
 impl<'a> Drop for Plugin<'a> {
     fn drop(&mut self) {
-        if self.context.pointer.is_null() {
-            return;
-        }
-        unsafe { bindings::extism_plugin_free(self.context.pointer, self.id) }
+        unsafe { bindings::extism_plugin_free(&mut *self.context.lock(), self.id) }
     }
 }