Implement Elixir / Erlang Host SDK

.github/workflows/ci.yml

@@ -11,6 +11,8 @@ jobs:
   build_and_test:
     name: Build & Test
     runs-on: ${{ matrix.os }}
+    env:
+      MIX_ENV: test
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
@@ -45,6 +47,20 @@ jobs:
         shell: bash
         run: sudo make install
 
+      - name: Setup Elixir Host SDK
+        if: ${{ runner.os != 'macOS' }}
+        uses: erlef/setup-beam@v1
+        with:
+          experimental-otp: true
+          otp-version: '25.0.4'
+          elixir-version: '1.14.0'
+
+      - name: Test Elixir Host SDK
+        if: ${{ runner.os != 'macOS' }}
+        run: |
+         cd elixir
+         LD_LIBRARY_PATH=/usr/local/lib mix do deps.get, test
+
       - name: Setup Go env
         uses: actions/setup-go@v3
 

Cargo.toml

@@ -4,3 +4,7 @@ members = [
   "runtime",
   "rust",
 ]
+exclude = [
+  "elixir/native/extism_nif"
+]
+

elixir/.formatter.exs

@@ -0,0 +1,4 @@
+# Used by "mix format"
+[
+  inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"]
+]

elixir/.gitignore

@@ -0,0 +1,28 @@
+# The directory Mix will write compiled artifacts to.
+/_build/
+
+# If you run "mix test --cover", coverage assets end up here.
+/cover/
+
+# The directory Mix downloads your dependencies sources to.
+/deps/
+
+# Where third-party dependencies like ExDoc output generated docs.
+/doc/
+
+# Ignore .fetch files in case you like to edit your project deps locally.
+/.fetch
+
+# If the VM crashes, it generates a dump, let's ignore it too.
+erl_crash.dump
+
+# Also ignore archive artifacts (built via "mix archive.build").
+*.ez
+
+# Ignore package tarball (built via "mix hex.build").
+extism-*.tar
+
+# Temporary files, for example, from tests.
+/tmp/
+
+/priv/

elixir/README.md

@@ -0,0 +1,38 @@
+# Extism
+
+Extism Host SDK for Elixir and Erlang
+
+## Installation
+
+> **N.B.:**: This library is not yet published to hex.pm
+
+```elixir
+def deps do
+  [
+    {:extism, "~> 0.0.1-rc.4"}
+  ]
+end
+```
+
+## Usage
+
+```elixir
+# Create a context for which plugins can be allocated and cleaned
+ctx = Extism.Context.new()
+
+# point to some wasm code, this is the count_vowels example that ships with extism
+manifest = %{ wasm: [ %{ path: "/Users/ben/code/extism/wasm/code.wasm" } ]}
+{:ok, plugin} = Extism.Context.new_plugin(ctx, manifest, false)
+# {:ok,
+# %Extism.Plugin{
+#   resource: 0,
+#   reference: #Reference<0.520418104.1263009793.80956>
+# }}
+{:ok, output} = Extism.Plugin.call(plugin, "count_vowels", "this is a test")
+# {:ok, "{\"count\": 4}"}
+{:ok, result} = JSON.decode(output)
+# {:ok, %{"count" => 4}}
+
+# free up the context and any plugins we allocated
+Extism.Context.free(ctx)
+```

elixir/lib/extism.ex

@@ -0,0 +1,5 @@
+defmodule Extism do
+  def set_log_file(filepath, level) do
+    Extism.Native.set_log_file(filepath, level)
+  end
+end

elixir/lib/extism/context.ex

@@ -0,0 +1,33 @@
+defmodule Extism.Context do
+  defstruct [
+    # The actual NIF Resource. A pointer in this case
+    ptr: nil
+  ]
+
+  def wrap_resource(ptr) do
+    %__MODULE__{
+      ptr: ptr
+    }
+  end
+
+  def new() do
+    ptr = Extism.Native.context_new()
+    Extism.Context.wrap_resource(ptr)
+  end
+
+  def reset(ctx) do
+    Extism.Native.context_reset(ctx.ptr)
+  end
+
+  def free(ctx) do
+    Extism.Native.context_free(ctx.ptr)
+  end
+
+  def new_plugin(ctx, manifest, wasi) do
+    {:ok, manifest_payload} = JSON.encode(manifest)
+    case Extism.Native.plugin_new_with_manifest(ctx.ptr, manifest_payload, wasi) do
+      {:error, err} -> {:error, err}
+      res -> {:ok, Extism.Plugin.wrap_resource(ctx, res)}
+    end
+  end
+end

elixir/lib/extism/native.ex

@@ -0,0 +1,17 @@
+defmodule Extism.Native do
+  use Rustler,
+  otp_app: :extism,
+  crate: :extism_nif
+
+  def context_new(), do: error()
+  def context_reset(_ctx), do: error()
+  def context_free(_ctx), do: error()
+  def plugin_new_with_manifest(_ctx, _manifest, _wasi), do: error()
+  def plugin_call(_ctx, _plugin_id, _name, _input), do: error()
+  def plugin_update_manifest(_ctx, _plugin_id, _manifest, _wasi), do: error()
+  def plugin_has_function(_ctx, _plugin_id, _function_name), do: error()
+  def plugin_free(_ctx, _plugin_id), do: error()
+  def set_log_file(_filename, _level), do: error()
+
+  defp error, do: :erlang.nif_error(:nif_not_loaded)
+end

elixir/lib/extism/plugin.ex

@@ -0,0 +1,45 @@
+defmodule Extism.Plugin do
+  defstruct [
+    # The actual NIF Resource. PluginIndex and the context
+    plugin_id: nil,
+    ctx: nil
+  ]
+
+  def wrap_resource(ctx, plugin_id) do
+    %__MODULE__{
+      ctx: ctx,
+      plugin_id: plugin_id
+    }
+  end
+
+  def call(plugin, name, input) do
+    case Extism.Native.plugin_call(plugin.ctx.ptr, plugin.plugin_id, name, input) do
+      {:error, err} -> {:error, err}
+      res -> {:ok, res}
+    end
+  end
+
+   def update(plugin, manifest, wasi) when is_map(manifest) do
+     {:ok, manifest_payload} = JSON.encode(manifest)
+     case Extism.Native.plugin_update_manifest(plugin.ctx.ptr, plugin.plugin_id, manifest_payload, wasi) do
+       {:error, err} -> {:error, err}
+       res -> :ok
+     end
+   end
+
+   def free(plugin) do
+     Extism.Native.plugin_free(plugin.ctx.ptr, plugin.plugin_id)
+   end
+
+   def has_function(plugin, function_name) do
+     Extism.Native.plugin_has_function(plugin.ctx.ptr, plugin.plugin_id, function_name)
+   end
+end
+
+defimpl Inspect, for: Extim.Plugin do
+  import Inspect.Algebra
+
+  def inspect(dict, opts) do
+    concat(["#Extism.Plugin<", to_doc(dict.plugin_id, opts), ">"])
+  end
+end

elixir/mix.exs

@@ -0,0 +1,37 @@
+defmodule Extism.MixProject do
+  use Mix.Project
+
+  def project do
+    [
+      app: :extism,
+      version: "0.0.1-rc.4",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps(),
+      aliases: aliases()
+    ]
+  end
+
+  # Run "mix help compile.app" to learn about applications.
+  def application do
+    [
+      extra_applications: [:logger]
+    ]
+  end
+
+  defp deps do
+    [
+      {:rustler, "~> 0.26.0"},
+      {:json, "~> 1.4"},
+    ]
+  end
+
+  defp aliases do
+    [
+      fmt: [
+        "format",
+        "cmd cargo fmt --manifest-path native/io/Cargo.toml"
+      ]
+    ]
+  end
+end

elixir/mix.lock

@@ -0,0 +1,6 @@
+%{
+  "jason": {:hex, :jason, "1.4.0", "e855647bc964a44e2f67df589ccf49105ae039d4179db7f6271dfd3843dc27e6", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "79a3791085b2a0f743ca04cec0f7be26443738779d09302e01318f97bdb82121"},
+  "json": {:hex, :json, "1.4.1", "8648f04a9439765ad449bc56a3ff7d8b11dd44ff08ffcdefc4329f7c93843dfa", [:mix], [], "hexpm", "9abf218dbe4ea4fcb875e087d5f904ef263d012ee5ed21d46e9dbca63f053d16"},
+  "rustler": {:hex, :rustler, "0.26.0", "06a2773d453ee3e9109efda643cf2ae633dedea709e2455ac42b83637c9249bf", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "42961e9d2083d004d5a53e111ad1f0c347efd9a05cb2eb2ffa1d037cdc74db91"},
+  "toml": {:hex, :toml, "0.6.2", "38f445df384a17e5d382befe30e3489112a48d3ba4c459e543f748c2f25dd4d1", [:mix], [], "hexpm", "d013e45126d74c0c26a38d31f5e8e9b83ea19fc752470feb9a86071ca5a672fa"},
+}

elixir/native/extism_nif/.cargo/config

@@ -0,0 +1,15 @@
+[target.'cfg(target_os = "macos")']
+rustflags = [
+    "-C", "link-arg=-undefined",
+    "-C", "link-arg=dynamic_lookup",
+]
+
+# See https://github.com/rust-lang/rust/issues/59302
+[target.x86_64-unknown-linux-musl]
+rustflags = [
+  "-C", "target-feature=-crt-static"
+]
+
+# Provides a small build size, but takes more time to build.
+[profile.release]
+lto = true

elixir/native/extism_nif/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "extism_nif"
+version = "0.0.1-rc.3"
+edition = "2021"
+authors = ["Benjamin Eckel <bhelx@simst.im>"]
+
+[lib]
+name = "extism_nif"
+path = "src/lib.rs"
+crate-type = ["cdylib"]
+
+[dependencies]
+rustler = "0.26.0"
+extism = { version = "0.0.1-rc.3", path = "../../../rust" }
+log = "0.4"
+

elixir/native/extism_nif/src/lib.rs

@@ -0,0 +1,149 @@
+use rustler::{Atom, Env, Term, ResourceArc};
+use extism::{Plugin, Context};
+use std::str;
+use std::path::Path;
+use std::str::FromStr;
+use std::sync::RwLock;
+use std::mem;
+
+mod atoms {
+    rustler::atoms! {
+        ok,
+        error,
+        unknown // Other error
+    }
+}
+
+struct ExtismContext {
+    ctx: RwLock<Context>
+}
+
+fn load(env: Env, _: Term) -> bool {
+    rustler::resource!(ExtismContext, env);
+    true
+}
+
+fn to_rustler_error(extism_error: extism::Error) -> rustler::Error {
+    match extism_error {
+        extism::Error::UnableToLoadPlugin(msg) => rustler::Error::Term(Box::new(msg)),
+        extism::Error::Message(msg) => rustler::Error::Term(Box::new(msg)),
+        extism::Error::Json(json_err) => rustler::Error::Term(Box::new(json_err.to_string()))
+    }
+}
+
+#[rustler::nif]
+fn context_new() -> ResourceArc<ExtismContext> {
+    ResourceArc::new(
+        ExtismContext { ctx: RwLock::new(Context::new()) }
+    )
+}
+
+#[rustler::nif]
+fn context_reset(ctx: ResourceArc<ExtismContext>) {
+    let context = &mut ctx.ctx.write().unwrap();
+    context.reset()
+}
+
+#[rustler::nif]
+fn context_free(ctx: ResourceArc<ExtismContext>) {
+    let context = &ctx.ctx.read().unwrap();
+    std::mem::drop(context)
+}
+
+#[rustler::nif]
+fn plugin_new_with_manifest(ctx: ResourceArc<ExtismContext>, manifest_payload: String, wasi: bool) -> Result<i32, rustler::Error> {
+    let context = &ctx.ctx.write().unwrap();
+    let result = match Plugin::new(context, manifest_payload, wasi) {
+        Err(e) => Err(to_rustler_error(e)),
+        Ok(plugin) => {
+            let plugin_id = plugin.as_i32();
+            // this forget should be safe because the context will clean up
+            // all it's plugins when it is dropped
+            mem::forget(plugin);
+            Ok(plugin_id)
+        }
+    };
+    result
+}
+
+#[rustler::nif]
+fn plugin_call(ctx: ResourceArc<ExtismContext>, plugin_id: i32, name: String, input: String) -> Result<String, rustler::Error> {
+    let context = &ctx.ctx.read().unwrap();
+    let plugin = unsafe { Plugin::from_id(plugin_id, context) };
+    let result = match plugin.call(name, input) {
+        Err(e) => Err(to_rustler_error(e)),
+        Ok(result) => {
+            match str::from_utf8(&result) {
+                Ok(output) => Ok(output.to_string()),
+                Err(_e) => Err(rustler::Error::Term(Box::new("Could not read output from plugin")))
+            }
+        }
+    };
+    // this forget should be safe because the context will clean up
+    // all it's plugins when it is dropped
+    mem::forget(plugin);
+    result
+}
+
+#[rustler::nif]
+fn plugin_update_manifest(ctx: ResourceArc<ExtismContext>, plugin_id: i32, manifest_payload: String, wasi: bool) -> Result<(), rustler::Error> {
+    let context = &ctx.ctx.read().unwrap();
+    let mut plugin = unsafe { Plugin::from_id(plugin_id, context) };
+    let result = match plugin.update(manifest_payload, wasi) {
+        Ok(()) => {
+            Ok(())
+        },
+        Err(e) => Err(to_rustler_error(e))
+    };
+    // this forget should be safe because the context will clean up
+    // all it's plugins when it is dropped
+    mem::forget(plugin);
+    result
+}
+
+#[rustler::nif]
+fn plugin_free(ctx: ResourceArc<ExtismContext>, plugin_id: i32) -> Result<(), rustler::Error> {
+    let context = &ctx.ctx.read().unwrap();
+    let plugin = unsafe { Plugin::from_id(plugin_id, context) };
+    std::mem::drop(plugin);
+    Ok(())
+}
+
+#[rustler::nif]
+fn set_log_file(filename: String, log_level: String) -> Result<Atom, rustler::Error> {
+    let path = Path::new(&filename);
+    match log::Level::from_str(&log_level) {
+        Err(_e) => Err(rustler::Error::Term(Box::new(format!("{} not a valid log level", log_level)))),
+        Ok(level) => {
+            extism::set_log_file(path, Some(level));
+            Ok(atoms::ok())
+        }
+    }
+}
+
+#[rustler::nif]
+fn plugin_has_function(ctx: ResourceArc<ExtismContext>, plugin_id: i32, function_name: String) -> Result<bool, rustler::Error> {
+    let context = &ctx.ctx.read().unwrap();
+    let plugin = unsafe { Plugin::from_id(plugin_id, context) };
+    let has_function = plugin.has_function(function_name);
+    // this forget should be safe because the context will clean up
+    // all it's plugins when it is dropped
+    mem::forget(plugin);
+    Ok(has_function)
+}
+
+rustler::init!(
+    "Elixir.Extism.Native",
+    [
+        context_new,
+        context_reset,
+        context_free,
+        plugin_new_with_manifest,
+        plugin_call,
+        plugin_update_manifest,
+        plugin_has_function,
+        plugin_free,
+        set_log_file,
+    ],
+    load = load
+);

elixir/test/extism_test.exs

@@ -0,0 +1,81 @@
+defmodule ExtismTest do
+  use ExUnit.Case
+  doctest Extism
+
+  test "context create & reset" do
+    ctx = Extism.Context.new()
+    path = Path.join([__DIR__, "../../wasm/code.wasm"])
+    manifest = %{wasm: [%{path: path}]}
+    {:ok, plugin} = Extism.Context.new_plugin(ctx, manifest, false)
+    Extism.Context.reset(ctx)
+    # we should expect an error after resetting context
+    {:error, _err} = Extism.Plugin.call(plugin, "count_vowels", "this is a test")
+  end
+
+  defp new_plugin() do
+    ctx = Extism.Context.new()
+    path = Path.join([__DIR__, "../../wasm/code.wasm"])
+    manifest = %{wasm: [%{path: path}]}
+    {:ok, plugin} = Extism.Context.new_plugin(ctx, manifest, false)
+    {ctx, plugin}
+  end
+
+  test "counts vowels" do
+    {ctx, plugin} = new_plugin()
+    {:ok, output} = Extism.Plugin.call(plugin, "count_vowels", "this is a test")
+    assert JSON.decode(output) == {:ok, %{"count" => 4}}
+    Extism.Context.free(ctx)
+  end
+
+  test "can make multiple calls on a plugin" do
+    {ctx, plugin} = new_plugin()
+    {:ok, output} = Extism.Plugin.call(plugin, "count_vowels", "this is a test")
+    assert JSON.decode(output) == {:ok, %{"count" => 4}}
+    {:ok, output} = Extism.Plugin.call(plugin, "count_vowels", "this is a test again")
+    assert JSON.decode(output) == {:ok, %{"count" => 7}}
+    {:ok, output} = Extism.Plugin.call(plugin, "count_vowels", "this is a test thrice")
+    assert JSON.decode(output) == {:ok, %{"count" => 6}}
+    Extism.Context.free(ctx)
+  end
+
+  test "can free a plugin" do
+    {ctx, plugin} = new_plugin()
+    {:ok, output} = Extism.Plugin.call(plugin, "count_vowels", "this is a test")
+    assert JSON.decode(output) == {:ok, %{"count" => 4}}
+    Extism.Plugin.free(plugin)
+    # Expect an error when calling a plugin that was freed
+    {:error, _err} = Extism.Plugin.call(plugin, "count_vowels", "this is a test")
+    Extism.Context.free(ctx)
+  end
+
+  test "can update manifest" do
+    {ctx, plugin} = new_plugin()
+    path = Path.join([__DIR__, "../../wasm/code.wasm"])
+    manifest = %{wasm: [%{path: path}]}
+    assert Extism.Plugin.update(plugin, manifest, true) == :ok
+    Extism.Context.free(ctx)
+  end
+
+  test "errors on bad manifest" do
+    ctx = Extism.Context.new()
+    {:error, _msg} = Extism.Context.new_plugin(ctx, %{"wasm" => 123}, false)
+    Extism.Context.free(ctx)
+  end
+
+  test "errors on unknown function" do
+    {ctx, plugin} = new_plugin()
+    {:error, _msg} = Extism.Plugin.call(plugin, "unknown", "this is a test")
+    Extism.Context.free(ctx)
+  end
+
+  test "set_log_file" do
+    Extism.set_log_file("/tmp/logfile.log", "debug")
+  end
+
+  test "has_function" do
+    {ctx, plugin} = new_plugin()
+    assert Extism.Plugin.has_function(plugin, "count_vowels")
+    assert !Extism.Plugin.has_function(plugin, "unknown")
+    Extism.Context.free(ctx)
+  end
+end

elixir/test/test_helper.exs

@@ -0,0 +1 @@
+ExUnit.start()

ruby/lib/extism/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Extism
-  VERSION = "0.1.0"
+  VERSION = "0.0.1.rc4"
 end

rust/src/context.rs

@@ -23,6 +23,9 @@ impl Context {
     }
 }
 
+unsafe impl Send for Context {}
+unsafe impl Sync for Context {}
+
 impl Drop for Context {
     fn drop(&mut self) {
         if self.pointer.is_null() {

rust/src/plugin.rs

@@ -6,6 +6,17 @@ pub struct Plugin<'a> {
 }
 
 impl<'a> Plugin<'a> {
+    pub unsafe fn from_id(id: i32, context: &'a Context) -> Plugin<'a> {
+        Plugin {
+            id,
+            context: context,
+        }
+    }
+
+    pub fn as_i32(&self) -> i32 {
+        self.id
+    }
+
     /// Create a new plugin from the given manifest
     pub fn new_with_manifest(
         ctx: &'a Context,