feat(ruby): Host functions and clean up FFI code (#442)

Adds support for host functions and cleans up some of the FFI code.


## API

To make a host function, you can pass a proc to `Function::new`:

```ruby
func = proc do |current_plugin, inputs, outputs, user_data|
  input = current_plugin.input_as_bytes(inputs.first)
  current_plugin.return_string(outputs.first, "#{input} #{user_data}")
end
f = Extism::Function.new('transform_string', [Extism::ValType::I64], [Extism::ValType::I64], func, 'My User Data')
plugin = Extism::Plugin.new(host_manifest, [f], true)
result = plugin.call('reflect_string', 'Hello, World!')
assert_equal result, 'Hello, World! My User Data'
```

If your function is in a module or a class, you can use
`method(name).to_proc`. Example:

```ruby
module Test
  def self.my_function(current_plugin, inputs, outputs, user_data)
    input = current_plugin.input_as_bytes(inputs.first)
    current_plugin.return_string(outputs.first, "#{input} #{user_data}")
  end
end

func = Test.method(:my_function).to_proc
f = Extism::Function.new('my_function', [Extism::ValType::I64], [Extism::ValType::I64], func, 'My User Data')
```


`current_plugin` is of the type CurrentPlugin which has some helpful
methods:

* `CurrentPlugin#memory_at_offset(int)` returns a `Memory` object given
a memory pointer
* `CurrentPlugin#free(Memory)` frees the memory
* `CurrentPlugin#alloc(int)` allocates new memory and returns a `Memory`
* `CurrentPlugin#input_as_bytes(Value)` returns the bytes for the given
input param
* `CurrentPlugin#return_bytes(Value, Array)` Sets the array of bytes to
the return for the given output value
* `CurrentPlugin#input_as_bytes(Value, String)` Sets the string to the
return for the given output value

ruby/Gemfile

@@ -1,15 +1,16 @@
 # frozen_string_literal: true
 
-source "https://rubygems.org"
+source 'https://rubygems.org'
 
 # Specify your gem's dependencies in extism.gemspec
 gemspec
 
-gem "rake", "~> 13.0"
-gem "ffi", "~> 1.15.5"
+gem 'ffi', '~> 1.15.5'
+gem 'rake', '~> 13.0'
 
 group :development do
-  gem "yard", "~> 0.9.28"
-  gem "rufo", "~> 0.13.0"
-  gem "minitest", "~> 5.20.0"
+  gem 'debug'
+  gem 'minitest', '~> 5.20.0'
+  gem 'rufo', '~> 0.13.0'
+  gem 'yard', '~> 0.9.28'
 end

ruby/bin/irb

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'irb' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__)
+
+bundle_binstub = File.expand_path("bundle", __dir__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("irb", "irb")

ruby/bin/rdbg

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rdbg' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__)
+
+bundle_binstub = File.expand_path("bundle", __dir__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("debug", "rdbg")

ruby/bin/rdoc

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rdoc' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__)
+
+bundle_binstub = File.expand_path("bundle", __dir__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rdoc", "rdoc")

ruby/bin/ri

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'ri' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__)
+
+bundle_binstub = File.expand_path("bundle", __dir__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rdoc", "ri")

ruby/lib/extism.rb

@@ -1,6 +1,6 @@
-require "ffi"
-require "json"
-require_relative "./extism/version"
+require 'ffi'
+require 'json'
+require_relative './extism/version'
 
 module Extism
   class Error < StandardError
@@ -17,16 +17,13 @@ module Extism
   # @param name [String] The path to the logfile
   # @param level [String] The log level. One of {"debug", "error", "info", "trace" }
   def self.set_log_file(name, level = nil)
-    if level
-      level = FFI::MemoryPointer::from_string(level)
-    end
     C.extism_log_file(name, level)
   end
 
   $PLUGINS = {}
   $FREE_PLUGIN = proc { |ptr|
     x = $PLUGINS[ptr]
-    if !x.nil?
+    unless x.nil?
       C.extism_plugin_free(x[:plugin])
       $PLUGINS.delete(ptr)
     end
@@ -40,7 +37,7 @@ module Extism
 
     # Cancel the plugin used to generate the handle
     def cancel
-      return C.extism_plugin_cancel(@handle)
+      C.extism_plugin_cancel(@handle)
     end
   end
 
@@ -51,26 +48,26 @@ module Extism
     # @param wasm [Hash, String] The manifest or WASM binary. See https://extism.org/docs/concepts/manifest/.
     # @param wasi [Boolean] Enable WASI support
     # @param config [Hash] The plugin config
-    def initialize(wasm, wasi = false, config = nil)
-      if wasm.class == Hash
-        wasm = JSON.generate(wasm)
-      end
+    def initialize(wasm, functions = [], wasi = false, config = nil)
+      wasm = JSON.generate(wasm) if wasm.instance_of?(Hash)
       code = FFI::MemoryPointer.new(:char, wasm.bytesize)
-      errmsg = FFI::MemoryPointer.new(:pointer) 
+      errmsg = FFI::MemoryPointer.new(:pointer)
       code.put_bytes(0, wasm)
-      @plugin = C.extism_plugin_new(code, wasm.bytesize, nil, 0, wasi, errmsg)
+      funcs_ptr = FFI::MemoryPointer.new(C::ExtismFunction)
+      funcs_ptr.write_array_of_pointer(functions.map { |f| f.pointer })
+      @plugin = C.extism_plugin_new(code, wasm.bytesize, funcs_ptr, functions.length, wasi, errmsg)
       if @plugin.null?
         err = errmsg.read_pointer.read_string
         C.extism_plugin_new_error_free errmsg.read_pointer
-        raise Error.new err
+        raise Error, err
       end
-      $PLUGINS[self.object_id] = { :plugin => @plugin }
+      $PLUGINS[object_id] = { plugin: @plugin }
       ObjectSpace.define_finalizer(self, $FREE_PLUGIN)
-      if config != nil and @plugin.null?
-        s = JSON.generate(config)
-        ptr = FFI::MemoryPointer::from_string(s)
-        C.extism_plugin_config(@plugin, ptr, s.bytesize)
-      end
+      return unless !config.nil? and @plugin.null?
+
+      s = JSON.generate(config)
+      ptr = FFI::MemoryPointer.from_string(s)
+      C.extism_plugin_config(@plugin, ptr, s.bytesize)
     end
 
     # Check if a function exists
@@ -89,16 +86,16 @@ module Extism
     def call(name, data, &block)
       # If no block was passed then use Pointer::read_string
       block ||= ->(buf, len) { buf.read_string(len) }
-      input = FFI::MemoryPointer::from_string(data)
+      input = FFI::MemoryPointer.from_string(data)
       rc = C.extism_plugin_call(@plugin, name, input, data.bytesize)
       if rc != 0
         err = C.extism_plugin_error(@plugin)
-        if err&.empty?
-          raise Error.new "extism_call failed"
-        else
-          raise Error.new err
-        end
+        raise Error, 'extism_call failed' if err&.empty?
+
+        raise Error, err
+
       end
+
       out_len = C.extism_plugin_output_length(@plugin)
       buf = C.extism_plugin_output_data(@plugin)
       block.call(buf, out_len)
@@ -110,36 +107,209 @@ module Extism
     def free
       return if @plugin.null?
 
-      $PLUGINS.delete(self.object_id)
+      $PLUGINS.delete(object_id)
       C.extism_plugin_free(@plugin)
       @plugin = nil
     end
 
     # Get a CancelHandle for a plugin
     def cancel_handle
-      return CancelHandle.new(C.extism_plugin_cancel_handle(@plugin))
+      CancelHandle.new(C.extism_plugin_cancel_handle(@plugin))
     end
   end
 
-  private
+  Memory = Struct.new(:offset, :len)
+
+  class CurrentPlugin
+    def initialize(ptr)
+      @ptr = ptr
+    end
+
+    def alloc(amount)
+      offset = C.extism_current_plugin_memory_alloc(@ptr, amount)
+      Memory.new(offset, amount)
+    end
+
+    def free(memory)
+      C.extism_current_plugin_memory_free(@ptr, memory.offset)
+    end
+
+    def memory_at_offset(offset)
+      len = C.extism_current_plugin_memory_length(@ptr, offset)
+      Memory.new(offset, len)
+    end
+
+    def input_as_bytes(input)
+      # TODO: should assert that this is an int input
+      mem = memory_at_offset(input.value)
+      memory_ptr(mem).read_bytes(mem.len)
+    end
+
+    def return_bytes(output, bytes)
+      mem = alloc(bytes.length)
+      memory_ptr(mem).put_bytes(0, bytes)
+      output.value = mem.offset
+    end
+
+    def return_string(output, string)
+      return_bytes(output, string)
+    end
+
+    private
+
+    def memory_ptr(mem)
+      plugin_ptr = C.extism_current_plugin_memory(@ptr)
+      FFI::Pointer.new(plugin_ptr.address + mem.offset)
+    end
+  end
+
+  module ValType
+    I32 = 0
+    I64 = 1
+    F32 = 2
+    F64 = 3
+    V128 = 4
+    FUNC_REF = 5
+    EXTERN_REF = 6
+  end
+
+  class Val
+    def initialize(ptr)
+      @c_val = C::ExtismVal.new(ptr)
+    end
+
+    def type
+      case @c_val[:t]
+      when :I32
+        :i32
+      when :I64
+        :i64
+      when :F32
+        :f32
+      when :F64
+        :f64
+      else
+        raise "Unsupported wasm value type #{type}"
+      end
+    end
+
+    def value
+      @c_val[:v][type]
+    end
+
+    def value=(val)
+      @c_val[:v][type] = val
+    end
+  end
+
+  class Function
+    def initialize(name, args, returns, func_proc, user_data)
+      @name = name
+      @args = args
+      @returns = returns
+      @func = func_proc
+      @user_data = user_data
+    end
+
+    def pointer
+      return @pointer if @pointer
+
+      free = proc { puts 'freeing ' }
+      args = C.from_int_array(@args)
+      returns = C.from_int_array(@returns)
+      @pointer = C.extism_function_new(@name, args, @args.length, returns, @returns.length, c_func, free, nil)
+    end
+
+    private
+
+    def c_func
+      @c_func ||= proc do |plugin_ptr, inputs_ptr, inputs_size, outputs_ptr, outputs_size, _data_ptr|
+        current_plugin = CurrentPlugin.new(plugin_ptr)
+        val_struct_size = C::ExtismVal.size
+
+        inputs = (0...inputs_size).map do |i|
+          Val.new(inputs_ptr + i * val_struct_size)
+        end
+        outputs = (0...outputs_size).map do |i|
+          Val.new(outputs_ptr + i * val_struct_size)
+        end
+
+        @func.call(current_plugin, inputs, outputs, @user_data)
+      end
+    end
+  end
 
   # Private module used to interface with the Extism runtime.
   # *Warning*: Do not use or rely on this directly.
   module C
     extend FFI::Library
-    ffi_lib "extism"
-    attach_function :extism_plugin_new_error_free, [:pointer], :void
-    attach_function :extism_plugin_new, [:pointer, :uint64, :pointer, :uint64, :bool, :pointer], :pointer
-    attach_function :extism_plugin_error, [:pointer], :string
-    attach_function :extism_plugin_call, [:pointer, :string, :pointer, :uint64], :int32
-    attach_function :extism_plugin_function_exists, [:pointer, :string], :bool
-    attach_function :extism_plugin_output_length, [:pointer], :uint64
-    attach_function :extism_plugin_output_data, [:pointer], :pointer
-    attach_function :extism_log_file, [:string, :pointer], :void
-    attach_function :extism_plugin_free, [:pointer], :void
-    attach_function :extism_version, [], :string
+    ffi_lib 'extism'
+
+    def self.from_int_array(ruby_array)
+      ptr = FFI::MemoryPointer.new(:int, ruby_array.length)
+      ptr.write_array_of_int(ruby_array)
+      ptr
+    end
+
+    typedef :uint64, :ExtismMemoryHandle
+    typedef :uint64, :ExtismSize
+
+    enum :ExtismValType, %i[I32 I64 F32 F64 V128 FuncRef ExternRef]
+
+    class ExtismValUnion < FFI::Union
+      layout :i32, :int32,
+             :i64, :int64,
+             :f32, :float,
+             :f64, :double
+    end
+
+    class ExtismVal < FFI::Struct
+      layout :t, :ExtismValType,
+             :v, ExtismValUnion
+    end
+
+    class ExtismFunction < FFI::Struct
+      layout :name, :string,
+             :inputs, :pointer,
+             :n_inputs, :uint64,
+             :outputs, :pointer,
+             :n_outputs, :uint64,
+             :data, :pointer
+    end
+
+    callback :ExtismFunctionType, [
+      :pointer, # plugin
+      :pointer, # inputs
+      :ExtismSize, # n_inputs
+      :pointer, # outputs
+      :ExtismSize, # n_outputs
+      :pointer # user_data
+    ], :void
+
+    callback :ExtismFreeFunctionType, [], :void
+
     attach_function :extism_plugin_id, [:pointer], :pointer
+    attach_function :extism_current_plugin_memory, [:pointer], :pointer
+    attach_function :extism_current_plugin_memory_alloc, %i[pointer ExtismSize], :ExtismMemoryHandle
+    attach_function :extism_current_plugin_memory_length, %i[pointer ExtismMemoryHandle], :ExtismSize
+    attach_function :extism_current_plugin_memory_free, %i[pointer ExtismMemoryHandle], :void
+    attach_function :extism_function_new,
+                    %i[string pointer ExtismSize pointer ExtismSize ExtismFunctionType ExtismFreeFunctionType pointer], :pointer
+    attach_function :extism_function_free, [:pointer], :void
+    attach_function :extism_function_set_namespace, %i[pointer string], :void
+    attach_function :extism_plugin_new, %i[pointer ExtismSize pointer ExtismSize bool pointer], :pointer
+    attach_function :extism_plugin_new_error_free, [:pointer], :void
+    attach_function :extism_plugin_free, [:pointer], :void
     attach_function :extism_plugin_cancel_handle, [:pointer], :pointer
     attach_function :extism_plugin_cancel, [:pointer], :bool
+    attach_function :extism_plugin_config, %i[pointer pointer ExtismSize], :bool
+    attach_function :extism_plugin_function_exists, %i[pointer string], :bool
+    attach_function :extism_plugin_call, %i[pointer string pointer ExtismSize], :int32
+    attach_function :extism_error, [:pointer], :string
+    attach_function :extism_plugin_error, [:pointer], :string
+    attach_function :extism_plugin_output_length, [:pointer], :ExtismSize
+    attach_function :extism_plugin_output_data, [:pointer], :pointer
+    attach_function :extism_log_file, %i[string string], :bool
+    attach_function :extism_version, [], :string
   end
 end

ruby/lib/extism/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Extism
-  VERSION = '0.5.0'
+  VERSION = '1.0.0-rc.1'
 end

ruby/test/test_extism.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "test_helper"
+require 'test_helper'
 
 class TestExtism < Minitest::Test
   def test_that_it_has_a_version_number
@@ -9,22 +9,22 @@ class TestExtism < Minitest::Test
 
   def test_plugin_call
     plugin = Extism::Plugin.new(manifest)
-    res = JSON.parse(plugin.call("count_vowels", "this is a test"))
-    assert_equal res["count"], 4
-    res = JSON.parse(plugin.call("count_vowels", "this is a test again"))
-    assert_equal res["count"], 7
-    res = JSON.parse(plugin.call("count_vowels", "this is a test thrice"))
-    assert_equal res["count"], 6
-    res = JSON.parse(plugin.call("count_vowels", "🌎hello🌎world🌎"))
-    assert_equal res["count"], 3
+    res = JSON.parse(plugin.call('count_vowels', 'this is a test'))
+    assert_equal res['count'], 4
+    res = JSON.parse(plugin.call('count_vowels', 'this is a test again'))
+    assert_equal res['count'], 7
+    res = JSON.parse(plugin.call('count_vowels', 'this is a test thrice'))
+    assert_equal res['count'], 6
+    res = JSON.parse(plugin.call('count_vowels', '🌎hello🌎world🌎'))
+    assert_equal res['count'], 3
   end
 
   def test_can_free_plugin
     plugin = Extism::Plugin.new(manifest)
-    _res = plugin.call("count_vowels", "this is a test")
+    _res = plugin.call('count_vowels', 'this is a test')
     plugin.free
     assert_raises(Extism::Error) do
-      _res = plugin.call("count_vowels", "this is a test")
+      _res = plugin.call('count_vowels', 'this is a test')
     end
   end
 
@@ -36,26 +36,48 @@ class TestExtism < Minitest::Test
 
   def test_has_function
     plugin = Extism::Plugin.new(manifest)
-    assert plugin.has_function? "count_vowels"
-    refute plugin.has_function? "i_am_not_a_function"
+    assert plugin.has_function? 'count_vowels'
+    refute plugin.has_function? 'i_am_not_a_function'
   end
 
   def test_errors_on_unknown_function
     plugin = Extism::Plugin.new(manifest)
     assert_raises(Extism::Error) do
-      plugin.call("non_existent_function", "input")
+      plugin.call('non_existent_function', 'input')
     end
   end
 
+  def test_host_functions
+    Extism.set_log_file('stdout', 'info')
+    func = proc do |current_plugin, inputs, outputs, user_data|
+      input = current_plugin.input_as_bytes(inputs.first)
+      current_plugin.return_string(outputs.first, "#{input} #{user_data}")
+    end
+    f = Extism::Function.new('transform_string', [Extism::ValType::I64], [Extism::ValType::I64], func, 'My User Data')
+    plugin = Extism::Plugin.new(host_manifest, [f], true)
+    result = plugin.call('reflect_string', 'Hello, World!')
+    assert_equal result, 'Hello, World! My User Data'
+  end
+
   private
 
   def manifest
     {
       wasm: [
         {
-          path: File.join(__dir__, "../../wasm/code.wasm"),
-        },
-      ],
+          path: File.join(__dir__, '../../wasm/code.wasm')
+        }
+      ]
+    }
+  end
+
+  def host_manifest
+    {
+      wasm: [
+        {
+          path: File.join(__dir__, '../../wasm/kitchensink.wasm')
+        }
+      ]
     }
   end
 end