feat: add ability to access response headers when using extism:host/env::http_request (#774)

- Adds `extism:host/env::http_headers` to access HTTP response headers
- Adds `PluginBuilder::with_http_response_headers` to enable response
headers from Rust
- Adds `extism_plugin_allow_http_response_headers` to enable response
headers using the C API

TODO:
- [x] Update a PDK to use `extism:host/env::http_headers` so I can test
this

runtime/extism.h

@@ -206,6 +206,11 @@ ExtismPlugin *extism_plugin_new_with_fuel_limit(const uint8_t *wasm,
                                                 uint64_t fuel_limit,
                                                 char **errmsg);
 
+/**
+ * Enable HTTP response headers in plugins using `extism:host/env::http_request`
+ */
+void extism_plugin_allow_http_response_headers(ExtismPlugin *plugin);
+
 /**
  * Free the error returned by `extism_plugin_new`, errors returned from `extism_plugin_error` don't need to be freed
  */

runtime/src/current_plugin.rs

@@ -14,6 +14,7 @@ pub struct CurrentPlugin {
     pub(crate) linker: *mut wasmtime::Linker<CurrentPlugin>,
     pub(crate) wasi: Option<Wasi>,
     pub(crate) http_status: u16,
+    pub(crate) http_headers: Option<std::collections::BTreeMap<String, String>>,
     pub(crate) available_pages: Option<u32>,
     pub(crate) memory_limiter: Option<MemoryLimiter>,
     pub(crate) id: uuid::Uuid,
@@ -332,6 +333,7 @@ impl CurrentPlugin {
         manifest: extism_manifest::Manifest,
         wasi: bool,
         available_pages: Option<u32>,
+        allow_http_response_headers: bool,
         id: uuid::Uuid,
     ) -> Result<Self, Error> {
         let wasi = if wasi {
@@ -394,6 +396,11 @@ impl CurrentPlugin {
             memory_limiter,
             id,
             start_time: std::time::Instant::now(),
+            http_headers: if allow_http_response_headers {
+                Some(BTreeMap::new())
+            } else {
+                None
+            },
         })
     }
 

runtime/src/pdk.rs

@@ -194,6 +194,9 @@ pub(crate) fn http_request(
 
     #[cfg(feature = "http")]
     {
+        data.http_headers.iter_mut().for_each(|x| x.clear());
+        data.http_status = 0;
+
         use std::io::Read;
         let handle = match data.memory_handle(http_req_offset) {
             Some(h) => h,
@@ -260,6 +263,13 @@ pub(crate) fn http_request(
 
         let reader = match res {
             Ok(res) => {
+                if let Some(headers) = &mut data.http_headers {
+                    for name in res.headers_names() {
+                        if let Some(h) = res.header(&name) {
+                            headers.insert(name, h.to_string());
+                        }
+                    }
+                }
                 data.http_status = res.status();
                 Some(res.into_reader())
             }
@@ -317,6 +327,24 @@ pub(crate) fn http_status_code(
     Ok(())
 }
 
+/// Get the HTTP response headers from the last HTTP request
+/// Params: none
+/// Returns: i64 (offset)
+pub(crate) fn http_headers(
+    mut caller: Caller<CurrentPlugin>,
+    _input: &[Val],
+    output: &mut [Val],
+) -> Result<(), Error> {
+    let data: &mut CurrentPlugin = caller.data_mut();
+    if let Some(h) = &data.http_headers {
+        let headers = serde_json::to_string(h)?;
+        data.memory_set_val(&mut output[0], headers)?;
+    } else {
+        output[0] = Val::I64(0);
+    }
+    Ok(())
+}
+
 pub fn log(
     level: tracing::Level,
     mut caller: Caller<CurrentPlugin>,

runtime/src/plugin.rs

@@ -1,7 +1,6 @@
 use std::{
     any::Any,
     collections::{BTreeMap, BTreeSet},
-    path::PathBuf,
 };
 
 use anyhow::Context;
@@ -256,6 +255,7 @@ fn relink(
         var_set(I64, I64);
         http_request(I64, I64) -> I64;
         http_status_code() -> I32;
+        http_headers() -> I64;
         log_warn(I64);
         log_info(I64);
         log_debug(I64);
@@ -311,42 +311,30 @@ impl Plugin {
         with_wasi: bool,
     ) -> Result<Plugin, Error> {
         Self::build_new(
-            wasm.into(),
-            imports,
-            with_wasi,
-            Default::default(),
-            None,
-            None,
-            None,
+            PluginBuilder::new(wasm)
+                .with_functions(imports)
+                .with_wasi(with_wasi),
         )
     }
 
-    pub(crate) fn build_new(
-        wasm: WasmInput<'_>,
-        imports: impl IntoIterator<Item = Function>,
-        with_wasi: bool,
-        debug_options: DebugOptions,
-        cache_dir: Option<Option<PathBuf>>,
-        fuel: Option<u64>,
-        config: Option<Config>,
-    ) -> Result<Plugin, Error> {
+    pub(crate) fn build_new(builder: PluginBuilder) -> Result<Plugin, Error> {
         // Setup wasmtime types
-        let mut config = config.unwrap_or_default();
+        let mut config = builder.config.unwrap_or_default();
         config
             .async_support(false)
             .epoch_interruption(true)
-            .debug_info(debug_options.debug_info)
-            .coredump_on_trap(debug_options.coredump.is_some())
-            .profiler(debug_options.profiling_strategy)
+            .debug_info(builder.debug_options.debug_info)
+            .coredump_on_trap(builder.debug_options.coredump.is_some())
+            .profiler(builder.debug_options.profiling_strategy)
             .wasm_tail_call(true)
             .wasm_function_references(true)
             .wasm_gc(true);
 
-        if fuel.is_some() {
+        if builder.fuel.is_some() {
             config.consume_fuel(true);
         }
 
-        match cache_dir {
+        match builder.cache_config {
             Some(None) => (),
             Some(Some(path)) => {
                 config.cache_config_load(path)?;
@@ -363,7 +351,7 @@ impl Plugin {
         }
 
         let engine = Engine::new(&config)?;
-        let (manifest, modules) = manifest::load(&engine, wasm)?;
+        let (manifest, modules) = manifest::load(&engine, builder.source)?;
         if modules.len() <= 1 {
             anyhow::bail!("No wasm modules provided");
         } else if !modules.contains_key(MAIN_KEY) {
@@ -376,16 +364,22 @@ impl Plugin {
         let id = uuid::Uuid::new_v4();
         let mut store = Store::new(
             &engine,
-            CurrentPlugin::new(manifest, with_wasi, available_pages, id)?,
+            CurrentPlugin::new(
+                manifest,
+                builder.wasi,
+                available_pages,
+                builder.http_response_headers,
+                id,
+            )?,
         );
         store.set_epoch_deadline(1);
-        if let Some(fuel) = fuel {
+        if let Some(fuel) = builder.fuel {
             store.set_fuel(fuel)?;
         }
 
-        let imports: Vec<Function> = imports.into_iter().collect();
+        let imports: Vec<Function> = builder.functions.into_iter().collect();
         let (instance_pre, linker, host_context) =
-            relink(&engine, &mut store, &imports, &modules, with_wasi)?;
+            relink(&engine, &mut store, &imports, &modules, builder.wasi)?;
         let timer_tx = Timer::tx();
         let mut plugin = Plugin {
             modules,
@@ -400,10 +394,10 @@ impl Plugin {
             instantiations: 0,
             output: Output::default(),
             store_needs_reset: false,
-            debug_options,
+            debug_options: builder.debug_options,
             _functions: imports,
             error_msg: None,
-            fuel,
+            fuel: builder.fuel,
             host_context,
         };
 
@@ -433,6 +427,7 @@ impl Plugin {
                     internal.manifest.clone(),
                     internal.wasi.is_some(),
                     internal.available_pages,
+                    internal.http_headers.is_some(),
                     self.id,
                 )?,
             );

runtime/src/plugin_builder.rs

@@ -34,13 +34,14 @@ impl Default for DebugOptions {
 
 /// PluginBuilder is used to configure and create `Plugin` instances
 pub struct PluginBuilder<'a> {
-    source: WasmInput<'a>,
-    wasi: bool,
-    functions: Vec<Function>,
-    debug_options: DebugOptions,
-    cache_config: Option<Option<PathBuf>>,
-    fuel: Option<u64>,
-    config: Option<wasmtime::Config>,
+    pub(crate) source: WasmInput<'a>,
+    pub(crate) wasi: bool,
+    pub(crate) functions: Vec<Function>,
+    pub(crate) debug_options: DebugOptions,
+    pub(crate) cache_config: Option<Option<PathBuf>>,
+    pub(crate) fuel: Option<u64>,
+    pub(crate) config: Option<wasmtime::Config>,
+    pub(crate) http_response_headers: bool,
 }
 
 impl<'a> PluginBuilder<'a> {
@@ -54,6 +55,7 @@ impl<'a> PluginBuilder<'a> {
             cache_config: None,
             fuel: None,
             config: None,
+            http_response_headers: false,
         }
     }
 
@@ -176,16 +178,14 @@ impl<'a> PluginBuilder<'a> {
         self
     }
 
+    /// Enables `http_response_headers`, which allows for plugins to access response headers when using `extism:host/env::http_request`
+    pub fn with_http_response_headers(mut self, allow: bool) -> Self {
+        self.http_response_headers = allow;
+        self
+    }
+
     /// Generate a new plugin with the configured settings
     pub fn build(self) -> Result<Plugin, Error> {
-        Plugin::build_new(
-            self.source,
-            self.functions,
-            self.wasi,
-            self.debug_options,
-            self.cache_config,
-            self.fuel,
-            self.config,
-        )
+        Plugin::build_new(self)
     }
 }

runtime/src/sdk.rs

@@ -395,13 +395,10 @@ pub unsafe extern "C" fn extism_plugin_new_with_fuel_limit(
     }
 
     let plugin = Plugin::build_new(
-        data.into(),
-        funcs,
-        with_wasi,
-        Default::default(),
-        None,
-        Some(fuel_limit),
-        None,
+        PluginBuilder::new(data)
+            .with_functions(funcs)
+            .with_wasi(with_wasi)
+            .with_fuel_limit(fuel_limit),
     );
 
     match plugin {
@@ -417,6 +414,13 @@ pub unsafe extern "C" fn extism_plugin_new_with_fuel_limit(
     }
 }
 
+/// Enable HTTP response headers in plugins using `extism:host/env::http_request`
+#[no_mangle]
+pub unsafe extern "C" fn extism_plugin_allow_http_response_headers(plugin: *mut Plugin) {
+    let plugin = &mut *plugin;
+    plugin.store.data_mut().http_headers = Some(BTreeMap::new());
+}
+
 /// Free the error returned by `extism_plugin_new`, errors returned from `extism_plugin_error` don't need to be freed
 #[no_mangle]
 pub unsafe extern "C" fn extism_plugin_new_error_free(err: *mut std::ffi::c_char) {

runtime/src/tests/runtime.rs

@@ -1,7 +1,7 @@
-use extism_manifest::MemoryOptions;
+use extism_manifest::{HttpRequest, MemoryOptions};
 
 use crate::*;
-use std::{io::Write, time::Instant};
+use std::{collections::HashMap, io::Write, time::Instant};
 
 const WASM: &[u8] = include_bytes!("../../../wasm/code-functions.wasm");
 const WASM_NO_FUNCTIONS: &[u8] = include_bytes!("../../../wasm/code.wasm");
@@ -9,6 +9,7 @@ const WASM_LOOP: &[u8] = include_bytes!("../../../wasm/loop.wasm");
 const WASM_GLOBALS: &[u8] = include_bytes!("../../../wasm/globals.wasm");
 const WASM_REFLECT: &[u8] = include_bytes!("../../../wasm/reflect.wasm");
 const WASM_HTTP: &[u8] = include_bytes!("../../../wasm/http.wasm");
+const WASM_HTTP_HEADERS: &[u8] = include_bytes!("../../../wasm/http_headers.wasm");
 const WASM_FS: &[u8] = include_bytes!("../../../wasm/read_write.wasm");
 
 host_fn!(pub hello_world (a: String) -> String { Ok(a) });
@@ -793,3 +794,33 @@ fn test_readonly_dirs() {
         "Expected try_write to fail, but it succeeded."
     );
 }
+
+#[test]
+#[cfg(feature = "http")]
+fn test_http_response_headers() {
+    let mut plugin = PluginBuilder::new(
+        Manifest::new([Wasm::data(WASM_HTTP_HEADERS)]).with_allowed_host("extism.org"),
+    )
+    .with_http_response_headers(true)
+    .build()
+    .unwrap();
+    let req = HttpRequest::new("https://extism.org");
+    let Json(res): Json<HashMap<String, String>> = plugin.call("http_get", Json(req)).unwrap();
+    println!("{:?}", res);
+    assert_eq!(res["content-type"], "text/html; charset=utf-8");
+}
+
+#[test]
+#[cfg(feature = "http")]
+fn test_http_response_headers_disabled() {
+    let mut plugin = PluginBuilder::new(
+        Manifest::new([Wasm::data(WASM_HTTP_HEADERS)]).with_allowed_host("extism.org"),
+    )
+    .with_http_response_headers(false)
+    .build()
+    .unwrap();
+    let req = HttpRequest::new("https://extism.org");
+    let Json(res): Json<HashMap<String, String>> = plugin.call("http_get", Json(req)).unwrap();
+    println!("{:?}", res);
+    assert!(res.is_empty());
+}