fix(kernel): fix potential overflow in bounds check when lots of memory has been allocated (#663)

- Fixes potential overflow in bounds checking function
- Found by running the `check_large_allocations` quickcheck test in a
loop

kernel/src/lib.rs

@@ -198,8 +198,8 @@ impl MemoryRoot {
     fn pointer_in_bounds_fast(p: Pointer) -> bool {
         // Similar to `pointer_in_bounds` but less accurate on the upper bound. This uses the total memory size,
         // instead of checking `MemoryRoot::length`
-        let end = core::arch::wasm32::memory_size(0) << 16;
-        p >= core::mem::size_of::<Self>() as Pointer && p <= end as Pointer
+        let end = (core::arch::wasm32::memory_size(0) as u64) << 16;
+        p >= core::mem::size_of::<Self>() as Pointer && p <= end as u64
     }
 
     // Find a block that is free to use, this can be a new block or an existing freed block. The `self_position` argument
@@ -619,4 +619,23 @@ mod test {
             assert_eq!(length(3788), 4);
         }
     }
+
+    #[wasm_bindgen_test]
+    fn test_oom() {
+        let size = 1024 * 1024 * 5;
+
+        let mut last = 0;
+        for _ in 0..1024 {
+            unsafe {
+                let ptr = alloc(size);
+                last = ptr;
+                if ptr == 0 {
+                    break;
+                }
+                assert_eq!(length(ptr), size);
+            }
+        }
+
+        assert_eq!(last, 0);
+    }
 }

runtime/src/tests/kernel.rs

@@ -341,6 +341,22 @@ fn test_failed_quickcheck1() {
     }
 }
 
+#[test]
+fn test_failed_quickcheck2() {
+    let (mut store, mut instance) = init_kernel_test();
+    let allocs = [352054710, 1248853976, 2678441931, 14567928];
+
+    extism_reset(&mut store, &mut instance);
+    for a in allocs {
+        println!("Alloc: {a}");
+        let n = extism_alloc(&mut store, &mut instance, a);
+        if n == 0 {
+            continue;
+        }
+        assert_eq!(a, extism_length(&mut store, &mut instance, n));
+    }
+}
+
 quickcheck! {
     fn check_alloc(amounts: Vec<u16>) -> bool {
         let (mut store, mut instance) = init_kernel_test();
@@ -365,10 +381,11 @@ quickcheck! {
         let instance = &mut instance;
         for a in amounts {
             let ptr = extism_alloc(&mut store, instance, a as u64);
-            if ptr == 0 || ptr == u64::MAX {
+            if ptr == 0 {
                 continue
             }
-            if extism_length(&mut store, instance, ptr) != a as u64 {
+            let len = extism_length_unsafe(&mut store, instance, ptr);
+            if len != a as u64 {
                 return false
             }
         }