diff --git a/storage/src/lib.rs b/storage/src/lib.rs index bc7abb2..b455e65 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -16,8 +16,9 @@ mod tests; pub use self::storage::Storage; pub use self::storage_backend::StorageBackend; pub use memory_backend::MemoryBackend; +pub use rc_key::RcKey; pub use sled_backend::SledBackend; pub use storage_entity::StorageEntity; -pub use storage_entry::{StorageEntry, StorageEntryReader}; +pub use storage_entry::{StorageEntry, StorageEntryReader, StorageEntryWriter}; pub use storage_ops::StorageOps; pub use storage_ptr::{storage_head_ptr, StorageEntryPtr, StorageHeadPtr, StoragePtr}; diff --git a/storage/src/storage_entry.rs b/storage/src/storage_entry.rs index 15a47ad..e697a22 100644 --- a/storage/src/storage_entry.rs +++ b/storage/src/storage_entry.rs @@ -6,7 +6,7 @@ use crate::storage_ptr::StorageEntryPtr; #[derive(Serialize, Deserialize)] pub struct StorageEntry { - pub(crate) ref_count: u64, + pub ref_count: u64, pub refs: Vec, pub data: Vec, } @@ -26,6 +26,10 @@ impl<'a> StorageEntryReader<'a> { } } + pub fn done(&self) -> bool { + self.refs_i == self.entry.refs.len() && self.data_i == self.entry.data.len() + } + pub fn read_ref(&mut self) -> std::io::Result { if self.refs_i >= self.entry.refs.len() { return Err(eof()); @@ -98,3 +102,39 @@ impl Read for StorageEntryReader<'_> { fn eof() -> std::io::Error { std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "not enough bytes") } + +pub struct StorageEntryWriter<'a> { + pub entry: &'a mut StorageEntry, +} + +impl<'a> StorageEntryWriter<'a> { + pub fn new(entry: &'a mut StorageEntry) -> Self { + Self { entry } + } + + pub fn write_u8(&mut self, byte: u8) { + self.entry.data.push(byte); + } + + pub fn write_bytes(&mut self, bytes: &[u8]) { + self.entry.data.extend_from_slice(bytes); + } + + pub fn write_vlq(&mut self, mut num: u64) { + loop { + let mut byte = (num & 0x7f) as u8; + + if num != 0 { + byte |= 0x80; + } + + self.write_u8(byte); + + if num == 0 { + break; + } + + num >>= 7; + } + } +} diff --git a/valuescript_vm/src/bytecode.rs b/valuescript_vm/src/bytecode.rs index d6e12d2..96363aa 100644 --- a/valuescript_vm/src/bytecode.rs +++ b/valuescript_vm/src/bytecode.rs @@ -1,5 +1,7 @@ use std::{cell::RefCell, collections::HashMap, fmt, ops::Index, rc::Rc, slice::SliceIndex}; +use storage::StorageEntity; + use crate::{bytecode_decoder::BytecodeDecoder, vs_value::Val}; pub struct Bytecode { @@ -42,3 +44,23 @@ impl DecoderMaker for Rc { } } } + +impl StorageEntity for Bytecode { + fn to_storage_entry>( + &self, + _tx: &mut Tx, + ) -> Result { + Ok(storage::StorageEntry { + ref_count: 1, + refs: vec![], + data: self.code.clone(), + }) + } + + fn from_storage_entry>( + _tx: &mut Tx, + entry: storage::StorageEntry, + ) -> Result { + Ok(Bytecode::new(entry.data)) + } +} diff --git a/valuescript_vm/src/val_storage.rs b/valuescript_vm/src/val_storage.rs index 33b8cb6..da9858b 100644 --- a/valuescript_vm/src/val_storage.rs +++ b/valuescript_vm/src/val_storage.rs @@ -3,7 +3,9 @@ use std::{collections::BTreeMap, io::Read, rc::Rc}; use num_bigint::BigInt; use num_derive::{FromPrimitive, ToPrimitive}; use num_traits::{FromPrimitive, ToPrimitive}; -use storage::{StorageEntity, StorageEntry, StorageEntryReader, StorageOps}; +use storage::{ + RcKey, StorageEntity, StorageEntry, StorageEntryReader, StorageEntryWriter, StorageOps, +}; use crate::{ vs_array::VsArray, @@ -47,87 +49,221 @@ impl Tag { impl StorageEntity for Val { fn from_storage_entry>(tx: &mut Tx, entry: StorageEntry) -> Result { - read_from_entry(tx, &mut StorageEntryReader::new(&entry)) - // TODO: assert that we've read the whole entry + let mut reader = StorageEntryReader::new(&entry); + let res = read_from_entry(tx, &mut reader); + assert!(reader.done()); + + res } fn to_storage_entry>(&self, tx: &mut Tx) -> Result { - todo!() + let mut entry = StorageEntry { + ref_count: 1, + refs: vec![], + data: vec![], + }; + + let writer = &mut StorageEntryWriter::new(&mut entry); + + match self { + Val::Array(a) => { + writer.write_u8(Tag::Array.to_byte()); + writer.write_vlq(a.elements.len() as u64); + + for item in a.elements.iter() { + write_to_entry(item, tx, writer)?; + } + } + Val::Object(obj) => { + writer.write_u8(Tag::Object.to_byte()); + + writer.write_vlq(obj.string_map.len() as u64); + + for (key, value) in obj.string_map.iter() { + let key_bytes = key.as_bytes(); + writer.write_vlq(key_bytes.len() as u64); + writer.write_bytes(key_bytes); + write_to_entry(value, tx, writer)?; + } + + writer.write_vlq(obj.symbol_map.len() as u64); + + for (key, value) in obj.symbol_map.iter() { + writer.write_vlq(key.to_u64().unwrap()); + write_to_entry(value, tx, writer)?; + } + + match &obj.prototype { + None => writer.write_u8(0), + Some(val) => { + writer.write_u8(1); + write_to_entry(val, tx, writer)? + } + }; + } + Val::Function(f) => { + let VsFunction { + bytecode, + meta_pos, + is_generator, + register_count, + parameter_count, + start, + binds, + } = f.as_ref(); + + writer.write_u8(Tag::Function.to_byte()); + + write_ref_bytecode_to_entry(tx, writer, bytecode)?; + + match *meta_pos { + None => writer.write_u8(0), + Some(pos) => { + writer.write_u8(1); + writer.write_vlq(pos as u64); + } + }; + + writer.write_u8(if *is_generator { 1 } else { 0 }); + writer.write_vlq(*register_count as u64); + writer.write_vlq(*parameter_count as u64); + writer.write_vlq(*start as u64); + writer.write_vlq(binds.len() as u64); + + for bind in binds.iter() { + write_to_entry(bind, tx, writer)?; + } + } + + Val::Void + | Val::Undefined + | Val::Null + | Val::Bool(_) + | Val::Number(_) + | Val::BigInt(_) + | Val::Symbol(_) + | Val::String(_) + | Val::Class(_) + | Val::Static(_) + | Val::Dynamic(_) + | Val::CopyCounter(_) + | Val::StoragePtr(_) => { + write_to_entry(self, tx, writer)?; + } + }; + + Ok(entry) } } fn write_to_entry>( val: &Val, tx: &mut SO, - entry: &mut StorageEntry, + writer: &mut StorageEntryWriter, ) -> Result<(), E> { match val { Val::Void => { - entry.data.push(Tag::Void.to_byte()); + writer.write_u8(Tag::Void.to_byte()); } Val::Undefined => { - entry.data.push(Tag::Undefined.to_byte()); + writer.write_u8(Tag::Undefined.to_byte()); } Val::Null => { - entry.data.push(Tag::Null.to_byte()); + writer.write_u8(Tag::Null.to_byte()); } Val::Bool(b) => { - entry.data.push(Tag::Bool.to_byte()); - entry.data.push(if *b { 1 } else { 0 }); + writer.write_u8(Tag::Bool.to_byte()); + writer.write_u8(if *b { 1 } else { 0 }); } Val::Number(n) => { - entry.data.push(Tag::Number.to_byte()); - entry.data.extend_from_slice(&n.to_le_bytes()); + writer.write_u8(Tag::Number.to_byte()); + writer.write_bytes(&n.to_le_bytes()); } Val::BigInt(b) => { - entry.data.push(Tag::BigInt.to_byte()); - todo!() + writer.write_u8(Tag::BigInt.to_byte()); + let bytes = b.to_signed_bytes_le(); + writer.write_vlq(bytes.len() as u64); + writer.write_bytes(&bytes); } Val::Symbol(s) => { - entry.data.push(Tag::Symbol.to_byte()); - todo!() + writer.write_u8(Tag::Symbol.to_byte()); + writer.write_vlq(s.to_u64().unwrap()); } Val::String(s) => { - entry.data.push(Tag::String.to_byte()); - todo!() - } - Val::Array(a) => { - entry.data.push(Tag::Array.to_byte()); - todo!() - } - Val::Object(o) => { - entry.data.push(Tag::Object.to_byte()); - todo!() - } - Val::Function(f) => { - entry.data.push(Tag::Function.to_byte()); - todo!() + writer.write_u8(Tag::String.to_byte()); + let bytes = s.as_bytes(); + writer.write_vlq(bytes.len() as u64); + writer.write_bytes(bytes); } + Val::Array(a) => write_ptr_to_entry(tx, writer, RcKey::from(a.clone()), val)?, + Val::Object(obj) => write_ptr_to_entry(tx, writer, RcKey::from(obj.clone()), val)?, + Val::Function(f) => write_ptr_to_entry(tx, writer, RcKey::from(f.clone()), val)?, Val::Class(c) => { - entry.data.push(Tag::Class.to_byte()); + writer.write_u8(Tag::Class.to_byte()); + + let VsClass { + name, + content_hash, + constructor, + prototype, + static_, + } = c.as_ref(); + + let name_bytes = name.as_bytes(); + writer.write_vlq(name_bytes.len() as u64); + writer.write_bytes(name_bytes); + + match *content_hash { + None => writer.write_u8(0), + Some(hash) => { + writer.write_u8(1); + writer.write_bytes(&hash); + } + }; + + write_to_entry(constructor, tx, writer)?; + write_to_entry(prototype, tx, writer)?; + write_to_entry(static_, tx, writer)?; + } + Val::Static(_s) => { + writer.write_u8(Tag::Static.to_byte()); todo!() } - Val::Static(s) => { - entry.data.push(Tag::Static.to_byte()); + Val::Dynamic(_d) => { + writer.write_u8(Tag::Dynamic.to_byte()); todo!() } - Val::Dynamic(d) => { - entry.data.push(Tag::Dynamic.to_byte()); - todo!() - } - Val::CopyCounter(cc) => { - entry.data.push(Tag::CopyCounter.to_byte()); + Val::CopyCounter(_cc) => { + writer.write_u8(Tag::CopyCounter.to_byte()); todo!() } Val::StoragePtr(ptr) => { - entry.data.push(Tag::CopyCounter.to_byte()); - todo!() + writer.write_u8(Tag::StoragePtr.to_byte()); + writer.entry.refs.push(ptr.ptr); } }; Ok(()) } +fn write_ptr_to_entry>( + tx: &mut SO, + writer: &mut StorageEntryWriter, + key: RcKey, + val: &Val, +) -> Result<(), E> { + if let Some(ptr) = tx.cache_get(key.clone()) { + writer.write_u8(Tag::StoragePtr.to_byte()); + writer.entry.refs.push(ptr); + } else { + let ptr = tx.store_and_cache(val, key)?; + writer.write_u8(Tag::StoragePtr.to_byte()); + writer.entry.refs.push(ptr); + } + + Ok(()) +} + fn read_from_entry>( tx: &mut SO, reader: &mut StorageEntryReader, @@ -205,33 +341,24 @@ fn read_from_entry>( .to_val() } Tag::Function => { - // pub bytecode: Rc, let bytecode = read_ref_bytecode_from_entry(tx, reader)?; - // pub meta_pos: Option, let meta_pos = match reader.read_u8().unwrap() { 0 => None, 1 => Some(reader.read_vlq().unwrap() as usize), _ => panic!("Invalid meta_pos byte"), }; - // pub is_generator: bool, let is_generator = match reader.read_u8().unwrap() { 0 => false, 1 => true, _ => panic!("Invalid is_generator byte"), }; - // pub register_count: usize, let register_count = reader.read_vlq().unwrap() as usize; - - // pub parameter_count: usize, let parameter_count = reader.read_vlq().unwrap() as usize; - - // pub start: usize, let start = reader.read_vlq().unwrap() as usize; - // pub binds: Vec, let len = reader.read_vlq().unwrap(); let mut binds = Vec::new(); @@ -251,10 +378,8 @@ fn read_from_entry>( .to_val() } Tag::Class => { - // pub name: String, let name = read_string_from_entry(reader); - // pub content_hash: Option<[u8; 32]>, let content_hash = match reader.read_u8().unwrap() { 0 => None, 1 => { @@ -266,13 +391,8 @@ fn read_from_entry>( _ => panic!("Invalid content_hash byte"), }; - // pub constructor: Val, let constructor = read_from_entry(tx, reader)?; - - // pub prototype: Val, let prototype = read_from_entry(tx, reader)?; - - // pub static_: Val, let static_ = read_from_entry(tx, reader)?; VsClass { @@ -310,5 +430,22 @@ fn read_ref_bytecode_from_entry>( let entry = tx.read(ptr)?.unwrap(); // TODO: Cached reads - Ok(Rc::new(Bytecode::new(entry.data))) + Ok(Rc::new(Bytecode::from_storage_entry(tx, entry)?)) +} + +fn write_ref_bytecode_to_entry>( + tx: &mut SO, + writer: &mut StorageEntryWriter, + bytecode: &Rc, +) -> Result<(), E> { + let key = RcKey::from(bytecode.clone()); + + if let Some(ptr) = tx.cache_get(key.clone()) { + writer.entry.refs.push(ptr); + } else { + let ptr = tx.store_and_cache(bytecode.as_ref(), key)?; + writer.entry.refs.push(ptr); + } + + Ok(()) } diff --git a/valuescript_vm/src/vs_storage_ptr.rs b/valuescript_vm/src/vs_storage_ptr.rs index 6cc5a4b..c5c8848 100644 --- a/valuescript_vm/src/vs_storage_ptr.rs +++ b/valuescript_vm/src/vs_storage_ptr.rs @@ -6,7 +6,7 @@ use crate::vs_value::{ToVal, Val}; #[derive(Debug)] pub struct VsStoragePtr { - ptr: StorageEntryPtr, + pub ptr: StorageEntryPtr, cache: RefCell>, }