Add InputItem::SerializedObject (#80)

This commit is contained in:
Han
2025-08-11 14:44:22 +08:00
committed by GitHub
parent f281ead60a
commit 0c8d4c381c
18 changed files with 174 additions and 215 deletions

2
Cargo.lock generated
View File

@@ -2327,6 +2327,7 @@ dependencies = [
"bincode",
"build-utils",
"bytemuck",
"ere-cli",
"risc0-zkvm",
"serde",
"tempfile",
@@ -2387,6 +2388,7 @@ dependencies = [
"bincode",
"build-utils",
"pico-sdk",
"pico-vm",
"thiserror 2.0.12",
"zkvm-interface",
]

View File

@@ -62,6 +62,7 @@ openvm-stark-sdk = { git = "https://github.com/openvm-org/stark-backend.git", ta
openvm-transpiler = { git = "https://github.com/openvm-org/openvm.git", tag = "v1.2.0", default-features = false }
# Pico dependencies
pico-vm = { git = "https://github.com/brevis-network/pico.git", tag = "v1.1.4" }
pico-sdk = { git = "https://github.com/brevis-network/pico.git", tag = "v1.1.4" }
# Risc0 dependencies
@@ -74,7 +75,7 @@ sp1-sdk = "5.1.0"
# Local dependencies
zkvm-interface = { path = "crates/zkvm-interface" }
build-utils = { path = "crates/build-utils" }
ere-cli = { path = "crates/ere-cli" }
ere-cli = { path = "crates/ere-cli", default-features = false }
ere-dockerized = { path = "crates/ere-dockerized" }
ere-jolt = { path = "crates/ere-jolt" }
ere-nexus = { path = "crates/ere-nexus" }

View File

@@ -8,9 +8,9 @@ license.workspace = true
[dependencies]
anyhow.workspace = true
bincode.workspace = true
clap.workspace = true
clap = { workspace = true, optional = true }
serde = { workspace = true, features = ["derive"] }
tracing-subscriber = { workspace = true, features = ["env-filter"] }
tracing-subscriber = { workspace = true, features = ["env-filter"], optional = true }
# Local dependencies
ere-jolt = { workspace = true, optional = true }
@@ -29,6 +29,7 @@ workspace = true
[features]
default = []
cli = ["dep:clap", "dep:tracing-subscriber"]
jolt = ["dep:ere-jolt"]
nexus = ["dep:ere-nexus"]
openvm = ["dep:ere-openvm"]

View File

@@ -0,0 +1,3 @@
#![cfg_attr(not(test), warn(unused_crate_dependencies))]
pub mod serde;

View File

@@ -1,26 +1,25 @@
#![cfg_attr(not(test), warn(unused_crate_dependencies))]
use anyhow::{Context, Error};
use clap::{Parser, Subcommand};
use ere_cli::serde;
use std::{fs, path::PathBuf};
use tracing_subscriber::EnvFilter;
use zkvm_interface::{Compiler, ProverResourceType, zkVM};
mod serde;
// Compile-time check to ensure exactly one backend feature is enabled
const _: () = {
assert!(
(cfg!(feature = "jolt") as u8
+ cfg!(feature = "nexus") as u8
+ cfg!(feature = "openvm") as u8
+ cfg!(feature = "pico") as u8
+ cfg!(feature = "risc0") as u8
+ cfg!(feature = "sp1") as u8
+ cfg!(feature = "zisk") as u8)
== 1,
"Exactly one zkVM backend feature must be enabled"
);
if cfg!(feature = "cli") {
assert!(
(cfg!(feature = "jolt") as u8
+ cfg!(feature = "nexus") as u8
+ cfg!(feature = "openvm") as u8
+ cfg!(feature = "pico") as u8
+ cfg!(feature = "risc0") as u8
+ cfg!(feature = "sp1") as u8
+ cfg!(feature = "zisk") as u8)
== 1,
"Exactly one zkVM backend feature must be enabled"
);
}
};
#[derive(Parser)]

View File

@@ -1,15 +1,30 @@
use anyhow::{Context, Error};
use serde::{Serialize, de::DeserializeOwned};
use serde::{Deserialize, Serialize, de::DeserializeOwned};
use std::{fs, path::Path};
use zkvm_interface::{Input, InputItem};
#[derive(Serialize, Deserialize)]
pub enum SerializableInputItem {
SerializedObject(Vec<u8>),
Bytes(Vec<u8>),
}
impl From<SerializableInputItem> for InputItem {
fn from(value: SerializableInputItem) -> Self {
match value {
SerializableInputItem::SerializedObject(bytes) => Self::SerializedObject(bytes),
SerializableInputItem::Bytes(bytes) => Self::Bytes(bytes),
}
}
}
/// Read `Input` from `input_path`.
///
/// `Input` is assumed to be serialized into sequence of bytes, and each bytes
/// in the sequence is serialized in the specific way the zkvm does.
pub fn read_input(input_path: &Path) -> Result<Input, Error> {
read::<Vec<Vec<u8>>>(input_path, "input")
.map(|seq| Input::from(Vec::from_iter(seq.into_iter().map(InputItem::Bytes))))
read::<Vec<SerializableInputItem>>(input_path, "input")
.map(|seq| Input::from(Vec::from_iter(seq.into_iter().map(Into::into))))
}
/// Serialize `value` with [`bincode`] and write to `path`.

View File

@@ -17,6 +17,7 @@ risc0-zkvm.workspace = true
# Local dependencies
zkvm-interface = { workspace = true, features = ["clap"] }
ere-cli.workspace = true
[dev-dependencies]

View File

@@ -1,4 +1,5 @@
use crate::{ErezkVM, error::CommonError};
use ere_cli::serde::SerializableInputItem;
use serde::Serialize;
use zkvm_interface::{Input, InputItem};
@@ -52,14 +53,22 @@ impl ErezkVM {
.iter()
.map(|input| {
Ok(match input {
InputItem::Object(obj) => self.serialize_object(&**obj)?,
InputItem::Bytes(bytes) => bytes.clone(),
InputItem::Object(obj) => {
SerializableInputItem::SerializedObject(self.serialize_object(&**obj)?)
}
InputItem::SerializedObject(bytes) => {
SerializableInputItem::SerializedObject(bytes.clone())
}
InputItem::Bytes(bytes) => SerializableInputItem::Bytes(bytes.clone()),
})
})
.collect::<Result<Vec<Vec<u8>>, CommonError>>()?,
.collect::<Result<Vec<SerializableInputItem>, CommonError>>()?,
)
.map_err(|err| {
CommonError::serilization(err, "Failed to serialize sequence of bytes with `bincode`")
CommonError::serilization(
err,
"Failed to serialize `Vec<SerializableInputItem>` with `bincode`",
)
})
}
}

View File

@@ -66,52 +66,32 @@ impl EreNexus {
}
}
impl zkVM for EreNexus {
fn execute(&self, inputs: &Input) -> Result<zkvm_interface::ProgramExecutionReport, zkVMError> {
let start = Instant::now();
fn execute(
&self,
_inputs: &Input,
) -> Result<zkvm_interface::ProgramExecutionReport, zkVMError> {
// TODO: Serialize inputs by `postcard` and make sure there is no double serailization.
// Issue for tracking: https://github.com/eth-act/ere/issues/63.
// let mut public_input = vec![];
let mut private_input = vec![];
for input in inputs.iter() {
private_input.extend(
input
.as_bytes()
.map_err(|err| NexusError::Prove(ProveError::Client(err)))
.map_err(zkVMError::from)?,
);
}
// TODO: Doesn't catch execute for guest in nexus. so only left some dummy code(parse input) here.
// Besides, public input is not supported yet, so we just pass an empty tuple
// TODO: Execute and get cycle count
Ok(ProgramExecutionReport {
execution_duration: start.elapsed(),
..Default::default()
})
Ok(ProgramExecutionReport::default())
}
fn prove(
&self,
inputs: &Input,
_inputs: &Input,
) -> Result<(Vec<u8>, zkvm_interface::ProgramProvingReport), zkVMError> {
let prover: Stwo<Local> = Stwo::new_from_file(&self.program.to_string_lossy().to_string())
.map_err(|e| NexusError::Prove(ProveError::Client(e.into())))
.map_err(zkVMError::from)?;
// One convention that may be useful for simplifying the design is that all inputs to the vm are private and all outputs are public.
// If an input should be public, then it could just be returned from the function.
// let mut public_input = vec![];
let mut private_input = vec![];
for input in inputs.iter() {
private_input.extend(
input
.as_bytes()
.map_err(|err| NexusError::Prove(ProveError::Client(err)))
.map_err(zkVMError::from)?,
);
}
// TODO: Serialize inputs by `postcard` and make sure there is no double serailization.
// Issue for tracking: https://github.com/eth-act/ere/issues/63.
let now = Instant::now();
let (_view, proof) = prover
.prove_with_input(&private_input, &())
.prove_with_input(&(), &())
.map_err(|e| NexusError::Prove(ProveError::Client(e.into())))
.map_err(zkVMError::from)?;
let elapsed = now.elapsed();

View File

@@ -133,12 +133,7 @@ impl zkVM for EreOpenVM {
let sdk = Sdk::new();
let mut stdin = StdIn::default();
for input in inputs.iter() {
match input {
InputItem::Object(serialize) => stdin.write(serialize),
InputItem::Bytes(items) => stdin.write_bytes(items),
}
}
serialize_inputs(&mut stdin, inputs);
let start = Instant::now();
let _outputs = sdk
@@ -162,12 +157,7 @@ impl zkVM for EreOpenVM {
let sdk = Sdk::new();
let mut stdin = StdIn::default();
for input in inputs.iter() {
match input {
InputItem::Object(serialize) => stdin.write(serialize),
InputItem::Bytes(items) => stdin.write_bytes(items),
}
}
serialize_inputs(&mut stdin, inputs);
let now = std::time::Instant::now();
let proof = sdk
@@ -201,6 +191,17 @@ impl zkVM for EreOpenVM {
}
}
fn serialize_inputs(stdin: &mut StdIn, inputs: &Input) {
for input in inputs.iter() {
match input {
InputItem::Object(obj) => stdin.write(obj),
InputItem::SerializedObject(bytes) | InputItem::Bytes(bytes) => {
stdin.write_bytes(bytes)
}
}
}
}
#[cfg(test)]
mod tests {
use zkvm_interface::Compiler;

View File

@@ -10,6 +10,7 @@ bincode.workspace = true
thiserror.workspace = true
# Pico dependencies
pico-vm.workspace = true
pico-sdk.workspace = true
# Local dependencies

View File

@@ -1,6 +1,7 @@
#![cfg_attr(not(test), warn(unused_crate_dependencies))]
use pico_sdk::client::DefaultProverClient;
use pico_vm::emulator::stdin::EmulatorStdinBuilder;
use std::{path::Path, process::Command, time::Instant};
use zkvm_interface::{
Compiler, Input, InputItem, ProgramExecutionReport, ProgramProvingReport, ProverResourceType,
@@ -72,12 +73,7 @@ impl zkVM for ErePico {
let client = DefaultProverClient::new(&self.program);
let mut stdin = client.new_stdin_builder();
for input in inputs.iter() {
match input {
InputItem::Object(serialize) => stdin.write(serialize),
InputItem::Bytes(items) => stdin.write_slice(items),
}
}
serialize_inputs(&mut stdin, inputs);
let start = Instant::now();
let emulation_result = client.emulate(stdin);
@@ -96,12 +92,8 @@ impl zkVM for ErePico {
let client = DefaultProverClient::new(&self.program);
let mut stdin = client.new_stdin_builder();
for input in inputs.iter() {
match input {
InputItem::Object(serialize) => stdin.write(serialize),
InputItem::Bytes(items) => stdin.write_slice(items),
}
}
serialize_inputs(&mut stdin, inputs);
let now = std::time::Instant::now();
let meta_proof = client.prove(stdin).expect("Failed to generate proof");
let elapsed = now.elapsed();
@@ -139,6 +131,17 @@ impl zkVM for ErePico {
}
}
fn serialize_inputs(stdin: &mut EmulatorStdinBuilder<Vec<u8>>, inputs: &Input) {
for input in inputs.iter() {
match input {
InputItem::Object(serialize) => stdin.write(serialize),
InputItem::SerializedObject(items) | InputItem::Bytes(items) => {
stdin.write_slice(items)
}
}
}
}
#[cfg(test)]
mod tests {
use crate::PICO_TARGET;

View File

@@ -2,7 +2,9 @@
use crate::error::Risc0Error;
use compile::compile_risc0_program;
use risc0_zkvm::{ExecutorEnv, ProverOpts, Receipt, default_executor, default_prover};
use risc0_zkvm::{
ExecutorEnv, ExecutorEnvBuilder, ProverOpts, Receipt, default_executor, default_prover,
};
use std::{path::Path, time::Instant};
use zkvm_interface::{
Compiler, Input, InputItem, ProgramExecutionReport, ProgramProvingReport, ProverResourceType,
@@ -67,16 +69,7 @@ impl zkVM for EreRisc0 {
fn execute(&self, inputs: &Input) -> Result<ProgramExecutionReport, zkVMError> {
let executor = default_executor();
let mut env = ExecutorEnv::builder();
for input in inputs.iter() {
match input {
InputItem::Object(serialize) => {
env.write(serialize).unwrap();
}
InputItem::Bytes(items) => {
env.write_frame(items);
}
}
}
serialize_inputs(&mut env, inputs).map_err(|err| zkVMError::Other(err.into()))?;
let env = env.build().map_err(|err| zkVMError::Other(err.into()))?;
let start = Instant::now();
@@ -93,16 +86,7 @@ impl zkVM for EreRisc0 {
fn prove(&self, inputs: &Input) -> Result<(Vec<u8>, ProgramProvingReport), zkVMError> {
let prover = default_prover();
let mut env = ExecutorEnv::builder();
for input in inputs.iter() {
match input {
InputItem::Object(serialize) => {
env.write(serialize).unwrap();
}
InputItem::Bytes(items) => {
env.write_frame(items);
}
}
}
serialize_inputs(&mut env, inputs).map_err(|err| zkVMError::Other(err.into()))?;
let env = env.build().map_err(|err| zkVMError::Other(err.into()))?;
let now = std::time::Instant::now();
@@ -134,6 +118,26 @@ impl zkVM for EreRisc0 {
}
}
fn serialize_inputs(env: &mut ExecutorEnvBuilder, inputs: &Input) -> Result<(), anyhow::Error> {
for input in inputs.iter() {
match input {
// Corresponding to `env.read::<T>()`.
InputItem::Object(obj) => env.write(obj)?,
// Corresponding to `env.read::<T>()`.
//
// Note that we call `write_slice` to append the bytes to the inputs
// directly, to avoid double serailization.
InputItem::SerializedObject(bytes) => env.write_slice(bytes),
// Corresponding to `env.read_frame()`.
//
// Note that `write_frame` is different from `write_slice`, it
// prepends the `bytes.len().to_le_bytes()`.
InputItem::Bytes(bytes) => env.write_frame(bytes),
};
}
Ok(())
}
#[cfg(test)]
mod prove_tests {
use std::path::PathBuf;

View File

@@ -162,12 +162,7 @@ impl EreSP1 {
impl zkVM for EreSP1 {
fn execute(&self, inputs: &Input) -> Result<zkvm_interface::ProgramExecutionReport, zkVMError> {
let mut stdin = SP1Stdin::new();
for input in inputs.iter() {
match input {
InputItem::Object(serialize) => stdin.write(serialize),
InputItem::Bytes(items) => stdin.write_slice(items),
}
}
serialize_inputs(&mut stdin, inputs);
let client = Self::create_client(&self.resource);
let start = Instant::now();
@@ -186,12 +181,7 @@ impl zkVM for EreSP1 {
info!("Generating proof…");
let mut stdin = SP1Stdin::new();
for input in inputs.iter() {
match input {
InputItem::Object(serialize) => stdin.write(serialize),
InputItem::Bytes(items) => stdin.write_slice(items),
};
}
serialize_inputs(&mut stdin, inputs);
let client = Self::create_client(&self.resource);
let start = std::time::Instant::now();
@@ -223,6 +213,17 @@ impl zkVM for EreSP1 {
}
}
fn serialize_inputs(stdin: &mut SP1Stdin, inputs: &Input) {
for input in inputs.iter() {
match input {
InputItem::Object(obj) => stdin.write(obj),
InputItem::SerializedObject(bytes) | InputItem::Bytes(bytes) => {
stdin.write_slice(bytes)
}
}
}
}
#[cfg(test)]
mod execute_tests {
use std::path::PathBuf;

View File

@@ -15,8 +15,8 @@ use std::{
};
use tempfile::{TempDir, tempdir};
use zkvm_interface::{
Compiler, Input, ProgramExecutionReport, ProgramProvingReport, ProverResourceType, zkVM,
zkVMError,
Compiler, Input, InputItem, ProgramExecutionReport, ProgramProvingReport, ProverResourceType,
zkVM, zkVMError,
};
include!(concat!(env!("OUT_DIR"), "/name_and_sdk_version.rs"));
@@ -56,18 +56,12 @@ impl EreZisk {
}
}
impl EreZisk {}
impl zkVM for EreZisk {
fn execute(&self, input: &Input) -> Result<ProgramExecutionReport, zkVMError> {
fn execute(&self, inputs: &Input) -> Result<ProgramExecutionReport, zkVMError> {
// Write ELF and serialized input to file.
let input_bytes = input
.iter()
.try_fold(Vec::new(), |mut acc, item| {
acc.extend(item.as_bytes().map_err(ExecuteError::SerializeInput)?);
Ok(acc)
})
let input_bytes = serialize_inputs(inputs)
.map_err(|err| ExecuteError::SerializeInput(err.into()))
.map_err(ZiskError::Execute)?;
let mut tempdir =
@@ -119,15 +113,11 @@ impl zkVM for EreZisk {
})
}
fn prove(&self, input: &Input) -> Result<(Vec<u8>, ProgramProvingReport), zkVMError> {
fn prove(&self, inputs: &Input) -> Result<(Vec<u8>, ProgramProvingReport), zkVMError> {
// Write ELF and serialized input to file.
let input_bytes = input
.iter()
.try_fold(Vec::new(), |mut acc, item| {
acc.extend(item.as_bytes().map_err(ProveError::SerializeInput)?);
Ok(acc)
})
let input_bytes = serialize_inputs(inputs)
.map_err(|err| ProveError::SerializeInput(err.into()))
.map_err(ZiskError::Prove)?;
let mut tempdir =
@@ -294,6 +284,18 @@ impl zkVM for EreZisk {
}
}
fn serialize_inputs(inputs: &Input) -> Result<Vec<u8>, bincode::Error> {
inputs.iter().try_fold(Vec::new(), |mut acc, item| {
match item {
InputItem::Object(obj) => {
bincode::serialize_into(&mut acc, obj)?;
}
InputItem::SerializedObject(bytes) | InputItem::Bytes(bytes) => acc.extend(bytes),
};
Ok(acc)
})
}
fn dot_zisk_dir_path() -> PathBuf {
PathBuf::from(std::env::var("HOME").expect("env `$HOME` should be set")).join(".zisk")
}

View File

@@ -7,7 +7,6 @@ license.workspace = true
[dependencies]
auto_impl.workspace = true
bincode.workspace = true
erased-serde.workspace = true
indexmap = { workspace = true, features = ["serde"] }
serde = { workspace = true, features = ["derive"] }
@@ -17,6 +16,7 @@ thiserror.workspace = true
clap = { workspace = true, features = ["derive"], optional = true }
[dev-dependencies]
bincode.workspace = true
serde_json.workspace = true
[lints]

View File

@@ -1,14 +1,18 @@
use std::{fmt::Debug, sync::Arc};
use bincode::Options;
use erased_serde::Serialize as ErasedSerialize;
use serde::Serialize;
use std::{fmt::Debug, sync::Arc};
#[derive(Clone)]
pub enum InputItem {
/// A serializable object stored as a trait object
Object(Arc<dyn ErasedSerialize + Send + Sync>),
/// Pre-serialized bytes (e.g., from bincode)
/// A serialized object with zkvm specific serializer.
///
/// This is only for `ere-dockerized` to serialize the inputs to be able to
/// pass to `ere-cli` to do the actual action, in normal case this should be
/// avoided, instead [`InputItem::Object`] should be used.
SerializedObject(Vec<u8>),
/// Serialized bytes with opaque serializer (e.g. bincode)
Bytes(Vec<u8>),
}
@@ -16,6 +20,9 @@ impl Debug for InputItem {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
InputItem::Object(_) => f.write_str("Object(<erased>)"),
InputItem::SerializedObject(bytes) => {
f.debug_tuple("SerializedObject").field(bytes).finish()
}
InputItem::Bytes(bytes) => f.debug_tuple("Bytes").field(bytes).finish(),
}
}
@@ -72,39 +79,6 @@ impl From<Vec<InputItem>> for Input {
}
}
// Optional: Implement methods to work with the enum
impl InputItem {
/// Serialize this item to bytes using the specified serializer
pub fn serialize_with<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
match self {
InputItem::Object(obj) => erased_serde::serialize(obj.as_ref(), serializer),
InputItem::Bytes(bytes) => {
// Serialize the bytes as a byte array
bytes.serialize(serializer)
}
}
}
/// Get the item as bytes (serialize objects, return bytes directly)
pub fn as_bytes(&self) -> Result<Vec<u8>, Box<dyn std::error::Error + Send + Sync>> {
match self {
InputItem::Object(obj) => {
let mut buf = Vec::new();
let mut serializer = bincode::Serializer::new(
&mut buf,
bincode::DefaultOptions::new().with_fixint_encoding(),
);
erased_serde::serialize(obj.as_ref(), &mut serializer)?;
Ok(buf)
}
InputItem::Bytes(bytes) => Ok(bytes.to_vec()),
}
}
}
#[cfg(test)]
mod input_erased_tests {
use super::*;
@@ -130,7 +104,9 @@ mod input_erased_tests {
match &input.items[0] {
InputItem::Object(_) => (), // Success
InputItem::Bytes(_) => panic!("Expected Object, got Bytes"),
InputItem::SerializedObject(_) | InputItem::Bytes(_) => {
panic!("Expected Object, got Bytes")
}
}
}
@@ -145,28 +121,9 @@ mod input_erased_tests {
match &input.items[0] {
InputItem::Bytes(stored_bytes) => assert_eq!(stored_bytes.to_vec(), bytes),
InputItem::Object(_) => panic!("Expected Bytes, got Object"),
}
}
#[test]
fn test_write_serialized() {
let mut input = Input::new();
let person = Person {
name: "Bob".to_string(),
age: 25,
};
// User serializes themselves and writes bytes
let serialized = bincode::serialize(&person).unwrap();
input.write_bytes(serialized);
assert_eq!(input.len(), 1);
match &input.items[0] {
InputItem::Bytes(_) => (), // Success
InputItem::Object(_) => panic!("Expected Bytes, got Object"),
InputItem::Object(_) | InputItem::SerializedObject(_) => {
panic!("Expected Bytes, got Object")
}
}
}
@@ -207,27 +164,6 @@ mod input_erased_tests {
}
}
#[test]
fn test_as_bytes() {
let mut input = Input::new();
// Add an object
input.write(42i32);
// Add raw bytes
input.write_bytes(vec![1, 2, 3]);
// Convert both to bytes
let obj_bytes = input.items[0].as_bytes().unwrap();
let raw_bytes = input.items[1].as_bytes().unwrap();
// The object should be serialized to some bytes
assert!(!obj_bytes.is_empty());
// The raw bytes should be returned as-is
assert_eq!(raw_bytes, vec![1, 2, 3]);
}
#[test]
fn test_iteration() {
let mut input = Input::new();

View File

@@ -8,7 +8,7 @@ WORKDIR /ere
ARG ZKVM
RUN cargo build --release --package ere-cli --bin ere-cli --features ${ZKVM} && \
RUN cargo build --release --package ere-cli --bin ere-cli --features cli,${ZKVM} && \
cp /ere/target/release/ere-cli /ere/ere-cli && \
cargo clean && \
rm -rf $CARGO_HOME/registry/src $CARGO_HOME/registry/cache