Support building jolt guest program with stock rust compiler. Add compilation and execution unit tests.

WiP. Works but needs cleanup

Cargo.lock

@@ -2270,8 +2270,10 @@ dependencies = [
 name = "ere-jolt"
 version = "0.0.11"
 dependencies = [
+ "anyhow",
  "ark-serialize 0.5.0",
  "build-utils",
+ "cargo_metadata 0.19.2",
  "common",
  "jolt",
  "jolt-core",
@@ -2281,6 +2283,7 @@ dependencies = [
  "test-utils",
  "thiserror 2.0.12",
  "toml",
+ "tracing",
  "zkvm-interface",
 ]
 

crates/ere-jolt/Cargo.toml

@@ -6,10 +6,13 @@ rust-version.workspace = true
 license.workspace = true
 
 [dependencies]
+anyhow.workspace = true
+cargo_metadata.workspace = true
 serde.workspace = true
 tempfile.workspace = true
 thiserror.workspace = true
 toml.workspace = true
+tracing.workspace = true
 
 # Jolt dependencies
 ark-serialize = { workspace = true, features = ["derive"] }

crates/ere-jolt/src/compile_stock_rust.rs

@@ -0,0 +1,154 @@
+use std::fs;
+use std::fs::File;
+use crate::error::CompileError;
+use std::io::Write;
+use std::path::Path;
+use std::process::{Command};
+use cargo_metadata::MetadataCommand;
+use tempfile::TempDir;
+use tracing::info;
+
+static CARGO_ENCODED_RUSTFLAGS_SEPARATOR: &str = "\x1f";
+
+pub fn compile_jolt_program_stock_rust(
+    guest_directory: &Path,
+    toolchain: &String,
+) -> Result<Vec<u8>, CompileError> {
+
+    let metadata = MetadataCommand::new().current_dir(guest_directory).exec()?;
+    let package = metadata
+        .root_package()
+        .ok_or_else(|| CompileError::MissingPackageName {
+            path: guest_directory.to_path_buf(),
+        })?;
+
+    let target_name = "riscv32im-unknown-none-elf";
+    let plus_toolchain = format!("+{}", toolchain);
+
+    let args = [
+        plus_toolchain.as_str(),
+        "build",
+        "--target",
+        target_name,
+        "--release",
+        // For bare metal we have to build core and alloc
+        "-Zbuild-std=core,alloc",
+        "--features",
+        "guest"
+    ];
+
+    let temp_output_dir = TempDir::new_in(guest_directory).unwrap();
+    let temp_output_dir_path = temp_output_dir.path();
+
+    let linker_path = temp_output_dir_path.join(format!("{}.ld", &package.name));
+    let linker_path_str = linker_path.to_str().unwrap();
+
+    let linker_script = LINKER_SCRIPT_TEMPLATE
+        .replace("{MEMORY_SIZE}", &DEFAULT_MEMORY_SIZE.to_string())
+        .replace("{STACK_SIZE}", &DEFAULT_STACK_SIZE.to_string());
+
+    let mut file = File::create(&linker_path).expect("could not create linker file");
+    file.write_all(linker_script.as_bytes())
+        .expect("could not save linker");
+
+    let rust_flags = [
+        "-C",
+        &format!("link-arg=-T{}", linker_path_str),
+        "-C",
+        "passes=lower-atomic",
+        "-C",
+        "panic=abort",
+        "-C",
+        "strip=symbols",
+        "-C",
+        "opt-level=z",
+    ];
+
+    let encoded_rust_flags = rust_flags
+        .into_iter()
+        .collect::<Vec<_>>()
+        .join(CARGO_ENCODED_RUSTFLAGS_SEPARATOR);
+
+    let result = Command::new("cargo")
+        .current_dir(guest_directory)
+        .env("CARGO_ENCODED_RUSTFLAGS", &encoded_rust_flags)
+        .args(args)
+        .stdout(std::process::Stdio::inherit())
+        .stderr(std::process::Stdio::inherit())
+        .status()
+        .map_err(|source| CompileError::BuildFailure {
+            source: source.into(),
+            crate_path: guest_directory.to_path_buf()
+        });
+
+    if result.is_err() {
+        return Err(result.err().unwrap());
+    }
+
+    let elf_path =
+        guest_directory
+            .join("target")
+            .join(target_name)
+            .join("release")
+            .join(&package.name);
+
+    let elf = fs::read(&elf_path).map_err(|e| CompileError::ReadElfFailed {
+        path: elf_path,
+        source: e,
+    })?;
+
+    info!("Jolt program compiled (toolchain {}) OK - {} bytes", toolchain, elf.len());
+
+    Ok(elf)
+}
+
+pub const DEFAULT_MEMORY_SIZE: u64 = 10 * 1024 * 1024;
+pub const DEFAULT_STACK_SIZE: u64 = 4096;
+
+const LINKER_SCRIPT_TEMPLATE: &str = r#"
+MEMORY {
+  program (rwx) : ORIGIN = 0x80000000, LENGTH = {MEMORY_SIZE}
+}
+
+SECTIONS {
+  .text.boot : {
+    *(.text.boot)
+  } > program
+
+  .text : {
+    *(.text)
+  } > program
+
+  .data : {
+    *(.data)
+  } > program
+
+  .bss : {
+    *(.bss)
+  } > program
+
+  . = ALIGN(8);
+  . = . + {STACK_SIZE};
+  _STACK_PTR = .;
+  . = ALIGN(8);
+  _HEAP_PTR = .;
+}
+"#;
+
+
+#[cfg(test)]
+mod tests {
+    use test_utils::host::testing_guest_directory;
+    use crate::compile_stock_rust::compile_jolt_program_stock_rust;
+
+    #[test]
+    fn test_stock_compiler_impl() {
+        let guest_directory = testing_guest_directory(
+            "jolt",
+            "stock_nightly_no_std");
+        let program = compile_jolt_program_stock_rust(&guest_directory, &"nightly".to_string());
+
+        assert!(!program.is_err(), "jolt compilation failed");
+        assert!(!program.unwrap().is_empty(), "ELF bytes should not be empty.");
+    }
+}

crates/ere-jolt/src/error.rs

@@ -34,10 +34,20 @@ pub enum CompileError {
     },
     #[error("Failed to build guest")]
     BuildFailed,
+    #[error("`openvm` build failure for {crate_path} failed: {source}")]
+    BuildFailure {
+        #[source]
+        source: anyhow::Error,
+        crate_path: PathBuf,
+    },
     #[error("Failed to read elf at {path}: {source}")]
     ReadElfFailed { source: io::Error, path: PathBuf },
     #[error("Failed to set current directory to {path}: {source}")]
     SetCurrentDirFailed { source: io::Error, path: PathBuf },
+    #[error("`cargo metadata` failed: {0}")]
+    MetadataCommand(#[from] cargo_metadata::Error),
+    #[error("Could not find `[package].name` in guest Cargo.toml at {path}")]
+    MissingPackageName { path: PathBuf },
 }
 
 #[derive(Debug, Error)]

crates/ere-jolt/src/lib.rs

@@ -21,11 +21,13 @@ use zkvm_interface::{
     Compiler, Input, ProgramExecutionReport, ProgramProvingReport, Proof, ProverResourceType,
     PublicValues, zkVM, zkVMError,
 };
+use compile_stock_rust::compile_jolt_program_stock_rust;
 
 include!(concat!(env!("OUT_DIR"), "/name_and_sdk_version.rs"));
 mod error;
 mod jolt_methods;
 mod utils;
+mod compile_stock_rust;
 
 #[allow(non_camel_case_types)]
 pub struct JOLT_TARGET;
@@ -35,31 +37,40 @@ impl Compiler for JOLT_TARGET {
 
     type Program = Vec<u8>;
 
-    fn compile(&self, guest_dir: &Path) -> Result<Self::Program, Self::Error> {
-        // Change current directory for `Program::build` to build guest program.
-        set_current_dir(guest_dir).map_err(|source| CompileError::SetCurrentDirFailed {
-            source,
-            path: guest_dir.to_path_buf(),
-        })?;
+    fn compile(&self, guest_directory: &Path) -> Result<Self::Program, Self::Error> {
+        let toolchain =
+            env::var("ERE_GUEST_TOOLCHAIN").unwrap_or_else(|_error| "jolt".into());
+        match toolchain.as_str() {
+            "jolt" => Ok(compile_jolt_program(guest_directory)?),
+            _ => Ok(compile_jolt_program_stock_rust(guest_directory, &toolchain)?),
+        }
+    }
+}
 
-        let package_name = package_name_from_manifest(Path::new("Cargo.toml"))?;
+fn compile_jolt_program(guest_directory: &Path) -> Result<Vec<u8>, JoltError> {
+    // Change current directory for `Program::build` to build guest program.
+    set_current_dir(guest_directory).map_err(|source| CompileError::SetCurrentDirFailed {
+        source,
+        path: guest_directory.to_path_buf(),
+    })?;
 
-        // Note that if this fails, it will panic, hence we need to catch it.
-        let elf_path = std::panic::catch_unwind(|| {
-            let mut program = Program::new(&package_name);
-            program.set_std(true);
-            program.build(DEFAULT_TARGET_DIR);
-            program.elf.unwrap()
-        })
+    let package_name = package_name_from_manifest(Path::new("Cargo.toml"))?;
+
+    // Note that if this fails, it will panic, hence we need to catch it.
+    let elf_path = std::panic::catch_unwind(|| {
+        let mut program = Program::new(&package_name);
+        program.set_std(true);
+        program.build(DEFAULT_TARGET_DIR);
+        program.elf.unwrap()
+    })
         .map_err(|_| CompileError::BuildFailed)?;
 
-        let elf = fs::read(&elf_path).map_err(|source| CompileError::ReadElfFailed {
-            source,
-            path: elf_path.to_path_buf(),
-        })?;
+    let elf = fs::read(&elf_path).map_err(|source| CompileError::ReadElfFailed {
+        source,
+        path: elf_path.to_path_buf(),
+    })?;
 
-        Ok(elf)
-    }
+    Ok(elf)
 }
 
 #[derive(CanonicalSerialize, CanonicalDeserialize)]
@@ -174,7 +185,7 @@ pub fn program(elf: &[u8]) -> Result<(TempDir, jolt::host::Program), zkVMError>
 mod tests {
     use super::*;
     use std::sync::OnceLock;
-    use test_utils::host::testing_guest_directory;
+    use test_utils::host::{run_zkvm_execute, testing_guest_directory};
 
     static BASIC_PRORGAM: OnceLock<Vec<u8>> = OnceLock::new();
 
@@ -193,4 +204,19 @@ mod tests {
         let elf_bytes = basic_program();
         assert!(!elf_bytes.is_empty(), "ELF bytes should not be empty.");
     }
+
+    #[test]
+    fn test_execute() {
+        let elf_bytes = basic_program();
+        let zkvm = EreJolt::new(elf_bytes, ProverResourceType::Cpu).unwrap();
+        run_zkvm_execute(&zkvm, &Input::new());
+    }
+    
+    #[test]
+    fn test_execute_nightly() {
+        let guest_directory = testing_guest_directory("jolt", "stock_nightly_no_std");
+        let program = compile_jolt_program_stock_rust(&guest_directory, &"nightly".to_string()).unwrap();
+        let zkvm = EreJolt::new(program, ProverResourceType::Cpu).unwrap();
+        run_zkvm_execute(&zkvm, &Input::new());
+    }
 }

tests/jolt/stock_nightly_no_std/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "addition_no_std"
+edition = "2021"
+
+[features]
+guest = []
+
+[dependencies]
+jolt = { package = "jolt-sdk", git = "https://github.com/a16z/jolt", rev = "55b9830a3944dde55d33a55c42522b81dd49f87a" }
+
+[workspace]

tests/jolt/stock_nightly_no_std/src/main.rs

@@ -0,0 +1,25 @@
+#![cfg_attr(feature = "guest", no_std)]
+#![no_main]
+extern crate alloc;
+
+use alloc::vec::Vec;
+use core::sync::atomic::Ordering;
+use core::sync::atomic::AtomicU16;
+
+#[jolt::provable]
+fn foo() {
+    let a: AtomicU16 = core::hint::black_box(AtomicU16::new(5));
+    let b: AtomicU16 = core::hint::black_box(AtomicU16::new(7));
+
+    if a.load(Ordering::SeqCst) + b.load(Ordering::SeqCst) != 12 {
+        panic!("Something went wrong!");
+    }
+
+    let mut v: Vec<AtomicU16> = Vec::new();
+    v.push(AtomicU16::new(5));
+    v.push(AtomicU16::new(7));
+
+    if v[0].load(Ordering::SeqCst) + v[1].load(Ordering::SeqCst) != 12 {
+        panic!("Something went wrong!");
+    }
+}