github/ere
1
1
Fork 0
mirror of https://github.com/eth-act/ere.git synced 2026-02-19 11:54:42 -05:00
Code Issues Packages Projects Releases Wiki Activity

sp1: use Docker for guest program compilation (#54)

Browse Source 
Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com>
This commit is contained in:
Ignacio Hagopian
2025-07-16 13:54:15 +02:00
committed by GitHub
parent 6af8c939ff
commit 3da61c14cb
15 changed files with 359 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
.github/workflows/rust-checks.yml vendored Normal file
View File

@@ -0,0 +1,59 @@
name: Rust Checks
on:
push:
branches: [ main, master ]
pull_request:
branches: [ main, master ]
env:
CARGO_TERM_COLOR: always
jobs:
check-fmt:
name: Lints
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@nightly
with:
components: rustfmt
- name: Cache dependencies
uses: Swatinem/rust-cache@v2
- name: Check formatting
run: cargo fmt --check --all
check-tests:
name: Tests
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
crate: [ere-sp1]
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@nightly
with:
components: clippy
- name: Cache dependencies
uses: Swatinem/rust-cache@v2
- name: Check clippy
run: cargo clippy --bins --lib --examples --tests --benches --all-features -p ${{ matrix.crate }}
- name: Run tests
run: cargo test --release -p ${{ matrix.crate }}

33
.github/workflows/test-sp1-docker.yml vendored
View File

@@ -1,33 +0,0 @@
name: Test SP1 (Docker)
on:
push:
branches:
- master
pull_request:
branches:
- master
jobs:
test-sp1-via-docker-build:
name: Build SP1 Docker Image
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build ere-base image
run: |
docker build \
--tag ere-base:latest \
--file docker/base/Dockerfile.base .
- name: Build ere-builder-sp1 image
run: |
docker build \
--tag ere-builder-sp1:latest \
--file docker/sp1/Dockerfile .

9
Cargo.toml
View File

@@ -10,6 +10,9 @@ members = [
"crates/ere-pico",
"crates/ere-jolt",
"crates/ere-zisk",
# Guest compilers
"docker/sp1",
]
resolver = "2"
@@ -22,6 +25,12 @@ license = "MIT OR Apache-2.0"
[workspace.lints]
[workspace.dependencies]
tracing = "0.1.41"
tempfile = "3.3"
toml = "0.8"
clap = { version = "4.5.41", features = ["derive"] }
anyhow = "1.0"
# local dependencies
zkvm-interface = { path = "crates/zkvm-interface" }
build-utils = { path = "crates/build-utils" }

10
README.md
View File

@@ -49,13 +49,19 @@
## Quick Start
### 1. Install SDKs
This guide assumes you have Rust and Cargo installed. If not, please refer to the [Rust installation guide](https://www.rust-lang.org/tools/install).
Also, you must have Docker installed since some of the SDKs require it.
### 1. Install SDKs (if required)
All zkVMs but SP1 require you to install their SDKs, for example:
```bash
bash scripts/sdk_installers/install_sp1_sdk.sh
bash scripts/sdk_installers/install_jolt_sdk.sh
```
For SP1, guest program compilation uses Docker. With time more zkVMs will follow this patterns so installing SDKs
in the host machine isn't necessary.
### 2. Add Dependencies
```toml

2
crates/build-utils/Cargo.toml
View File

@@ -6,7 +6,9 @@ rust-version.workspace = true
license.workspace = true
[dependencies]
tracing.workspace = true
cargo_metadata = "0.20.0"
thiserror = "2.0.12"
[lints]
workspace = true

85
crates/build-utils/src/docker.rs Normal file
View File

@@ -0,0 +1,85 @@
use std::{
path::{Path, PathBuf},
process::{Command, Stdio},
};
use thiserror::Error;
use tracing::info;
pub fn build_image(compiler_dockerfile: &Path, tag: &str) -> Result<(), Error> {
// Check that Docker is installed and available
if Command::new("docker")
.arg("--version")
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()
.is_err()
{
return Err(Error::DockerIsNotAvailable);
}
info!(
"Building Docker image in {} with tag {}",
compiler_dockerfile.display(),
tag
);
let cargo_workspace_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("../..")
.canonicalize()
.unwrap();
// Build base image
info!("Building base Docker image...");
let dockerfile_base_path = cargo_workspace_dir.join("docker/base/Dockerfile.base");
let status = Command::new("docker")
.args([
"build",
"-t",
"ere-base:latest",
"-f",
dockerfile_base_path
.to_str()
.ok_or_else(|| Error::InvalidDockerfilePath(dockerfile_base_path.clone()))?,
cargo_workspace_dir.to_str().unwrap(),
])
.status()
.map_err(|e| Error::DockerBuildFailed(e.into()))?;
if !status.success() {
return Err(Error::ImageBuildFailed);
}
info!("Building guest compiler image...");
let dockerfile_path = cargo_workspace_dir.join(compiler_dockerfile);
let status = Command::new("docker")
.args([
"build",
"-t",
tag,
"-f",
dockerfile_path
.to_str()
.ok_or_else(|| Error::InvalidDockerfilePath(dockerfile_path.clone()))?,
cargo_workspace_dir.to_str().unwrap(),
])
.status()
.map_err(|e| Error::DockerBuildFailed(e.into()))?;
if !status.success() {
return Err(Error::ImageBuildFailed);
}
Ok(())
}
#[derive(Debug, Error)]
pub enum Error {
#[error("Invalid Dockerfile path: {0}")]
InvalidDockerfilePath(PathBuf),
#[error("Docker image build failed: {0}")]
DockerBuildFailed(#[source] Box<dyn std::error::Error + Send + Sync + 'static>),
#[error("Docker image build failed")]
ImageBuildFailed,
#[error("Docker is not available. Please ensure Docker is installed and running.")]
DockerIsNotAvailable,
}

2
crates/build-utils/src/lib.rs
View File

@@ -1,5 +1,7 @@
use std::{env, fs, path::Path};
pub mod docker;
// Detect and generate a Rust source file that contains the name and version of the SDK.
pub fn detect_and_generate_name_and_sdk_version(name: &str, sdk_dep_name: &str) {
let meta = cargo_metadata::MetadataCommand::new()

6
crates/ere-sp1/Cargo.toml
View File

@@ -6,16 +6,16 @@ rust-version.workspace = true
license.workspace = true
[dependencies]
sp1-sdk = "5.0.5"
zkvm-interface = { workspace = true }
toml = "0.8"
build-utils.workspace = true
sp1-sdk = "5.0.5"
tempfile = "3.3"
bincode = "1.3"
thiserror = "2"
tracing = "0.1"
[build-dependencies]
build-utils = { workspace = true }
build-utils.workspace = true
[lib]
name = "ere_succinct"

123
crates/ere-sp1/src/compile.rs
View File

@@ -1,102 +1,59 @@
use std::{fs, path::Path, process::Command};
use std::{
path::{Path, PathBuf},
process::Command,
};
use build_utils::docker;
use tempfile::TempDir;
use toml::Value as TomlValue;
use tracing::info;
use crate::error::CompileError;
/// Compile the guest crate and return raw ELF bytes.
pub fn compile_sp1_program(program_crate_path: &Path) -> Result<Vec<u8>, CompileError> {
info!("Compiling SP1 program at {}", program_crate_path.display());
pub fn compile(guest_program_full_path: &Path) -> Result<Vec<u8>, CompileError> {
// Build the SP1 docker image
let tag = "ere-build-sp1:latest";
docker::build_image(&PathBuf::from("docker/sp1/Dockerfile"), tag)
.map_err(|e| CompileError::DockerImageBuildFailed(Box::new(e)))?;
if !program_crate_path.exists() || !program_crate_path.is_dir() {
return Err(CompileError::InvalidProgramPath(
program_crate_path.to_path_buf(),
));
}
// Compile the guest program using the SP1 docker image
let guest_program_path_str = guest_program_full_path
.to_str()
.ok_or_else(|| CompileError::InvalidGuestPath(guest_program_full_path.to_path_buf()))?;
let elf_output_dir = TempDir::new().map_err(CompileError::CreatingTempOutputDirectoryFailed)?;
let elf_output_dir_str = elf_output_dir
.path()
.to_str()
.ok_or_else(|| CompileError::InvalidTempOutputPath(elf_output_dir.path().to_path_buf()))?;
let guest_manifest_path = program_crate_path.join("Cargo.toml");
if !guest_manifest_path.exists() {
return Err(CompileError::CargoTomlMissing {
program_dir: program_crate_path.to_path_buf(),
manifest_path: guest_manifest_path.clone(),
});
}
info!("Compiling program: {}", guest_program_path_str);
// ── read + parse Cargo.toml ───────────────────────────────────────────
let manifest_content =
fs::read_to_string(&guest_manifest_path).map_err(|e| CompileError::ReadFile {
path: guest_manifest_path.clone(),
source: e,
})?;
let manifest_toml: TomlValue =
manifest_content
.parse::<TomlValue>()
.map_err(|e| CompileError::ParseCargoToml {
path: guest_manifest_path.clone(),
source: e,
})?;
let program_name = manifest_toml
.get("package")
.and_then(|p| p.get("name"))
.and_then(|n| n.as_str())
.ok_or_else(|| CompileError::MissingPackageName {
path: guest_manifest_path.clone(),
})?;
info!("Parsed program name: {program_name}");
// ── build into a temp dir ─────────────────────────────────────────────
let temp_output_dir = TempDir::new_in(program_crate_path)?;
let temp_output_dir_path = temp_output_dir.path();
let elf_name = format!("{program_name}.elf");
info!(
"Running `cargo prove build` → dir: {}, ELF: {}",
temp_output_dir_path.display(),
elf_name
);
let status = Command::new("cargo")
.current_dir(program_crate_path)
let status = Command::new("docker")
.args([
"prove",
"build",
"--output-directory",
temp_output_dir_path.to_str().unwrap(),
"--elf-name",
&elf_name,
"run",
"--rm",
// Mount volumes
"-v",
&format!("{guest_program_path_str}:/guest-program"),
"-v",
&format!("{elf_output_dir_str}:/output"),
tag,
// Guest compiler execution
"./guest-compiler",
"/guest-program",
"/output",
])
.stdout(std::process::Stdio::inherit())
.stderr(std::process::Stdio::inherit())
.status()
.map_err(|e| CompileError::CargoProveBuild {
cwd: program_crate_path.to_path_buf(),
source: e,
})?;
.map_err(CompileError::DockerCommandFailed)?;
if !status.success() {
return Err(CompileError::CargoBuildFailed {
status,
path: program_crate_path.to_path_buf(),
});
return Err(CompileError::DockerContainerRunFailed(status));
}
let elf_path = temp_output_dir_path.join(&elf_name);
if !elf_path.exists() {
return Err(CompileError::ElfNotFound(elf_path));
}
// Read the compiled ELF program from the output directory
let elf = std::fs::read(elf_output_dir.path().join("guest.elf"))
.map_err(CompileError::ReadCompiledELFProgram)?;
let elf_bytes = fs::read(&elf_path).map_err(|e| CompileError::ReadFile {
path: elf_path,
source: e,
})?;
info!("SP1 program compiled OK – {} bytes", elf_bytes.len());
Ok(elf_bytes)
Ok(elf)
}
#[cfg(test)]
@@ -125,7 +82,7 @@ mod tests {
fn test_compile_sp1_program() {
let test_guest_path = get_compile_test_guest_program_path();
match compile_sp1_program(&test_guest_path) {
match compile(&test_guest_path) {
Ok(elf_bytes) => {
assert!(!elf_bytes.is_empty(), "ELF bytes should not be empty.");
}

51
crates/ere-sp1/src/error.rs
View File

@@ -1,4 +1,4 @@
use std::{path::PathBuf, process::ExitStatus};
use std::path::PathBuf;
use thiserror::Error;
use zkvm_interface::zkVMError;
@@ -27,41 +27,20 @@ pub enum SP1Error {
/// Errors that can be encountered while compiling a SP1 program
#[derive(Debug, Error)]
pub enum CompileError {
#[error("Program path does not exist or is not a directory: {0}")]
InvalidProgramPath(PathBuf),
#[error(
"Cargo.toml not found in program directory: {program_dir}. Expected at: {manifest_path}"
)]
CargoTomlMissing {
program_dir: PathBuf,
manifest_path: PathBuf,
},
#[error("Could not find `[package].name` in guest Cargo.toml at {path}")]
MissingPackageName { path: PathBuf },
#[error("Compiled ELF not found at expected path: {0}")]
ElfNotFound(PathBuf),
#[error("`cargo prove build` failed with status: {status} for program at {path}")]
CargoBuildFailed { status: ExitStatus, path: PathBuf },
#[error("Failed to read file at {path}: {source}")]
ReadFile {
path: PathBuf,
#[source]
source: std::io::Error,
},
#[error("Failed to parse guest Cargo.toml at {path}: {source}")]
ParseCargoToml {
path: PathBuf,
#[source]
source: toml::de::Error,
},
#[error("Failed to execute `cargo prove build` in {cwd}: {source}")]
CargoProveBuild {
cwd: PathBuf,
#[source]
source: std::io::Error,
},
#[error("Failed to create temporary output directory: {0}")]
TempDir(#[from] std::io::Error),
#[error("Failed to build Docker image: {0}")]
DockerImageBuildFailed(#[source] Box<dyn std::error::Error + Send + Sync + 'static>),
#[error("Docker command failed to execute: {0}")]
DockerCommandFailed(#[source] std::io::Error),
#[error("Docker container run failed with status: {0}")]
DockerContainerRunFailed(std::process::ExitStatus),
#[error("Invalid guest program path: {0}")]
InvalidGuestPath(PathBuf),
#[error("Failed to create temporary directory: {0}")]
CreatingTempOutputDirectoryFailed(#[source] std::io::Error),
#[error("Failed to create temporary output path: {0}")]
InvalidTempOutputPath(PathBuf),
#[error("Failed to read compiled ELF program: {0}")]
ReadCompiledELFProgram(#[source] std::io::Error),
}
#[derive(Debug, Error)]

3
crates/ere-sp1/src/lib.rs
View File

@@ -2,7 +2,6 @@
use std::time::Instant;
use compile::compile_sp1_program;
use sp1_sdk::{
CpuProver, CudaProver, NetworkProver, Prover, ProverClient, SP1ProofWithPublicValues,
SP1ProvingKey, SP1Stdin, SP1VerifyingKey,
@@ -108,7 +107,7 @@ impl Compiler for RV32_IM_SUCCINCT_ZKVM_ELF {
type Program = Vec<u8>;
fn compile(path_to_program: &std::path::Path) -> Result<Self::Program, Self::Error> {
compile_sp1_program(path_to_program).map_err(SP1Error::from)
compile::compile(path_to_program).map_err(SP1Error::from)
}
}

16
docker/sp1/Cargo.toml Normal file
View File

@@ -0,0 +1,16 @@
[package]
name = "sp1-guest-compiler"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
[dependencies]
tempfile.workspace = true
toml.workspace = true
tracing.workspace = true
clap.workspace = true
anyhow.workspace = true
[lints]
workspace = true

21
docker/sp1/Dockerfile
View File

@@ -1,4 +1,13 @@
ARG BASE_IMAGE_TAG=latest
# Build guest-compiler binary
FROM rust:1.85 AS builder
RUN apt-get update && apt-get install -y build-essential libclang-dev
WORKDIR /guest-compiler
COPY . .
RUN cargo build --release -p sp1-guest-compiler
# Build zkVM builder image
FROM ere-base:${BASE_IMAGE_TAG}
ARG USERNAME=ere_user
@@ -18,6 +27,7 @@ RUN chmod +x /tmp/install_sp1_sdk.sh
# The install_sp1_sdk.sh script will respect these ENV variables.
# TODO: we are hardcoding /root which may not work for other users
ENV SP1UP_HOME="/root/.sp1up" \
SP1UP_SDK_INSTALL_VERSION="v5.0.8" \
SP1_HOME="/root/.sp1"
# Run the SP1 SDK installation script
@@ -31,14 +41,9 @@ ENV PATH="${SP1UP_HOME}/bin:${SP1_HOME}/bin:$PATH"
# Verify SP1 installation (optional here, as script does it, but good for sanity)
RUN cargo prove --version
# Copy the entire ere project context
# The WORKDIR is /app from the base image
WORKDIR /app
COPY . .
# Run tests
RUN echo "Running tests for ere-sp1 library..." && \
cargo test --release -p ere-sp1 --lib -- --color always
# Copy guest compiler binary
COPY --from=builder /guest-compiler/target/release/sp1-guest-compiler /guest-compiler/guest-compiler
WORKDIR /guest-compiler
CMD ["/bin/bash"]

103
docker/sp1/src/main.rs Normal file
View File

@@ -0,0 +1,103 @@
use std::{fs, path::PathBuf, process::Command};
use anyhow::Context;
use clap::Parser;
use toml::Value as TomlValue;
use tracing::info;
#[derive(Parser)]
#[command(author, version)]
struct Cli {
/// Path to the guest program crate directory.
guest_folder: PathBuf,
/// Compiled ELF output folder where guest.elf will be placed.
elf_output_folder: PathBuf,
}
pub fn main() -> anyhow::Result<()> {
let args = Cli::parse();
let dir = args.guest_folder;
info!("Compiling SP1 program at {}", dir.display());
if !dir.exists() || !dir.is_dir() {
anyhow::bail!(
"Program path does not exist or is not a directory: {}",
dir.display()
);
}
let guest_manifest_path = dir.join("Cargo.toml");
if !guest_manifest_path.exists() {
anyhow::bail!(
"Cargo.toml not found in program directory: {}. Expected at: {}",
dir.display(),
guest_manifest_path.display()
);
}
// ── read + parse Cargo.toml ───────────────────────────────────────────
let manifest_content = fs::read_to_string(&guest_manifest_path)
.with_context(|| format!("Failed to read file at {}", guest_manifest_path.display()))?;
let manifest_toml: TomlValue = manifest_content.parse::<TomlValue>().with_context(|| {
format!(
"Failed to parse guest Cargo.toml at {}",
guest_manifest_path.display()
)
})?;
let program_name = manifest_toml
.get("package")
.and_then(|p| p.get("name"))
.and_then(|n| n.as_str())
.with_context(|| {
format!(
"Could not find `[package].name` in guest Cargo.toml at {}",
guest_manifest_path.display()
)
})?;
info!("Parsed program name: {program_name}");
// ── build into a temp dir ─────────────────────────────────────────────
info!(
"Running `cargo prove build` → dir: {}",
args.elf_output_folder.display()
);
let status = Command::new("cargo")
.current_dir(&dir)
.args([
"prove",
"build",
"--output-directory",
args.elf_output_folder.to_str().unwrap(),
"--elf-name",
"guest.elf",
])
.stdout(std::process::Stdio::inherit())
.stderr(std::process::Stdio::inherit())
.status()
.with_context(|| format!("Failed to execute `cargo prove build` in {}", dir.display()))?;
if !status.success() {
anyhow::bail!("Failed to execute `cargo prove build` in {}", dir.display())
}
let elf_path = args.elf_output_folder.join("guest.elf");
if !elf_path.exists() {
anyhow::bail!(
"Compiled ELF not found at expected path: {}",
elf_path.display()
);
}
let elf_bytes = fs::read(&elf_path)
.with_context(|| format!("Failed to read file at {}", elf_path.display()))?;
info!("SP1 program compiled OK - {} bytes", elf_bytes.len());
Ok(())
}

4
scripts/sdk_installers/install_sp1_sdk.sh
View File

@@ -29,12 +29,14 @@ curl -L https://sp1up.succinct.xyz | bash -s -- --yes
# and for subsequent commands if this script is sourced.
export PATH="${SP1UP_HOME}/bin:${SP1_HOME}/bin:$PATH"
export SDK_VERSION="${SP1UP_SDK_INSTALL_VERSION:-latest}"
# Run sp1up to install/update the toolchain
if ! command -v sp1up &> /dev/null; then
echo "Error: sp1up command not found after installation script. Check PATH or installation." >&2
exit 1
fi
sp1up # Installs the toolchain and cargo-prove
sp1up -v ${SDK_VERSION} # Installs the toolchain and cargo-prove
echo "Verifying SP1 installation..."
if ! command -v cargo &> /dev/null; then