diff --git a/analysis/src/vm/mod.rs b/analysis/src/vm/mod.rs index 735997e2f..d7fc8b4c9 100644 --- a/analysis/src/vm/mod.rs +++ b/analysis/src/vm/mod.rs @@ -7,7 +7,7 @@ use number::FieldElement; pub mod batcher; pub mod inference; -pub fn analyze( +pub(crate) fn analyze( file: AnalysisASMFile, monitor: &mut DiffMonitor, ) -> Result, Vec> { diff --git a/ast/src/parsed/mod.rs b/ast/src/parsed/mod.rs index 26657af85..70ac2f4bf 100644 --- a/ast/src/parsed/mod.rs +++ b/ast/src/parsed/mod.rs @@ -9,7 +9,7 @@ use std::ops; use number::{DegreeType, FieldElement}; -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Clone)] pub struct PILFile(pub Vec>); #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] diff --git a/compiler/benches/executor_benchmark.rs b/compiler/benches/executor_benchmark.rs index c1d33d1fd..20fb1aa8c 100644 --- a/compiler/benches/executor_benchmark.rs +++ b/compiler/benches/executor_benchmark.rs @@ -1,8 +1,8 @@ use ::compiler::inputs_to_query_callback; +use ::compiler::pipeline::Pipeline; use ast::analyzed::Analyzed; use criterion::{criterion_group, criterion_main, Criterion}; -use executor::constant_evaluator; use mktemp::Temp; use number::{FieldElement, GoldilocksField}; use riscv::{compile_rust_crate_to_riscv_asm, compiler}; @@ -11,23 +11,8 @@ use riscv::CoProcessors; type T = GoldilocksField; -fn get_pil() -> Analyzed { - let tmp_dir = Temp::new_dir().unwrap(); - let riscv_asm_files = - compile_rust_crate_to_riscv_asm("../riscv/tests/riscv_data/keccak/Cargo.toml", &tmp_dir); - let contents = compiler::compile(riscv_asm_files, &CoProcessors::base(), false); - let parsed = parser::parse_asm::(None, &contents).unwrap(); - let resolved = importer::resolve(None, parsed).unwrap(); - let analyzed = analysis::convert_asm_to_pil(resolved).unwrap(); - let graph = airgen::compile(analyzed); - let pil = linker::link(graph).unwrap(); - let analyzed = pil_analyzer::analyze_string(&format!("{pil}")); - pilopt::optimize(analyzed) -} - -fn run_witgen(analyzed: &Analyzed, input: Vec) { - let query_callback = inputs_to_query_callback(input); - let constants = constant_evaluator::generate(analyzed); +fn run_witgen(analyzed: &Analyzed, constants: Vec<(String, Vec)>) { + let query_callback = inputs_to_query_callback(vec![]); executor::witgen::WitnessGenerator::new(analyzed, &constants, query_callback).generate(); } @@ -35,8 +20,23 @@ fn criterion_benchmark(c: &mut Criterion) { let mut group = c.benchmark_group("keccak-executor-benchmark"); group.sample_size(10); - let keccak_pil = get_pil(); - group.bench_function("keccak", |b| b.iter(|| run_witgen(&keccak_pil, vec![]))); + let tmp_dir = Temp::new_dir().unwrap(); + let riscv_asm_files = + compile_rust_crate_to_riscv_asm("../riscv/tests/riscv_data/keccak/Cargo.toml", &tmp_dir); + let contents = compiler::compile(riscv_asm_files, &CoProcessors::base(), false); + let pil_with_constants = Pipeline::::default() + .from_asm_string(contents, None) + .optimized_pil_with_constants() + .unwrap(); + + group.bench_function("keccak", |b| { + b.iter(|| { + run_witgen( + &pil_with_constants.pil, + pil_with_constants.constants.clone(), + ) + }) + }); group.finish(); } diff --git a/compiler/build.rs b/compiler/build.rs index ee644b405..c3430a155 100644 --- a/compiler/build.rs +++ b/compiler/build.rs @@ -36,7 +36,7 @@ fn build_book_tests(kind: &str) { r#" #[test] fn {test_name}() {{ - run_book_test("book/{relative_name}"); + run_book_test("{kind}/book/{relative_name}"); }} "#, ) diff --git a/compiler/src/lib.rs b/compiler/src/lib.rs index 39a81e540..b3cfab519 100644 --- a/compiler/src/lib.rs +++ b/compiler/src/lib.rs @@ -2,414 +2,17 @@ #![deny(clippy::print_stdout)] -use std::ffi::OsStr; -use std::fs; -use std::path::Path; -use std::path::PathBuf; -use std::time::Instant; - -use analysis::analyze; -use analysis::convert_analyzed_to_pil_constraints; -use ast::analyzed::Analyzed; -use ast::DiffMonitor; - +pub mod pipeline; +pub mod test_util; pub mod util; -mod verify; +pub mod verify; -use ast::asm_analysis::AnalysisASMFile; pub use backend::{BackendType, Proof}; use executor::witgen::QueryCallback; use itertools::Itertools; -pub use verify::{ - verify, verify_asm_string, write_commits_to_fs, write_constants_to_fs, write_constraints_to_fs, -}; -use ast::parsed::PILFile; -use executor::constant_evaluator; use number::FieldElement; -pub fn no_callback() -> Option Option> { - None -} - -/// Compiles a .pil or .asm file and runs witness generation. -/// If the file ends in .asm, converts it to .pil first. -/// Returns the compilation result if any compilation took place. -pub fn compile_pil_or_asm( - file_name: &str, - inputs: Vec, - output_dir: &Path, - force_overwrite: bool, - prove_with: Option, - external_witness_values: Vec<(&str, Vec)>, -) -> Result>, Vec> { - if file_name.ends_with(".asm") { - compile_asm( - file_name, - inputs, - output_dir, - force_overwrite, - prove_with, - external_witness_values, - ) - } else { - Ok(Some(compile_pil( - Path::new(file_name), - output_dir, - inputs_to_query_callback(inputs), - prove_with, - external_witness_values, - ))) - } -} - -pub fn analyze_pil(pil_file: &Path) -> Analyzed { - pil_analyzer::analyze(pil_file) -} - -/// Compiles a .pil file to its json form and also tries to generate -/// constants and committed polynomials. -/// @returns a compilation result, containing witness and fixed columns -/// if they could be successfully generated. -pub fn compile_pil>( - pil_file: &Path, - output_dir: &Path, - query_callback: Q, - prove_with: Option, - external_witness_values: Vec<(&str, Vec)>, -) -> CompilationResult { - compile( - pil_analyzer::analyze(pil_file), - pil_file.file_name().unwrap(), - output_dir, - query_callback, - prove_with, - external_witness_values, - ) -} - -/// Compiles a given PIL and tries to generate fixed and witness columns. -/// @returns a compilation result, containing witness and fixed columns -pub fn compile_pil_ast>( - pil: &PILFile, - file_name: &OsStr, - output_dir: &Path, - query_callback: Q, - prove_with: Option, - external_witness_values: Vec<(&str, Vec)>, -) -> CompilationResult { - // TODO exporting this to string as a hack because the parser - // is tied into the analyzer due to imports. - compile( - pil_analyzer::analyze_string(&format!("{pil}")), - file_name, - output_dir, - query_callback, - prove_with, - external_witness_values, - ) -} - -/// Compiles a .asm file, outputs the PIL on stdout and tries to generate -/// fixed and witness columns. -/// @returns a compilation result if any compilation was done. -pub fn compile_asm( - file_name: &str, - inputs: Vec, - output_dir: &Path, - force_overwrite: bool, - prove_with: Option, - external_witness_values: Vec<(&str, Vec)>, -) -> Result>, Vec> { - let contents = fs::read_to_string(file_name).unwrap(); - Ok(compile_asm_string( - file_name, - &contents, - inputs, - None, - output_dir, - force_overwrite, - prove_with, - external_witness_values, - )? - .1) -} - -#[allow(clippy::print_stderr)] -pub fn compile_asm_string_to_analyzed_ast( - file_name: &str, - contents: &str, - monitor: Option<&mut DiffMonitor>, -) -> Result, Vec> { - let parsed = parser::parse_asm(Some(file_name), contents).unwrap_or_else(|err| { - eprintln!("Error parsing .asm file:"); - err.output_to_stderr(); - panic!(); - }); - log::debug!("Resolve imports"); - let resolved = - importer::resolve(Some(PathBuf::from(file_name)), parsed).map_err(|e| vec![e])?; - log::debug!("Run analysis"); - let mut default_monitor = DiffMonitor::default(); - let monitor = monitor.unwrap_or(&mut default_monitor); - let analyzed = analyze(resolved, monitor)?; - log::debug!("Analysis done"); - log::trace!("{analyzed}"); - - Ok(analyzed) -} - -#[allow(clippy::too_many_arguments)] -pub fn convert_analyzed_to_pil( - file_name: &str, - monitor: &mut DiffMonitor, - analyzed: AnalysisASMFile, - inputs: Vec, - output_dir: &Path, - force_overwrite: bool, - prove_with: Option, - external_witness_values: Vec<(&str, Vec)>, -) -> Result<(PathBuf, Option>), Vec> { - let constraints = convert_analyzed_to_pil_constraints(analyzed, monitor); - log::debug!("Run airgen"); - let graph = airgen::compile(constraints); - log::debug!("Airgen done"); - log::trace!("{graph}"); - log::debug!("Run linker"); - let pil = linker::link(graph)?; - log::debug!("Linker done"); - log::trace!("{pil}"); - - let pil_file_name = format!( - "{}.pil", - Path::new(file_name).file_stem().unwrap().to_str().unwrap() - ); - - let pil_file_path = output_dir.join(pil_file_name); - if pil_file_path.exists() && !force_overwrite { - eprintln!( - "Target file {} already exists. Not overwriting.", - pil_file_path.to_str().unwrap() - ); - return Ok((pil_file_path, None)); - } - - fs::write(&pil_file_path, format!("{pil}")).unwrap(); - - let pil_file_name = pil_file_path.file_name().unwrap(); - Ok(( - pil_file_path.clone(), - Some(compile_pil_ast( - &pil, - pil_file_name, - output_dir, - inputs_to_query_callback(inputs), - prove_with, - external_witness_values, - )), - )) -} - -#[allow(clippy::too_many_arguments)] -pub fn convert_analyzed_to_pil_with_callback>( - file_name: &str, - monitor: &mut DiffMonitor, - analyzed: AnalysisASMFile, - query_callback: Q, - output_dir: &Path, - force_overwrite: bool, - prove_with: Option, - external_witness_values: Vec<(&str, Vec)>, -) -> Result<(PathBuf, Option>), Vec> { - let constraints = convert_analyzed_to_pil_constraints(analyzed, monitor); - log::debug!("Run airgen"); - let graph = airgen::compile(constraints); - log::debug!("Airgen done"); - log::trace!("{graph}"); - log::debug!("Run linker"); - let pil = linker::link(graph)?; - log::debug!("Linker done"); - log::trace!("{pil}"); - - let pil_file_name = format!( - "{}.pil", - Path::new(file_name).file_stem().unwrap().to_str().unwrap() - ); - - let pil_file_path = output_dir.join(pil_file_name); - if pil_file_path.exists() && !force_overwrite { - eprintln!( - "Target file {} already exists. Not overwriting.", - pil_file_path.to_str().unwrap() - ); - return Ok((pil_file_path, None)); - } - - fs::write(&pil_file_path, format!("{pil}")).unwrap(); - - let pil_file_name = pil_file_path.file_name().unwrap(); - Ok(( - pil_file_path.clone(), - Some(compile_pil_ast( - &pil, - pil_file_name, - output_dir, - query_callback, - prove_with, - external_witness_values, - )), - )) -} - -pub type AnalyzedASTHook<'a, T> = &'a mut dyn FnMut(&AnalysisASMFile); - -/// Compiles the contents of a .asm file, outputs the PIL on stdout and tries to generate -/// fixed and witness columns. -/// -/// Returns the relative pil file name and the compilation result if any compilation was done. -#[allow(clippy::too_many_arguments)] -pub fn compile_asm_string( - file_name: &str, - contents: &str, - inputs: Vec, - analyzed_hook: Option>, - output_dir: &Path, - force_overwrite: bool, - prove_with: Option, - external_witness_values: Vec<(&str, Vec)>, -) -> Result<(PathBuf, Option>), Vec> { - let mut monitor = DiffMonitor::default(); - let analyzed = compile_asm_string_to_analyzed_ast(file_name, contents, Some(&mut monitor))?; - if let Some(hook) = analyzed_hook { - hook(&analyzed); - }; - convert_analyzed_to_pil( - file_name, - &mut monitor, - analyzed, - inputs, - output_dir, - force_overwrite, - prove_with, - external_witness_values, - ) -} - -#[allow(clippy::too_many_arguments)] -pub fn compile_asm_string_with_callback>( - file_name: &str, - contents: &str, - query_callback: Q, - analyzed_hook: Option>, - output_dir: &Path, - force_overwrite: bool, - prove_with: Option, - external_witness_values: Vec<(&str, Vec)>, -) -> Result<(PathBuf, Option>), Vec> { - let mut monitor = DiffMonitor::default(); - let analyzed = compile_asm_string_to_analyzed_ast(file_name, contents, Some(&mut monitor))?; - if let Some(hook) = analyzed_hook { - hook(&analyzed); - }; - convert_analyzed_to_pil_with_callback( - file_name, - &mut monitor, - analyzed, - query_callback, - output_dir, - force_overwrite, - prove_with, - external_witness_values, - ) -} - -pub struct CompilationResult { - /// Constant columns, potentially incomplete (if success is false) - pub constants: Vec<(String, Vec)>, - /// Witness columns, potentially None (if success is false) - pub witness: Option)>>, - /// Proof, potentially None (if success is false) - pub proof: Option, - /// Serialized low level constraints, potentially None (if success is false) - pub constraints_serialization: Option, -} - -/// Optimizes a given pil and tries to generate constants and committed polynomials. -/// @returns a compilation result, containing witness and fixed columns, if successful. -fn compile>( - analyzed: Analyzed, - file_name: &OsStr, - output_dir: &Path, - query_callback: Q, - prove_with: Option, - external_witness_values: Vec<(&str, Vec)>, -) -> CompilationResult { - log::info!("Optimizing pil..."); - let analyzed = pilopt::optimize(analyzed); - let optimized_pil_file_name = output_dir.join(format!( - "{}_opt.pil", - Path::new(file_name).file_stem().unwrap().to_str().unwrap() - )); - fs::write(optimized_pil_file_name.clone(), format!("{analyzed}")).unwrap(); - log::info!("Wrote {}.", optimized_pil_file_name.to_str().unwrap()); - let start = Instant::now(); - log::info!("Evaluating fixed columns..."); - let constants = constant_evaluator::generate(&analyzed); - log::info!("Took {}", start.elapsed().as_secs_f32()); - - let witness = (analyzed.constant_count() == constants.len()).then(|| { - log::info!("Deducing witness columns..."); - let commits = - executor::witgen::WitnessGenerator::new(&analyzed, &constants, query_callback) - .with_external_witness_values(external_witness_values) - .generate(); - - commits - .into_iter() - .map(|(name, c)| (name.to_string(), c)) - .collect::>() - }); - - let constants = constants - .into_iter() - .map(|(name, c)| (name.to_string(), c)) - .collect::>(); - - // Even if we don't have all constants and witnesses, some backends will - // still output the constraint serialization. - let (proof, constraints_serialization) = if let Some(backend) = prove_with { - let factory = backend.factory::(); - let backend = factory.create(analyzed.degree()); - - backend.prove( - &analyzed, - &constants, - witness.as_deref().unwrap_or_default(), - None, - ) - } else { - (None, None) - }; - - let constants = constants - .into_iter() - .map(|(name, c)| (name.to_owned(), c)) - .collect(); - - let witness = witness.map(|v| { - v.into_iter() - .map(|(name, c)| (name.to_owned(), c)) - .collect() - }); - - CompilationResult { - constants, - witness, - proof, - constraints_serialization, - } -} - #[allow(clippy::print_stdout)] pub fn inputs_to_query_callback(inputs: Vec) -> impl QueryCallback { // TODO: Pass bootloader inputs into this function diff --git a/compiler/src/pipeline.rs b/compiler/src/pipeline.rs new file mode 100644 index 000000000..ccd2b7480 --- /dev/null +++ b/compiler/src/pipeline.rs @@ -0,0 +1,512 @@ +use std::{ + fmt::Display, + fs, + path::{Path, PathBuf}, + time::Instant, +}; + +use analysis::convert_analyzed_to_pil_constraints; +use ast::{ + analyzed::Analyzed, + asm_analysis::AnalysisASMFile, + object::PILGraph, + parsed::{asm::ASMProgram, PILFile}, + DiffMonitor, +}; +use backend::{BackendType, Proof}; +use executor::{constant_evaluator, witgen::QueryCallback}; +use log::Level; +use mktemp::Temp; +use number::FieldElement; + +use crate::verify::{write_commits_to_fs, write_constants_to_fs, write_constraints_to_fs}; + +pub struct GeneratedWitness { + pub pil: Analyzed, + pub constants: Vec<(String, Vec)>, + pub witness: Option)>>, +} + +pub struct PilWithConstants { + pub pil: Analyzed, + pub constants: Vec<(String, Vec)>, +} + +pub struct ProofResult { + /// Constant columns, potentially incomplete (if success is false) + pub constants: Vec<(String, Vec)>, + /// Witness columns, potentially None (if success is false) + pub witness: Option)>>, + /// Proof, potentially None (if success is false) + pub proof: Option, + /// Serialized low level constraints, potentially None (if success is false) + pub constraints_serialization: Option, +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +enum Stage { + AsmFile, + AsmString, + Parsed, + Resolved, + AnalyzedAsm, + PilConstraints, + Graph, + Linked, + PilFile, + PilString, + AnalyzedPil, + OptimizedPil, + PilWithConstants, + GeneratedWitness, + Proof, +} + +enum Artifact { + /// The path to the .asm file. + AsmFile(PathBuf), + /// The contents of the .asm file, with an optional Path (for error messages). + AsmString(Option, String), + /// The parsed .asm file, with an optional Path (for error messages). + Parsed(Option, ASMProgram), + /// The resolved .asm file. + Resolved(ASMProgram), + /// The analyzed .asm file. + AnalyzedAsm(AnalysisASMFile), + /// The pil constraints. + PilConstraints(AnalysisASMFile), + /// The airgen graph. + Graph(PILGraph), + /// The linked pil. + Linked(PILFile), + /// The path to the .pil file. + PilFile(PathBuf), + /// The contents of the .pil file. + PilString(String), + /// The analyzed .pil file. + AnaylyzedPil(Analyzed), + /// The optimized .pil file. + OptimzedPil(Analyzed), + /// The optimized .pil file with constants. + PilWithConstants(PilWithConstants), + /// The generated witness. + GeneratedWitness(GeneratedWitness), + /// The proof (if successful) + Proof(ProofResult), +} + +pub struct Pipeline { + /// The current artifact. It is never None in practice, making it an Option is + /// only necessary so that we can take ownership of it in advance(). + artifact: Option>, + /// The diff monitor is used to track changes between pipeline stages. + diff_monitor: DiffMonitor, + /// Output directory for intermediate files. If None, no files are written. + output_dir: Option, + /// The name of the pipeline. Used to name output files. + name: Option, + /// Whether to overwrite existing files. If false, an error is returned if a file + /// already exists. + force_overwrite: bool, + /// The log level to use for this pipeline. + log_level: Level, + // The output directory if `Pipeline::with_tmp_output` was called. + // Note that there is some redundancy with `output_dir`, but the Temp + // object has to live for the lifetime of the pipeline, so we keep it here. + tmp_dir: Option, +} + +impl Default for Pipeline +where + T: FieldElement, +{ + fn default() -> Self { + Pipeline { + artifact: None, + diff_monitor: DiffMonitor::default(), + output_dir: None, + log_level: Level::Debug, + name: None, + force_overwrite: false, + tmp_dir: None, + } + } +} + +/// A Powdr pipeline, going from a PIL or ASM file to a proof. +/// +/// The pipeline steps are: +/// ```mermaid +/// graph TD +/// AsmFile --> AsmString +/// AsmString --> Parsed +/// Parsed --> Resolved +/// Resolved --> AnalyzedAsm +/// AnalyzedAsm --> PilConstraints +/// PilConstraints --> Graph +/// Graph --> Linked +/// Linked --> AnaylyzedPil +/// PilFile --> AnaylyzedPil +/// PilString --> AnaylyzedPil +/// AnaylyzedPil --> OptimzedPil +/// OptimzedPil --> PilWithConstants +/// PilWithConstants --> GeneratedWitness +/// GeneratedWitness --> Proof +/// ``` +/// +/// # Example +/// ```rust +/// use compiler::{pipeline::Pipeline, verify, BackendType, test_util::resolve_test_file}; +/// use std::path::PathBuf; +/// use number::GoldilocksField; +/// +/// let mut pipeline = Pipeline::::default() +/// .from_file(resolve_test_file("pil/fibonacci.pil")); +/// pipeline +/// .generate_witness( +/// compiler::inputs_to_query_callback(vec![]), +/// vec![], +/// ) +/// .unwrap(); +/// pipeline.prove(BackendType::PilStarkCli).unwrap(); +/// ``` +impl Pipeline { + pub fn with_tmp_output(self) -> Self { + let tmp_dir = mktemp::Temp::new_dir().unwrap(); + Pipeline { + output_dir: Some(tmp_dir.to_path_buf()), + tmp_dir: Some(tmp_dir), + ..self + } + } + + pub fn with_output(self, output_dir: PathBuf, force_overwrite: bool) -> Self { + Pipeline { + output_dir: Some(output_dir), + force_overwrite, + ..self + } + } + + pub fn from_file(self, asm_file: PathBuf) -> Self { + if asm_file.extension().unwrap() == "asm" { + self.from_asm_file(asm_file) + } else { + self.from_pil_file(asm_file) + } + } + + pub fn from_asm_file(self, asm_file: PathBuf) -> Self { + let name = self.name.or(Some(Self::name_from_path(&asm_file))); + Pipeline { + artifact: Some(Artifact::AsmFile(asm_file)), + name, + ..self + } + } + + pub fn from_asm_string(self, asm_string: String, path: Option) -> Self { + let name = self.name.or(path.as_ref().map(|p| Self::name_from_path(p))); + Pipeline { + artifact: Some(Artifact::AsmString(path, asm_string)), + name, + ..self + } + } + + pub fn from_pil_file(self, pil_file: PathBuf) -> Self { + let name = self.name.or(Some(Self::name_from_path(&pil_file))); + Pipeline { + artifact: Some(Artifact::PilFile(pil_file)), + name, + ..self + } + } + + pub fn from_pil_string(self, pil_string: String) -> Self { + Pipeline { + artifact: Some(Artifact::PilString(pil_string)), + ..self + } + } + + fn name_from_path(path: &Path) -> String { + path.file_stem().unwrap().to_str().unwrap().to_string() + } + + fn log(&self, msg: &str) { + log::log!(self.log_level, "{}", msg); + } + + #[allow(clippy::print_stderr)] + fn advance(&mut self) -> Result<(), Vec> { + let artifact = std::mem::take(&mut self.artifact).unwrap(); + self.artifact = Some(match artifact { + Artifact::AsmFile(path) => Artifact::AsmString( + Some(path.clone()), + fs::read_to_string(&path).map_err(|e| { + vec![format!("Error reading .asm file: {}\n{e}", path.display())] + })?, + ), + Artifact::AsmString(path, asm_string) => { + let parsed_asm = parser::parse_asm(None, &asm_string).unwrap_or_else(|err| { + match path.as_ref() { + Some(path) => eprintln!("Error parsing .asm file: {}", path.display()), + None => eprintln!("Error parsing .asm file:"), + } + err.output_to_stderr(); + panic!(); + }); + self.diff_monitor.push(&parsed_asm); + Artifact::Parsed(path, parsed_asm) + } + Artifact::Parsed(path, parsed) => { + self.log("Resolve imports"); + let resolved = importer::resolve(path, parsed).map_err(|e| vec![e])?; + self.diff_monitor.push(&resolved); + Artifact::Resolved(resolved) + } + Artifact::Resolved(resolved) => { + self.log("Run analysis"); + self.log("Analysis done"); + let analyzed_asm = analysis::analyze(resolved, &mut self.diff_monitor)?; + log::trace!("{analyzed_asm}"); + Artifact::AnalyzedAsm(analyzed_asm) + } + Artifact::AnalyzedAsm(analyzed_asm) => Artifact::PilConstraints( + convert_analyzed_to_pil_constraints(analyzed_asm, &mut self.diff_monitor), + ), + Artifact::PilConstraints(analyzed_asm) => { + self.log("Run airgen"); + let graph = airgen::compile(analyzed_asm); + self.diff_monitor.push(&graph); + self.log("Airgen done"); + log::trace!("{graph}"); + Artifact::Graph(graph) + } + Artifact::Graph(graph) => { + self.log("Run linker"); + self.log("Linker done"); + let linked = linker::link(graph)?; + self.diff_monitor.push(&linked); + log::trace!("{linked}"); + self.maybe_write_pil(&linked, "")?; + Artifact::Linked(linked) + } + Artifact::Linked(linked) => { + Artifact::AnaylyzedPil(pil_analyzer::analyze_string(&format!("{linked}"))) + } + Artifact::PilFile(pil_file) => Artifact::AnaylyzedPil(pil_analyzer::analyze(&pil_file)), + Artifact::PilString(pil_string) => { + Artifact::AnaylyzedPil(pil_analyzer::analyze_string(&pil_string)) + } + Artifact::AnaylyzedPil(analyzed_pil) => { + self.log("Optimizing pil..."); + let optimized = pilopt::optimize(analyzed_pil); + self.maybe_write_pil(&optimized, "_opt")?; + Artifact::OptimzedPil(optimized) + } + Artifact::OptimzedPil(pil) => { + self.log("Evaluating fixed columns..."); + let start = Instant::now(); + let constants = constant_evaluator::generate(&pil); + let constants = constants + .into_iter() + .map(|(k, v)| (k.to_string(), v)) + .collect(); + self.log(&format!("Took {}", start.elapsed().as_secs_f32())); + Artifact::PilWithConstants(PilWithConstants { pil, constants }) + } + Artifact::PilWithConstants(_) => { + return Err(vec![ + "Witness generation requires arguments, call generate_witness() instead!" + .into(), + ]) + } + Artifact::GeneratedWitness(_) => { + return Err(vec![ + "Proof generation requires arguments, call prove() instead!".into(), + ]) + } + Artifact::Proof(_) => panic!("Last pipeline step!"), + }); + Ok(()) + } + + fn maybe_write_pil(&self, content: &C, suffix: &str) -> Result<(), Vec> { + if let Some(output_dir) = &self.output_dir { + let name = self + .name + .as_ref() + .expect("name must be set if output_dir is set"); + let output_file = output_dir.join(format!("{name}{suffix}.pil")); + if !output_file.exists() || self.force_overwrite { + fs::write(&output_file, format!("{content}")).map_err(|e| { + vec![format!( + "Error writing {}: {e}", + output_file.to_str().unwrap() + )] + })?; + self.log(&format!("Wrote {}.", output_file.to_str().unwrap())); + } else { + return Err(vec![format!( + "{} already exists! Use --force to overwrite.", + output_file.to_str().unwrap() + )]); + } + } + Ok(()) + } + + fn stage(&self) -> Stage { + match self.artifact.as_ref().unwrap() { + Artifact::AsmFile(_) => Stage::AsmFile, + Artifact::AsmString(_, _) => Stage::AsmString, + Artifact::Parsed(_, _) => Stage::Parsed, + Artifact::Resolved(_) => Stage::Resolved, + Artifact::AnalyzedAsm(_) => Stage::AnalyzedAsm, + Artifact::PilConstraints(_) => Stage::PilConstraints, + Artifact::Graph(_) => Stage::Graph, + Artifact::Linked(_) => Stage::Linked, + Artifact::PilFile(_) => Stage::PilFile, + Artifact::PilString(_) => Stage::PilString, + Artifact::AnaylyzedPil(_) => Stage::AnalyzedPil, + Artifact::OptimzedPil(_) => Stage::OptimizedPil, + Artifact::PilWithConstants(_) => Stage::PilWithConstants, + Artifact::GeneratedWitness(_) => Stage::GeneratedWitness, + Artifact::Proof(_) => Stage::Proof, + } + } + + fn advance_to(&mut self, target_stage: Stage) -> Result<(), Vec> { + while self.stage() != target_stage { + self.advance()?; + } + Ok(()) + } + + pub fn generate_witness>( + &mut self, + query_callback: Q, + external_witness_values: Vec<(&str, Vec)>, + ) -> Result<(), Vec> { + self.advance_to(Stage::PilWithConstants)?; + let Artifact::PilWithConstants(PilWithConstants { pil, constants }) = + std::mem::take(&mut self.artifact).unwrap() + else { + panic!() + }; + let witness = (pil.constant_count() == constants.len()).then(|| { + self.log("Deducing witness columns..."); + let start = Instant::now(); + let witness = executor::witgen::WitnessGenerator::new(&pil, &constants, query_callback) + .with_external_witness_values(external_witness_values) + .generate(); + + self.log(&format!("Took {}", start.elapsed().as_secs_f32())); + witness + .into_iter() + .map(|(name, c)| (name.to_string(), c)) + .collect::>() + }); + self.artifact = Some(Artifact::GeneratedWitness(GeneratedWitness { + pil, + constants, + witness, + })); + Ok(()) + } + + pub fn prove(&mut self, backend: BackendType) -> Result<(), Vec> { + self.advance_to(Stage::GeneratedWitness)?; + let Artifact::GeneratedWitness(GeneratedWitness { + pil: analyzed, + constants, + witness, + }) = std::mem::take(&mut self.artifact).unwrap() + else { + panic!() + }; + let factory = backend.factory::(); + let backend = factory.create(analyzed.degree()); + + // Even if we don't have all constants and witnesses, some backends will + // still output the constraint serialization. + let (proof, constraints_serialization) = backend.prove( + &analyzed, + &constants, + witness.as_deref().unwrap_or_default(), + None, + ); + + if let Some(output_dir) = &self.output_dir { + write_constants_to_fs(&constants, output_dir); + if let Some(witness) = &witness { + write_commits_to_fs(witness, output_dir); + } + if let Some(constraints_serialization) = &constraints_serialization { + write_constraints_to_fs(constraints_serialization, output_dir); + } + }; + + self.artifact = Some(Artifact::Proof(ProofResult { + constants, + witness, + proof, + constraints_serialization, + })); + Ok(()) + } + + pub fn analyzed_asm(mut self) -> Result, Vec> { + self.advance_to(Stage::AnalyzedAsm)?; + let Artifact::AnalyzedAsm(analyzed_asm) = self.artifact.unwrap() else { + panic!() + }; + Ok(analyzed_asm) + } + + pub fn analyzed_asm_ref(&mut self) -> Result<&AnalysisASMFile, Vec> { + self.advance_to(Stage::AnalyzedAsm)?; + match self.artifact.as_ref().unwrap() { + Artifact::AnalyzedAsm(analyzed_asm) => Ok(analyzed_asm), + _ => panic!(), + } + } + + pub fn optimized_pil(mut self) -> Result, Vec> { + self.advance_to(Stage::OptimizedPil)?; + let Artifact::OptimzedPil(optimized_pil) = self.artifact.unwrap() else { + panic!() + }; + Ok(optimized_pil) + } + + pub fn optimized_pil_with_constants(mut self) -> Result, Vec> { + self.advance_to(Stage::PilWithConstants)?; + let Artifact::PilWithConstants(pil_with_constants) = self.artifact.unwrap() else { + panic!() + }; + Ok(pil_with_constants) + } + + pub fn generated_witness(mut self) -> Result, Vec> { + self.advance_to(Stage::GeneratedWitness)?; + let Artifact::GeneratedWitness(generated_witness) = self.artifact.unwrap() else { + panic!() + }; + Ok(generated_witness) + } + + pub fn proof(mut self) -> Result, Vec> { + self.advance_to(Stage::Proof)?; + let Artifact::Proof(proof) = self.artifact.unwrap() else { + panic!() + }; + Ok(proof) + } + + pub fn tmp_dir(&self) -> &Path { + self.tmp_dir.as_ref().unwrap() + } +} diff --git a/compiler/src/test_util.rs b/compiler/src/test_util.rs new file mode 100644 index 000000000..24bf8551c --- /dev/null +++ b/compiler/src/test_util.rs @@ -0,0 +1,75 @@ +use backend::BackendType; +use number::FieldElement; +use number::{Bn254Field, GoldilocksField}; +use std::path::PathBuf; + +use crate::inputs_to_query_callback; +use crate::pipeline::Pipeline; +use crate::verify::verify; + +pub fn resolve_test_file(file_name: &str) -> PathBuf { + PathBuf::from(format!( + "{}/../test_data/{file_name}", + env!("CARGO_MANIFEST_DIR") + )) +} + +pub fn verify_test_file( + file_name: &str, + inputs: Vec, + external_witness_values: Vec<(&str, Vec)>, +) { + let pipeline = Pipeline::default().from_file(resolve_test_file(file_name)); + verify_pipeline(pipeline, inputs, external_witness_values) +} + +pub fn verify_asm_string( + file_name: &str, + contents: &str, + inputs: Vec, + external_witness_values: Vec<(&str, Vec)>, +) { + let pipeline = + Pipeline::default().from_asm_string(contents.to_string(), Some(PathBuf::from(file_name))); + verify_pipeline(pipeline, inputs, external_witness_values) +} + +pub fn verify_pipeline( + pipeline: Pipeline, + inputs: Vec, + external_witness_values: Vec<(&str, Vec)>, +) { + let mut pipeline = pipeline.with_tmp_output(); + pipeline + .generate_witness(inputs_to_query_callback(inputs), external_witness_values) + .unwrap(); + pipeline.prove(BackendType::PilStarkCli).unwrap(); + + verify(pipeline.tmp_dir()); +} + +pub fn gen_estark_proof(file_name: &str, inputs: Vec) { + let file_name = format!("{}/../test_data/{file_name}", env!("CARGO_MANIFEST_DIR")); + let mut pipeline = Pipeline::default() + .with_tmp_output() + .from_file(PathBuf::from(file_name)); + pipeline + .generate_witness(inputs_to_query_callback(inputs), vec![]) + .unwrap(); + pipeline.prove(backend::BackendType::EStark).unwrap(); +} + +#[cfg(feature = "halo2")] +pub fn gen_halo2_proof(file_name: &str, inputs: Vec) { + let file_name = format!("{}/../test_data/{file_name}", env!("CARGO_MANIFEST_DIR")); + let mut pipeline = Pipeline::default() + .with_tmp_output() + .from_file(PathBuf::from(file_name)); + pipeline + .generate_witness(inputs_to_query_callback(inputs), vec![]) + .unwrap(); + pipeline.prove(backend::BackendType::Halo2).unwrap(); +} + +#[cfg(not(feature = "halo2"))] +pub fn gen_halo2_proof(_file_name: &str, _inputs: Vec) {} diff --git a/compiler/src/verify.rs b/compiler/src/verify.rs index 218f1e632..39ff92b07 100644 --- a/compiler/src/verify.rs +++ b/compiler/src/verify.rs @@ -1,4 +1,3 @@ -use backend::BackendType; use number::write_polys_file; use number::FieldElement; use std::{ @@ -8,35 +7,6 @@ use std::{ process::Command, }; -use crate::compile_asm_string; - -pub fn verify_asm_string( - file_name: &str, - contents: &str, - inputs: Vec, - external_witness_values: Vec<(&str, Vec)>, -) { - let temp_dir = mktemp::Temp::new_dir().unwrap(); - let (_, result) = compile_asm_string( - file_name, - contents, - inputs, - None, - &temp_dir, - true, - Some(BackendType::PilStarkCli), - external_witness_values, - ) - .unwrap(); - - let result = result.unwrap(); - write_constants_to_fs(&result.constants, &temp_dir); - write_commits_to_fs(&result.witness.unwrap(), &temp_dir); - write_constraints_to_fs(&result.constraints_serialization.unwrap(), &temp_dir); - - verify(&temp_dir); -} - pub fn write_constants_to_fs(constants: &[(String, Vec)], output_dir: &Path) { let to_write = output_dir.join("constants.bin"); write_polys_file( diff --git a/compiler/tests/asm.rs b/compiler/tests/asm.rs index 0f00fc483..e5e1e350f 100644 --- a/compiler/tests/asm.rs +++ b/compiler/tests/asm.rs @@ -1,70 +1,18 @@ -use compiler::verify_asm_string; -use number::{Bn254Field, FieldElement, GoldilocksField}; -use std::fs; +use compiler::test_util::{gen_estark_proof, gen_halo2_proof, verify_test_file}; +use number::{FieldElement, GoldilocksField}; use test_log::test; fn verify_asm(file_name: &str, inputs: Vec) { - verify_asm_with_external_witness(file_name, inputs, vec![]); + verify_test_file(file_name, inputs, vec![]); } -fn verify_asm_with_external_witness( - file_name: &str, - inputs: Vec, - external_witness_values: Vec<(&str, Vec)>, -) { - let file_name = format!( - "{}/../test_data/asm/{file_name}", - env!("CARGO_MANIFEST_DIR") - ); - - let contents = fs::read_to_string(&file_name).unwrap(); - - verify_asm_string(&file_name, &contents, inputs, external_witness_values); -} - -fn gen_estark_proof(file_name: &str, inputs: Vec) { - compiler::compile_pil_or_asm( - format!( - "{}/../test_data/asm/{file_name}", - env!("CARGO_MANIFEST_DIR") - ) - .as_str(), - inputs, - &mktemp::Temp::new_dir().unwrap(), - true, - Some(backend::BackendType::EStark), - vec![], - ) - .unwrap(); -} - -#[cfg(feature = "halo2")] -fn gen_halo2_proof(file_name: &str, inputs: Vec) { - compiler::compile_pil_or_asm( - format!( - "{}/../test_data/asm/{file_name}", - env!("CARGO_MANIFEST_DIR") - ) - .as_str(), - inputs, - &mktemp::Temp::new_dir().unwrap(), - true, - Some(backend::BackendType::Halo2), - vec![], - ) - .unwrap(); -} - -#[cfg(not(feature = "halo2"))] -fn gen_halo2_proof(_file_name: &str, _inputs: Vec) {} - fn slice_to_vec(arr: &[i32]) -> Vec { arr.iter().cloned().map(|x| x.into()).collect() } #[test] fn simple_sum_asm() { - let f = "simple_sum.asm"; + let f = "asm/simple_sum.asm"; let i = [16, 4, 1, 2, 8, 5]; verify_asm::(f, slice_to_vec(&i)); gen_halo2_proof(f, slice_to_vec(&i)); @@ -73,7 +21,7 @@ fn simple_sum_asm() { #[test] fn secondary_block_machine_add2() { - let f = "secondary_block_machine_add2.asm"; + let f = "asm/secondary_block_machine_add2.asm"; verify_asm::(f, vec![]); gen_halo2_proof(f, vec![]); gen_estark_proof(f, vec![]); @@ -81,7 +29,7 @@ fn secondary_block_machine_add2() { #[test] fn mem_write_once() { - let f = "mem_write_once.asm"; + let f = "asm/mem_write_once.asm"; verify_asm::(f, vec![]); gen_halo2_proof(f, vec![]); gen_estark_proof(f, vec![]); @@ -89,17 +37,17 @@ fn mem_write_once() { #[test] fn mem_write_once_external_write() { - let f = "mem_write_once_external_write.asm"; + let f = "asm/mem_write_once_external_write.asm"; let mut mem = vec![GoldilocksField::from(0); 256]; mem[17] = GoldilocksField::from(42); mem[62] = GoldilocksField::from(123); mem[255] = GoldilocksField::from(-1); - verify_asm_with_external_witness::(f, vec![], vec![("main.v", mem)]); + verify_test_file::(f, vec![], vec![("main.v", mem)]); } #[test] fn block_machine_cache_miss() { - let f = "block_machine_cache_miss.asm"; + let f = "asm/block_machine_cache_miss.asm"; verify_asm::(f, vec![]); gen_halo2_proof(f, vec![]); gen_estark_proof(f, vec![]); @@ -107,7 +55,7 @@ fn block_machine_cache_miss() { #[test] fn palindrome() { - let f = "palindrome.asm"; + let f = "asm/palindrome.asm"; let i = [7, 1, 7, 3, 9, 3, 7, 1]; verify_asm::(f, slice_to_vec(&i)); gen_halo2_proof(f, slice_to_vec(&i)); @@ -119,7 +67,7 @@ fn palindrome() { #[test] fn single_function_vm() { - let f = "single_function_vm.asm"; + let f = "asm/single_function_vm.asm"; let i = []; verify_asm::(f, slice_to_vec(&i)); gen_halo2_proof(f, slice_to_vec(&i)); @@ -128,7 +76,7 @@ fn single_function_vm() { #[test] fn empty() { - let f = "empty.asm"; + let f = "asm/empty.asm"; let i = []; verify_asm::(f, slice_to_vec(&i)); gen_halo2_proof(f, slice_to_vec(&i)); @@ -137,7 +85,7 @@ fn empty() { #[test] fn single_operation() { - let f = "single_operation.asm"; + let f = "asm/single_operation.asm"; let i = []; verify_asm::(f, slice_to_vec(&i)); gen_halo2_proof(f, slice_to_vec(&i)); @@ -146,7 +94,7 @@ fn single_operation() { #[test] fn empty_vm() { - let f = "empty_vm.asm"; + let f = "asm/empty_vm.asm"; let i = []; verify_asm::(f, slice_to_vec(&i)); gen_halo2_proof(f, slice_to_vec(&i)); @@ -155,7 +103,7 @@ fn empty_vm() { #[test] fn vm_to_block_unique_interface() { - let f = "vm_to_block_unique_interface.asm"; + let f = "asm/vm_to_block_unique_interface.asm"; let i = []; verify_asm::(f, slice_to_vec(&i)); gen_halo2_proof(f, slice_to_vec(&i)); @@ -167,7 +115,7 @@ fn vm_to_block_unique_interface() { #[test] fn vm_to_block_to_block() { - let f = "vm_to_block_to_block.asm"; + let f = "asm/vm_to_block_to_block.asm"; let i = []; verify_asm::(f, slice_to_vec(&i)); gen_halo2_proof(f, slice_to_vec(&i)); @@ -175,7 +123,7 @@ fn vm_to_block_to_block() { #[test] fn block_to_block() { - let f = "block_to_block.asm"; + let f = "asm/block_to_block.asm"; let i = []; verify_asm::(f, slice_to_vec(&i)); gen_halo2_proof(f, slice_to_vec(&i)); @@ -184,7 +132,7 @@ fn block_to_block() { #[test] fn vm_to_block_multiple_interfaces() { - let f = "vm_to_block_multiple_interfaces.asm"; + let f = "asm/vm_to_block_multiple_interfaces.asm"; let i = []; verify_asm::(f, slice_to_vec(&i)); gen_halo2_proof(f, slice_to_vec(&i)); @@ -193,7 +141,7 @@ fn vm_to_block_multiple_interfaces() { #[test] fn vm_to_vm() { - let f = "vm_to_vm.asm"; + let f = "asm/vm_to_vm.asm"; let i = []; verify_asm::(f, slice_to_vec(&i)); gen_halo2_proof(f, slice_to_vec(&i)); @@ -202,7 +150,7 @@ fn vm_to_vm() { #[test] fn vm_to_vm_dynamic_trace_length() { - let f = "vm_to_vm_dynamic_trace_length.asm"; + let f = "asm/vm_to_vm_dynamic_trace_length.asm"; let i = []; verify_asm::(f, slice_to_vec(&i)); gen_halo2_proof(f, slice_to_vec(&i)); @@ -211,7 +159,7 @@ fn vm_to_vm_dynamic_trace_length() { #[test] fn vm_to_vm_to_block() { - let f = "vm_to_vm_to_block.asm"; + let f = "asm/vm_to_vm_to_block.asm"; let i = []; verify_asm::(f, slice_to_vec(&i)); gen_halo2_proof(f, slice_to_vec(&i)); @@ -220,7 +168,7 @@ fn vm_to_vm_to_block() { #[test] fn vm_to_block_array() { - let f = "vm_to_block_array.asm"; + let f = "asm/vm_to_block_array.asm"; let i = []; verify_asm::(f, slice_to_vec(&i)); gen_halo2_proof(f, slice_to_vec(&i)); @@ -229,7 +177,7 @@ fn vm_to_block_array() { #[test] fn vm_to_vm_to_vm() { - let f = "vm_to_vm_to_vm.asm"; + let f = "asm/vm_to_vm_to_vm.asm"; let i = []; verify_asm::(f, slice_to_vec(&i)); gen_halo2_proof(f, slice_to_vec(&i)); @@ -238,7 +186,7 @@ fn vm_to_vm_to_vm() { #[test] fn test_mem_read_write() { - let f = "mem_read_write.asm"; + let f = "asm/mem_read_write.asm"; verify_asm::(f, Default::default()); gen_halo2_proof(f, Default::default()); gen_estark_proof(f, Default::default()); @@ -246,7 +194,7 @@ fn test_mem_read_write() { #[test] fn test_multi_assign() { - let f = "multi_assign.asm"; + let f = "asm/multi_assign.asm"; let i = [7]; verify_asm::(f, slice_to_vec(&i)); gen_halo2_proof(f, slice_to_vec(&i)); @@ -255,7 +203,7 @@ fn test_multi_assign() { #[test] fn test_multi_return() { - let f = "multi_return.asm"; + let f = "asm/multi_return.asm"; let i = []; verify_asm::(f, slice_to_vec(&i)); gen_halo2_proof(f, slice_to_vec(&i)); @@ -265,7 +213,7 @@ fn test_multi_return() { #[test] #[should_panic = "called `Result::unwrap()` on an `Err` value: [\"Assignment register `Z` is incompatible with `square_and_double(3)`. Try using `<==` with no explicit assignment registers.\", \"Assignment register `Y` is incompatible with `square_and_double(3)`. Try using `<==` with no explicit assignment registers.\"]"] fn test_multi_return_wrong_assignment_registers() { - let f = "multi_return_wrong_assignment_registers.asm"; + let f = "asm/multi_return_wrong_assignment_registers.asm"; let i = []; verify_asm::(f, slice_to_vec(&i)); } @@ -273,14 +221,14 @@ fn test_multi_return_wrong_assignment_registers() { #[test] #[should_panic = "Result::unwrap()` on an `Err` value: [\"Mismatched number of registers for assignment A, B <=Y= square_and_double(3);\"]"] fn test_multi_return_wrong_assignment_register_length() { - let f = "multi_return_wrong_assignment_register_length.asm"; + let f = "asm/multi_return_wrong_assignment_register_length.asm"; let i = []; verify_asm::(f, slice_to_vec(&i)); } #[test] fn test_bit_access() { - let f = "bit_access.asm"; + let f = "asm/bit_access.asm"; let i = [20]; verify_asm::(f, slice_to_vec(&i)); gen_halo2_proof(f, slice_to_vec(&i)); @@ -292,7 +240,7 @@ fn test_bit_access() { #[test] fn test_sqrt() { - let f = "sqrt.asm"; + let f = "asm/sqrt.asm"; verify_asm::(f, Default::default()); gen_halo2_proof(f, Default::default()); gen_estark_proof(f, Default::default()); @@ -300,7 +248,7 @@ fn test_sqrt() { #[test] fn functional_instructions() { - let f = "functional_instructions.asm"; + let f = "asm/functional_instructions.asm"; let i = [20]; verify_asm::(f, slice_to_vec(&i)); gen_halo2_proof(f, slice_to_vec(&i)); @@ -312,7 +260,7 @@ fn functional_instructions() { #[test] fn full_pil_constant() { - let f = "full_pil_constant.asm"; + let f = "asm/full_pil_constant.asm"; verify_asm::(f, Default::default()); gen_halo2_proof(f, Default::default()); gen_estark_proof(f, Default::default()); @@ -320,7 +268,7 @@ fn full_pil_constant() { #[test] fn intermediate() { - let f = "intermediate.asm"; + let f = "asm/intermediate.asm"; verify_asm::(f, Default::default()); gen_halo2_proof(f, Default::default()); gen_estark_proof(f, Default::default()); @@ -328,7 +276,7 @@ fn intermediate() { #[test] fn intermediate_nested() { - let f = "intermediate_nested.asm"; + let f = "asm/intermediate_nested.asm"; verify_asm::(f, Default::default()); gen_halo2_proof(f, Default::default()); gen_estark_proof(f, Default::default()); @@ -343,7 +291,7 @@ mod book { // passing 0 to all tests currently works as they either take no prover input or 0 works let i = [0]; - verify_asm::(file, slice_to_vec(&i)); + verify_asm::(&format!("{file}"), slice_to_vec(&i)); gen_halo2_proof(file, slice_to_vec(&i)); gen_estark_proof(file, slice_to_vec(&i)); } @@ -354,7 +302,7 @@ mod book { #[test] #[should_panic = "Witness generation failed."] fn hello_world_asm_fail() { - let f = "book/hello_world.asm"; + let f = "asm/book/hello_world.asm"; let i = [1]; verify_asm::(f, slice_to_vec(&i)); } diff --git a/compiler/tests/pil.rs b/compiler/tests/pil.rs index 56518d8e5..b91f8ed36 100644 --- a/compiler/tests/pil.rs +++ b/compiler/tests/pil.rs @@ -1,100 +1,32 @@ -use backend::BackendType; -use number::{Bn254Field, GoldilocksField}; +use compiler::test_util::{gen_estark_proof, gen_halo2_proof, verify_test_file}; +use number::GoldilocksField; use std::path::Path; use test_log::test; -type QueryCallbackFn = fn(&str) -> Result, String>; - -pub fn verify_pil(file_name: &str, query_callback: Option) { - verify_pil_with_external_witness(file_name, query_callback, vec![]); +pub fn verify_pil(file_name: &str, inputs: Vec) { + verify_test_file(file_name, inputs, vec![]); } -pub fn verify_pil_with_external_witness( - file_name: &str, - query_callback: Option, - external_witness_values: Vec<(&str, Vec)>, -) { - let input_file = Path::new(&format!( - "{}/../test_data/pil/{file_name}", - env!("CARGO_MANIFEST_DIR") - )) - .canonicalize() - .unwrap(); - - let query_callback = query_callback.unwrap_or(|_| -> _ { unreachable!() }); - - let temp_dir = mktemp::Temp::new_dir().unwrap(); - let result = compiler::compile_pil( - &input_file, - &temp_dir, - query_callback, - Some(BackendType::PilStarkCli), - external_witness_values, - ); - - compiler::write_constants_to_fs(&result.constants, &temp_dir); - compiler::write_commits_to_fs(&result.witness.unwrap(), &temp_dir); - compiler::write_constraints_to_fs(&result.constraints_serialization.unwrap(), &temp_dir); - - compiler::verify(&temp_dir); -} - -fn gen_estark_proof(file_name: &str, inputs: Vec) { - compiler::compile_pil_or_asm( - format!( - "{}/../test_data/pil/{file_name}", - env!("CARGO_MANIFEST_DIR") - ) - .as_str(), - inputs, - &mktemp::Temp::new_dir().unwrap(), - true, - Some(BackendType::EStark), - vec![], - ) - .unwrap(); -} - -#[cfg(feature = "halo2")] -fn gen_halo2_proof(file_name: &str, inputs: Vec) { - compiler::compile_pil_or_asm( - format!( - "{}/../test_data/pil/{file_name}", - env!("CARGO_MANIFEST_DIR") - ) - .as_str(), - inputs, - &mktemp::Temp::new_dir().unwrap(), - true, - Some(BackendType::Halo2), - vec![], - ) - .unwrap(); -} - -#[cfg(not(feature = "halo2"))] -fn gen_halo2_proof(_file_name: &str, _inputs: Vec) {} - #[test] fn test_fibonacci() { - let f = "fibonacci.pil"; - verify_pil(f, None); + let f = "pil/fibonacci.pil"; + verify_pil(f, vec![]); gen_halo2_proof(f, Default::default()); gen_estark_proof(f, Default::default()); } #[test] fn test_constant_in_identity() { - let f = "constant_in_identity.pil"; - verify_pil(f, None); + let f = "pil/constant_in_identity.pil"; + verify_pil(f, vec![]); gen_halo2_proof(f, Default::default()); gen_estark_proof(f, Default::default()); } #[test] fn fib_arrays() { - let f = "fib_arrays.pil"; - verify_pil(f, None); + let f = "pil/fib_arrays.pil"; + verify_pil(f, vec![]); gen_halo2_proof(f, Default::default()); gen_estark_proof(f, Default::default()); } @@ -102,49 +34,49 @@ fn fib_arrays() { #[test] #[should_panic = "Witness generation failed."] fn test_external_witgen_fails_if_none_provided() { - let f = "external_witgen.pil"; - verify_pil(f, None); + let f = "pil/external_witgen.pil"; + verify_pil(f, vec![]); } #[test] fn test_external_witgen_a_provided() { - let f = "external_witgen.pil"; + let f = "pil/external_witgen.pil"; let external_witness = vec![("main.a", vec![GoldilocksField::from(3); 16])]; - verify_pil_with_external_witness(f, None, external_witness); + verify_test_file(f, vec![], external_witness); } #[test] fn test_external_witgen_b_provided() { - let f = "external_witgen.pil"; + let f = "pil/external_witgen.pil"; let external_witness = vec![("main.b", vec![GoldilocksField::from(4); 16])]; - verify_pil_with_external_witness(f, None, external_witness); + verify_test_file(f, vec![], external_witness); } #[test] fn test_external_witgen_both_provided() { - let f = "external_witgen.pil"; + let f = "pil/external_witgen.pil"; let external_witness = vec![ ("main.a", vec![GoldilocksField::from(3); 16]), ("main.b", vec![GoldilocksField::from(4); 16]), ]; - verify_pil_with_external_witness(f, None, external_witness); + verify_test_file(f, vec![], external_witness); } #[test] #[should_panic = "called `Result::unwrap()` on an `Err` value: Generic(\"main.b = (main.a + 1);:\\n Linear constraint is not satisfiable: -1 != 0\")"] fn test_external_witgen_fails_on_conflicting_external_witness() { - let f = "external_witgen.pil"; + let f = "pil/external_witgen.pil"; let external_witness = vec![ ("main.a", vec![GoldilocksField::from(3); 16]), // Does not satisfy b = a + 1 ("main.b", vec![GoldilocksField::from(3); 16]), ]; - verify_pil_with_external_witness(f, None, external_witness); + verify_test_file(f, vec![], external_witness); } #[test] fn test_global() { - verify_pil("global.pil", None); + verify_pil("pil/global.pil", vec![]); // Halo2 would take too long for this. // Starky requires at least one witness column, this test has none. } @@ -152,16 +84,9 @@ fn test_global() { #[test] fn test_sum_via_witness_query() { verify_pil( - "sum_via_witness_query.pil", - Some(|q| { - Ok(match q { - "\"in\", 0" => Some(7.into()), - "\"in\", 1" => Some(8.into()), - "\"in\", 2" => Some(2.into()), - "\"in\", 3" => None, // This line checks that if we return "None", the system still tries to figure it out on its own. - _ => None, - }) - }), + "pil/sum_via_witness_query.pil", + // Only 3 inputs -> Checks that if we return "None", the system still tries to figure it out on its own. + vec![7.into(), 8.into(), 2.into()], ); // prover query string uses a different convention, // so we cannot directly use the halo2_proof and estark functions here. @@ -169,101 +94,91 @@ fn test_sum_via_witness_query() { #[test] fn test_witness_lookup() { - let f = "witness_lookup.pil"; - verify_pil( - f, - Some(|q| { - Ok(match q { - "\"input\", 0" => Some(3.into()), - "\"input\", 1" => Some(5.into()), - "\"input\", 2" => Some(2.into()), - _ => Some(7.into()), - }) - }), - ); + let f = "pil/witness_lookup.pil"; + let inputs = [3, 5, 2, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7] + .into_iter() + .map(GoldilocksField::from) + .collect::>(); + verify_pil(f, inputs.clone()); // halo2 fails with "gates must contain at least one constraint" - let inputs = vec![3, 5, 2, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7]; - gen_estark_proof(f, inputs.into_iter().map(GoldilocksField::from).collect()); + gen_estark_proof(f, inputs); } #[test] #[should_panic(expected = "Witness generation failed.")] fn test_underdetermined_zero_no_solution() { - verify_pil("underdetermined_zero_no_solution.pil", None); + verify_pil("pil/underdetermined_zero_no_solution.pil", vec![]); } #[test] fn test_pair_lookup() { - let f = "pair_lookup.pil"; - verify_pil(f, None); + let f = "pil/pair_lookup.pil"; + verify_pil(f, vec![]); // halo2 would take too long for this // starky would take too long for this in debug mode } #[test] fn test_block_lookup_or() { - let f = "block_lookup_or.pil"; - verify_pil(f, None); + let f = "pil/block_lookup_or.pil"; + verify_pil(f, vec![]); // halo2 would take too long for this // starky would take too long for this in debug mode } #[test] fn test_halo_without_lookup() { - let f = "halo_without_lookup.pil"; - verify_pil(f, None); + let f = "pil/halo_without_lookup.pil"; + verify_pil(f, vec![]); gen_halo2_proof(f, Default::default()); gen_estark_proof(f, Default::default()); } #[test] fn test_simple_div() { - let f = "simple_div.pil"; - verify_pil(f, None); + let f = "pil/simple_div.pil"; + verify_pil(f, vec![]); // starky would take too long for this in debug mode } #[test] fn test_single_line_blocks() { - let f = "single_line_blocks.pil"; - verify_pil(f, None); + let f = "pil/single_line_blocks.pil"; + verify_pil(f, vec![]); gen_estark_proof(f, Default::default()); } #[test] fn test_two_block_machine_functions() { - let f = "two_block_machine_functions.pil"; - verify_pil(f, None); + let f = "pil/two_block_machine_functions.pil"; + verify_pil(f, vec![]); gen_estark_proof(f, Default::default()); } #[test] fn test_fixed_columns() { - let f = "fixed_columns.pil"; - verify_pil(f, None); + let f = "pil/fixed_columns.pil"; + verify_pil(f, vec![]); // Starky requires at least one witness column, this test has none. } #[test] fn test_witness_via_let() { - verify_pil("witness_via_let.pil", None); + verify_pil("pil/witness_via_let.pil", vec![]); } #[test] fn conditional_fixed_constraints() { - verify_pil("conditional_fixed_constraints.pil", None); + verify_pil("pil/conditional_fixed_constraints.pil", vec![]); } #[test] fn arith_improved() { - let f = "arith_improved.pil"; + let f = "pil/arith_improved.pil"; pil_analyzer::analyze::( - &Path::new(&format!( - "{}/../test_data/pil/{f}", - env!("CARGO_MANIFEST_DIR") - )) - .canonicalize() - .unwrap(), + &Path::new(&format!("{}/../test_data/{f}", env!("CARGO_MANIFEST_DIR"))) + .canonicalize() + .unwrap(), ); } @@ -272,7 +187,7 @@ mod book { use test_log::test; fn run_book_test(file: &str) { - verify_pil(file, None); + verify_pil(file, vec![]); gen_halo2_proof(file, Default::default()); gen_estark_proof(file, Default::default()); } diff --git a/compiler/tests/powdr_std.rs b/compiler/tests/powdr_std.rs index 7f46d118a..cab53b2cb 100644 --- a/compiler/tests/powdr_std.rs +++ b/compiler/tests/powdr_std.rs @@ -1,77 +1,29 @@ -use compiler::verify_asm_string; -use number::{Bn254Field, FieldElement, GoldilocksField}; -use std::fs; +use compiler::test_util::{gen_estark_proof, gen_halo2_proof, verify_test_file}; +use number::GoldilocksField; use test_log::test; -fn verify_asm(file_name: &str, inputs: Vec) { - let file_name = format!( - "{}/../test_data/std/{file_name}", - env!("CARGO_MANIFEST_DIR") - ); - - let contents = fs::read_to_string(&file_name).unwrap(); - - verify_asm_string(&file_name, &contents, inputs, vec![]) -} - -fn gen_estark_proof(file_name: &str, inputs: Vec) { - compiler::compile_pil_or_asm( - format!( - "{}/../test_data/std/{file_name}", - env!("CARGO_MANIFEST_DIR") - ) - .as_str(), - inputs, - &mktemp::Temp::new_dir().unwrap(), - true, - Some(backend::BackendType::EStark), - vec![], - ) - .unwrap(); -} - -#[cfg(feature = "halo2")] -fn gen_halo2_proof(file_name: &str, inputs: Vec) { - compiler::compile_pil_or_asm( - format!( - "{}/../test_data/std/{file_name}", - env!("CARGO_MANIFEST_DIR") - ) - .as_str(), - inputs, - &mktemp::Temp::new_dir().unwrap(), - true, - Some(backend::BackendType::Halo2Mock), - vec![], - ) - .unwrap(); -} - -#[cfg(not(feature = "halo2"))] -fn gen_halo2_proof(_file_name: &str, _inputs: Vec) {} - #[test] fn poseidon_bn254_test() { - let f = "poseidon_bn254_test.asm"; + let f = "std/poseidon_bn254_test.asm"; gen_halo2_proof(f, Default::default()); } #[test] fn poseidon_gl_test() { - let f = "poseidon_gl_test.asm"; - verify_asm::(f, Default::default()); + let f = "std/poseidon_gl_test.asm"; + verify_test_file::(f, vec![], vec![]); gen_estark_proof(f, Default::default()); } #[test] fn split_bn254_test() { - let f = "split_bn254_test.asm"; + let f = "std/split_bn254_test.asm"; gen_halo2_proof(f, Default::default()); } #[test] fn split_gl_test() { - let f = "split_gl_test.asm"; - verify_asm::(f, Default::default()); + let f = "std/split_gl_test.asm"; + verify_test_file::(f, vec![], vec![]); gen_estark_proof(f, Default::default()); } diff --git a/executor/src/witgen/block_processor.rs b/executor/src/witgen/block_processor.rs index 8a052d31a..4ba88aa9c 100644 --- a/executor/src/witgen/block_processor.rs +++ b/executor/src/witgen/block_processor.rs @@ -134,7 +134,10 @@ mod tests { f: impl Fn(BlockProcessor, BTreeMap, u64, usize) -> R, ) -> R { let analyzed = analyze_string(src); - let constants = generate(&analyzed); + let constants = generate(&analyzed) + .into_iter() + .map(|(n, c)| (n.to_string(), c)) + .collect::>(); let fixed_data = FixedData::new(&analyzed, &constants, vec![]); // No global range constraints diff --git a/executor/src/witgen/mod.rs b/executor/src/witgen/mod.rs index ba3ad1761..f314b122d 100644 --- a/executor/src/witgen/mod.rs +++ b/executor/src/witgen/mod.rs @@ -53,7 +53,7 @@ pub struct MutableState<'a, 'b, T: FieldElement, Q: QueryCallback> { pub struct WitnessGenerator<'a, 'b, T: FieldElement, Q: QueryCallback> { analyzed: &'a Analyzed, - fixed_col_values: &'b [(&'a str, Vec)], + fixed_col_values: &'b [(String, Vec)], query_callback: Q, external_witness_values: Vec<(&'a str, Vec)>, } @@ -61,7 +61,7 @@ pub struct WitnessGenerator<'a, 'b, T: FieldElement, Q: QueryCallback> { impl<'a, 'b, T: FieldElement, Q: QueryCallback> WitnessGenerator<'a, 'b, T, Q> { pub fn new( analyzed: &'a Analyzed, - fixed_col_values: &'b [(&'a str, Vec)], + fixed_col_values: &'b [(String, Vec)], query_callback: Q, ) -> Self { WitnessGenerator { @@ -167,7 +167,7 @@ pub struct FixedData<'a, T> { impl<'a, T: FieldElement> FixedData<'a, T> { pub fn new( analyzed: &'a Analyzed, - fixed_col_values: &'a [(&str, Vec)], + fixed_col_values: &'a [(String, Vec)], external_witness_values: Vec<(&'a str, Vec)>, ) -> Self { let mut external_witness_values = BTreeMap::from_iter(external_witness_values); diff --git a/halo2/src/mock_prover.rs b/halo2/src/mock_prover.rs index fcebab12e..db033b812 100644 --- a/halo2/src/mock_prover.rs +++ b/halo2/src/mock_prover.rs @@ -5,16 +5,17 @@ use super::circuit_builder::analyzed_to_circuit; use halo2_proofs::{dev::MockProver, halo2curves::bn256::Fr}; use number::{BigInt, FieldElement}; +// Can't depend on compiler::pipeline::GeneratedWitness because of circular dependencies... pub fn mock_prove( pil: &Analyzed, - fixed: &[(String, Vec)], + constants: &[(String, Vec)], witness: &[(String, Vec)], ) { if polyexen::expr::get_field_p::() != T::modulus().to_arbitrary_integer() { panic!("powdr modulus doesn't match halo2 modulus. Make sure you are using Bn254"); } - let circuit = analyzed_to_circuit(pil, fixed, witness); + let circuit = analyzed_to_circuit(pil, constants, witness); // double the row count in order to make space for the cells introduced by the backend // TODO: use a precise count of the extra rows needed to avoid using so many rows @@ -33,76 +34,53 @@ pub fn mock_prove( #[cfg(test)] mod test { - use std::{fs, path::PathBuf}; - - use analysis::convert_asm_to_pil; + use compiler::{inputs_to_query_callback, pipeline::Pipeline, test_util::resolve_test_file}; use executor::witgen::unused_query_callback; use number::Bn254Field; - use parser::parse_asm; use test_log::test; use super::*; #[allow(clippy::print_stdout)] fn mock_prove_asm(file_name: &str, inputs: &[Bn254Field]) { - // read and compile PIL. - - let location = format!( - "{}/../test_data/asm/{file_name}", - env!("CARGO_MANIFEST_DIR") + let mut pipeline = Pipeline::default().from_file(resolve_test_file(file_name)); + pipeline + .generate_witness(inputs_to_query_callback(inputs.to_vec()), vec![]) + .unwrap(); + let result = pipeline.generated_witness().unwrap(); + mock_prove( + &result.pil, + &result.constants, + result.witness.as_ref().unwrap(), ); - - let contents = fs::read_to_string(&location).unwrap(); - let parsed = parse_asm::(Some(&location), &contents).unwrap(); - let resolved = importer::resolve(Some(PathBuf::from(location)), parsed).unwrap(); - let analysed = convert_asm_to_pil(resolved).unwrap(); - let graph = airgen::compile(analysed); - let pil = linker::link(graph).unwrap(); - - let query_callback = compiler::inputs_to_query_callback(inputs.to_vec()); - - let analyzed = pil_analyzer::analyze_string(&format!("{pil}")); - - let fixed = executor::constant_evaluator::generate(&analyzed); - let witness = - executor::witgen::WitnessGenerator::new(&analyzed, &fixed, query_callback).generate(); - - let fixed = to_owned_values(fixed); - - mock_prove(&analyzed, &fixed, &witness); } #[test] fn simple_pil_halo2() { - let content = "namespace Global(8); pol fixed z = [0]*; pol witness a; a = 0;"; - let analyzed: Analyzed = pil_analyzer::analyze_string(content); - let fixed = executor::constant_evaluator::generate(&analyzed); + let content = "namespace Global(8); pol fixed z = [1, 2]*; pol witness a; a = z + 1;"; - let witness = - executor::witgen::WitnessGenerator::new(&analyzed, &fixed, unused_query_callback()) - .generate(); + let mut pipeline = Pipeline::::default().from_pil_string(content.to_string()); + pipeline + .generate_witness(unused_query_callback(), vec![]) + .unwrap(); - let fixed = to_owned_values(fixed); - - mock_prove(&analyzed, &fixed, &witness); + let result = pipeline.generated_witness().unwrap(); + mock_prove( + &result.pil, + &result.constants, + result.witness.as_ref().unwrap(), + ); } #[test] fn simple_sum() { let inputs = [165, 5, 11, 22, 33, 44, 55].map(From::from); - mock_prove_asm("simple_sum.asm", &inputs); + mock_prove_asm("asm/simple_sum.asm", &inputs); } #[test] fn palindrome() { let inputs = [3, 11, 22, 11].map(From::from); - mock_prove_asm("palindrome.asm", &inputs); - } - - fn to_owned_values(values: Vec<(&str, Vec)>) -> Vec<(String, Vec)> { - values - .into_iter() - .map(|(s, fields)| (s.to_string(), fields.clone())) - .collect::>() + mock_prove_asm("asm/palindrome.asm", &inputs); } } diff --git a/powdr_cli/src/main.rs b/powdr_cli/src/main.rs index ec3b3789f..ce6abcd7e 100644 --- a/powdr_cli/src/main.rs +++ b/powdr_cli/src/main.rs @@ -4,8 +4,9 @@ mod util; use backend::{Backend, BackendType, Proof}; use clap::{CommandFactory, Parser, Subcommand}; +use compiler::inputs_to_query_callback; +use compiler::pipeline::{Pipeline, ProofResult}; use compiler::util::{read_poly_set, FixedPolySet, WitnessPolySet}; -use compiler::{compile_asm_string, compile_pil_or_asm, CompilationResult}; use env_logger::fmt::Color; use env_logger::{Builder, Target}; use log::LevelFilter; @@ -16,6 +17,7 @@ use riscv::continuations::rust_continuations; use riscv::{compile_riscv_asm, compile_rust}; use std::collections::HashMap; use std::io::{self, BufReader, BufWriter, Read}; +use std::path::PathBuf; use std::{borrow::Cow, fs, io::Write, path::Path}; use strum::{Display, EnumString, EnumVariantNames}; @@ -601,16 +603,16 @@ fn handle_riscv_asm( unimplemented!("Running witgen with continuations is not supported yet.") } (false, false) => { - compile_asm_string( - file_name, - contents, - inputs, - None, - output_dir, - force_overwrite, - prove_with, - vec![], - )?; + let mut pipeline = Pipeline::default() + .with_output(output_dir.to_path_buf(), force_overwrite) + .from_asm_string(contents.to_string(), Some(PathBuf::from(file_name))); + pipeline + .generate_witness(inputs_to_query_callback(inputs), vec![]) + .unwrap(); + pipeline.prove(BackendType::PilStarkCli).unwrap(); + if let Some(backend) = prove_with { + pipeline.prove(backend).unwrap(); + } } } Ok(()) @@ -640,19 +642,27 @@ fn compile_with_csv_export( let external_witness_values = strings.iter().map(AsRef::as_ref).zip(values).collect(); let output_dir = Path::new(&output_directory); - let result = compile_pil_or_asm::( - &file, - split_inputs(&inputs), - output_dir, - force, - prove_with.clone(), - external_witness_values, - )?; + + let mut pipeline = Pipeline::default() + .with_output(output_dir.to_path_buf(), force) + .from_file(PathBuf::from(file)); + pipeline + .generate_witness( + inputs_to_query_callback(split_inputs(&inputs)), + external_witness_values, + ) + .unwrap(); + let result = if let Some(backend) = &prove_with { + pipeline.prove(backend.clone()).unwrap(); + Some(pipeline.proof().unwrap()) + } else { + None + }; if let Some(ref compilation_result) = result { serialize_result_witness(output_dir, compilation_result); - if let Some(_backend) = prove_with { + if let Some(_backend) = &prove_with { write_proving_results_to_fs( false, &compilation_result.proof, @@ -708,7 +718,10 @@ fn read_and_prove( proof_path: Option, params: Option, ) { - let pil = pilopt::optimize(compiler::analyze_pil::(file)); + let pil = Pipeline::default() + .from_file(file.to_path_buf()) + .optimized_pil() + .unwrap(); let fixed = read_poly_set::(&pil, dir); let witness = read_poly_set::(&pil, dir); @@ -741,11 +754,14 @@ fn read_and_prove( fn optimize_and_output(file: &str) { println!( "{}", - pilopt::optimize(compiler::analyze_pil::(Path::new(file))) + Pipeline::::default() + .from_file(PathBuf::from(file)) + .optimized_pil() + .unwrap() ); } -fn serialize_result_witness(output_dir: &Path, results: &CompilationResult) { +fn serialize_result_witness(output_dir: &Path, results: &ProofResult) { write_constants_to_fs(&results.constants, output_dir); let witness = results.witness.as_ref().unwrap(); write_commits_to_fs(witness, output_dir); diff --git a/riscv/src/continuations.rs b/riscv/src/continuations.rs index 23038f0b0..ee2b8ab6e 100644 --- a/riscv/src/continuations.rs +++ b/riscv/src/continuations.rs @@ -1,5 +1,9 @@ -use std::collections::{BTreeSet, HashMap}; +use std::{ + collections::{BTreeSet, HashMap}, + path::PathBuf, +}; +use compiler::pipeline::Pipeline; use number::FieldElement; use riscv_executor::ExecutionTrace; @@ -28,8 +32,10 @@ fn transposed_trace(trace: &ExecutionTrace) -> HashMap(file_name: &str, contents: &str, inputs: Vec) { let mut bootloader_inputs = default_input(); - let program = - compiler::compile_asm_string_to_analyzed_ast::(file_name, contents, None).unwrap(); + let program = Pipeline::default() + .from_asm_string(contents.to_string(), Some(PathBuf::from(file_name))) + .analyzed_asm() + .unwrap(); let inputs: HashMap> = vec![(F::from(0), inputs)].into_iter().collect(); diff --git a/riscv/tests/common/mod.rs b/riscv/tests/common/mod.rs index 917a7f9dc..3faded12f 100644 --- a/riscv/tests/common/mod.rs +++ b/riscv/tests/common/mod.rs @@ -1,42 +1,25 @@ -use compiler::{ - compile_asm_string, verify, write_commits_to_fs, write_constants_to_fs, - write_constraints_to_fs, BackendType, -}; +use compiler::{pipeline::Pipeline, test_util::verify_pipeline}; use number::GoldilocksField; use riscv::bootloader::default_input; -use std::collections::HashMap; +use std::{collections::HashMap, path::PathBuf}; -/// Like compiler::verify::verify_asm_string, but also runs RISCV executor. +/// Like compiler::test_util::verify_asm_string, but also runs RISCV executor. pub fn verify_riscv_asm_string(file_name: &str, contents: &str, inputs: Vec) { let temp_dir = mktemp::Temp::new_dir().unwrap().release(); let mut inputs_hash: HashMap> = HashMap::default(); inputs_hash.insert(0u32.into(), inputs.clone()); - let (_, result) = compile_asm_string( - file_name, - contents, - inputs.clone(), - Some(&mut |analyzed| { - riscv_executor::execute_ast( - analyzed, - &inputs_hash.clone(), - &default_input(), - usize::MAX, - riscv_executor::ExecMode::Fast, - ); - }), - &temp_dir, - true, - Some(BackendType::PilStarkCli), - vec![], - ) - .unwrap(); - - let result = result.unwrap(); - write_constants_to_fs(&result.constants, &temp_dir); - write_commits_to_fs(&result.witness.unwrap(), &temp_dir); - write_constraints_to_fs(&result.constraints_serialization.unwrap(), &temp_dir); - - verify(&temp_dir); + let mut pipeline = Pipeline::default() + .with_output(temp_dir.to_path_buf(), false) + .from_asm_string(contents.to_string(), Some(PathBuf::from(file_name))); + let analyzed = pipeline.analyzed_asm_ref().unwrap(); + riscv_executor::execute_ast( + analyzed, + &inputs_hash.clone(), + &default_input(), + usize::MAX, + riscv_executor::ExecMode::Fast, + ); + verify_pipeline(pipeline, inputs, vec![]); } diff --git a/riscv/tests/riscv.rs b/riscv/tests/riscv.rs index aa5fd62cc..1d9d4fa7f 100644 --- a/riscv/tests/riscv.rs +++ b/riscv/tests/riscv.rs @@ -1,7 +1,7 @@ mod common; use common::verify_riscv_asm_string; -use compiler::verify_asm_string; +use compiler::test_util::verify_asm_string; use mktemp::Temp; use number::GoldilocksField; use test_log::test; diff --git a/test_data/pil/sum_via_witness_query.pil b/test_data/pil/sum_via_witness_query.pil index fddd527bd..752b655c8 100644 --- a/test_data/pil/sum_via_witness_query.pil +++ b/test_data/pil/sum_via_witness_query.pil @@ -7,7 +7,7 @@ namespace Sum(%N); pol fixed ISALMOSTLAST(i) { i == %last_row - 1 }; pol fixed ISFIRST = [ 1, 0 ] + [0]*; - col witness input(i) query ("in", i); + col witness input(i) query ("input", i); col witness sum; ISLAST * sum' = 0;