Add many_chunks_memory test

pipeline/src/test_util.rs

@@ -18,8 +18,11 @@ pub fn verify_test_file<T: FieldElement>(
     inputs: Vec<T>,
     external_witness_values: Vec<(String, Vec<T>)>,
 ) {
-    let pipeline = Pipeline::default().from_file(resolve_test_file(file_name));
-    verify_pipeline(pipeline, inputs, external_witness_values)
+    let pipeline = Pipeline::default()
+        .from_file(resolve_test_file(file_name))
+        .with_prover_inputs(inputs)
+        .add_external_witness_values(external_witness_values);
+    verify_pipeline(pipeline)
 }
 
 pub fn verify_asm_string<T: FieldElement>(
@@ -28,20 +31,15 @@ pub fn verify_asm_string<T: FieldElement>(
     inputs: Vec<T>,
     external_witness_values: Vec<(String, Vec<T>)>,
 ) {
-    let pipeline =
-        Pipeline::default().from_asm_string(contents.to_string(), Some(PathBuf::from(file_name)));
-    verify_pipeline(pipeline, inputs, external_witness_values)
+    let pipeline = Pipeline::default()
+        .from_asm_string(contents.to_string(), Some(PathBuf::from(file_name)))
+        .with_prover_inputs(inputs)
+        .add_external_witness_values(external_witness_values);
+    verify_pipeline(pipeline)
 }
 
-pub fn verify_pipeline<T: FieldElement>(
-    pipeline: Pipeline<T>,
-    inputs: Vec<T>,
-    external_witness_values: Vec<(String, Vec<T>)>,
-) {
-    let mut pipeline = pipeline
-        .add_external_witness_values(external_witness_values)
-        .with_prover_inputs(inputs)
-        .with_backend(BackendType::PilStarkCli);
+pub fn verify_pipeline<T: FieldElement>(pipeline: Pipeline<T>) {
+    let mut pipeline = pipeline.with_backend(BackendType::PilStarkCli);
 
     if pipeline.output_dir().is_none() {
         pipeline = pipeline.with_tmp_output();
@@ -51,7 +49,7 @@ pub fn verify_pipeline<T: FieldElement>(
     // which owns the temporary directory.
     pipeline.advance_to(Stage::Proof).unwrap();
 
-    verify(pipeline.output_dir().unwrap(), pipeline.name());
+    verify(pipeline.output_dir().unwrap(), pipeline.name(), None);
 }
 
 pub fn gen_estark_proof(file_name: &str, inputs: Vec<GoldilocksField>) {

pipeline/src/verify.rs

@@ -1,10 +1,15 @@
 use std::{path::Path, process::Command};
 
-pub fn verify(temp_dir: &Path, name: &str) {
+pub fn verify(temp_dir: &Path, name: &str, constants_name: Option<&str>) {
     let pilcom = std::env::var("PILCOM")
         .expect("Please set the PILCOM environment variable to the path to the pilcom repository.");
 
-    let constants_file = format!("{}/{name}_constants.bin", temp_dir.to_str().unwrap());
+    let constants_name = constants_name.unwrap_or(name);
+
+    let constants_file = format!(
+        "{}/{constants_name}_constants.bin",
+        temp_dir.to_str().unwrap()
+    );
     let commits_file = format!("{}/{name}_commits.bin", temp_dir.to_str().unwrap());
     let constraints_file = format!("{}/{name}_constraints.json", temp_dir.to_str().unwrap());
 

riscv/Cargo.toml

@@ -33,6 +33,7 @@ lalrpop = "^0.19"
 
 [dev-dependencies]
 powdr-number = { path = "../number" }
+powdr-backend = { path = "../backend" }
 
 test-log = "0.2.12"
 env_logger = "0.10.0"

riscv/tests/common/mod.rs

@@ -20,5 +20,5 @@ pub fn verify_riscv_asm_string(file_name: &str, contents: &str, inputs: Vec<Gold
         usize::MAX,
         powdr_riscv_executor::ExecMode::Fast,
     );
-    verify_pipeline(pipeline, inputs, vec![]);
+    verify_pipeline(pipeline);
 }

riscv/tests/riscv.rs

@@ -1,14 +1,15 @@
 mod common;
 
-use std::path::PathBuf;
-
 use common::verify_riscv_asm_string;
 use mktemp::Temp;
+use powdr_backend::BackendType;
 use powdr_number::GoldilocksField;
 use powdr_pipeline::{
     test_util::{verify_asm_string, verify_pipeline},
-    Pipeline,
+    verify::verify,
+    Pipeline, Stage,
 };
+use std::path::PathBuf;
 use test_log::test;
 
 use powdr_riscv::{
@@ -16,6 +17,37 @@ use powdr_riscv::{
     CoProcessors,
 };
 
+/// Compiles and runs a rust file with continuations, runs the full
+/// witness generation & verifies it using Pilcom.
+pub fn test_continuations(case: &str) {
+    let rust_file = format!("{case}.rs");
+    let coprocessors = CoProcessors::base().with_poseidon();
+    let temp_dir = Temp::new_dir().unwrap();
+    let riscv_asm =
+        powdr_riscv::compile_rust_to_riscv_asm(&format!("tests/riscv_data/{rust_file}"), &temp_dir);
+    let powdr_asm = powdr_riscv::compiler::compile(riscv_asm, &coprocessors, true);
+
+    // Manually create tmp dir, so that it is the same in all chunks.
+    let tmp_dir = mktemp::Temp::new_dir().unwrap();
+
+    let pipeline_factory = || {
+        Pipeline::<GoldilocksField>::default()
+            .from_asm_string(powdr_asm.clone(), Some(PathBuf::from(&rust_file)))
+            .with_prover_inputs(Default::default())
+            .with_output(tmp_dir.to_path_buf(), false)
+    };
+    let pipeline_callback = |pipeline: Pipeline<GoldilocksField>| -> Result<(), ()> {
+        // Can't use `verify_pipeline`, because the pipeline was renamed in the middle of after
+        // computing the constants file.
+        let mut pipeline = pipeline.with_backend(BackendType::PilStarkCli);
+        pipeline.advance_to(Stage::Proof).unwrap();
+        verify(pipeline.output_dir().unwrap(), pipeline.name(), Some(case));
+        Ok(())
+    };
+    let bootloader_inputs = rust_continuations_dry_run(pipeline_factory());
+    rust_continuations(pipeline_factory, pipeline_callback, bootloader_inputs).unwrap();
+}
+
 #[test]
 #[ignore = "Too slow"]
 fn test_trivial() {
@@ -145,7 +177,7 @@ fn test_evm() {
         .from_asm_string(powdr_asm, None)
         .add_data(666, &bytes);
 
-    verify_pipeline(pipeline, Default::default(), vec![]);
+    verify_pipeline(pipeline);
 }
 
 #[test]
@@ -177,38 +209,13 @@ fn test_many_chunks_dry() {
 #[test]
 #[ignore = "Too slow"]
 fn test_many_chunks() {
-    // Compiles and runs the many_chunks.rs example with continuations, runs the full
-    // witness generation & verifies it using Pilcom.
-    // TODO: Make this test pass!
-    let case = "many_chunks.rs";
-    let coprocessors = CoProcessors::base().with_poseidon();
-    let temp_dir = Temp::new_dir().unwrap();
-    let riscv_asm =
-        powdr_riscv::compile_rust_to_riscv_asm(&format!("tests/riscv_data/{case}"), &temp_dir);
-    let powdr_asm = powdr_riscv::compiler::compile(riscv_asm, &coprocessors, true);
+    test_continuations("many_chunks")
+}
 
-    // Manually create tmp dir, so that it is the same in all chunks.
-    let tmp_dir = mktemp::Temp::new_dir().unwrap();
-
-    let pipeline_factory = || {
-        Pipeline::<GoldilocksField>::default()
-            .from_asm_string(powdr_asm.clone(), Some(PathBuf::from(case)))
-            .with_prover_inputs(Default::default())
-            .with_output(tmp_dir.to_path_buf(), false)
-    };
-    let pipeline_callback = |pipeline: Pipeline<GoldilocksField>| -> Result<(), ()> {
-        // The continuations code runs the pipeline until the point were fixed columns
-        // are evaluated and then renames the pipeline. This doesn't play well with
-        // verify_pipeline, so we copy the artifacts here.
-        // Specifically, we copy many_chunks_constants.bin to <pipeline_name>_constants.bin.
-        let tmp_dir = pipeline.output_dir().unwrap();
-        let constants_file = tmp_dir.join(format!("{}_constants.bin", pipeline.name()));
-        std::fs::copy(tmp_dir.join("many_chunks_constants.bin"), constants_file).unwrap();
-        verify_pipeline(pipeline, Default::default(), vec![]);
-        Ok(())
-    };
-    let bootloader_inputs = rust_continuations_dry_run(pipeline_factory());
-    rust_continuations(pipeline_factory, pipeline_callback, bootloader_inputs).unwrap();
+#[test]
+#[ignore = "Too slow"]
+fn test_many_chunks_memory() {
+    test_continuations("many_chunks_memory")
 }
 
 fn verify_file(case: &str, inputs: Vec<GoldilocksField>, coprocessors: &CoProcessors) {

riscv/tests/riscv_data/many_chunks_memory.rs

@@ -0,0 +1,27 @@
+#![no_std]
+
+extern crate alloc;
+use alloc::vec::Vec;
+
+const N: usize = 20000;
+
+#[no_mangle]
+pub fn main() {
+    let mut fibonacci_numbers = Vec::<u8>::with_capacity(N);
+
+    fibonacci_numbers.push(1);
+    fibonacci_numbers.push(1);
+    for _ in 0..N {
+        let tmp = fibonacci_numbers[fibonacci_numbers.len() - 1]
+            + fibonacci_numbers[fibonacci_numbers.len() - 2];
+        fibonacci_numbers.push(tmp);
+    }
+
+    // Some random accesses in the last chunk:
+    for i in 0..10 {
+        assert!(
+            fibonacci_numbers[i * N / 10] + fibonacci_numbers[i * N / 10 + 1]
+                == fibonacci_numbers[i * N / 10 + 2]
+        );
+    }
+}