Always test PILCOM + Composite (#1587)

This PR contains the following changes:
- I renamed a bunch of functions `verify_*` to `run_pilcom_*`, because I
think it better describes what they do
- They all call `run_pilcom_with_backend_variant`, which works analogous
to `gen_estark_proof_with_backend_variant` and
`test_halo2_with_backend_variant`
- In functions like `run_pilcom_test_file` (previously
`verify_test_file`), which are used for many tests, we now test both the
monolithic and composite backend variant (but share the generated
witness & constants). This is anlogous to `gen_estark_proof` and
`test_halo2`
- In the RISC-V tests, we only test the composite variant, because with
registers in memory (#1443) we don't expect the monolithic backend
variant to work anymore.
This commit is contained in:
Georg Wiese
2024-07-17 15:40:08 +02:00
committed by GitHub
parent 4f873e6205
commit e63c59be09
8 changed files with 114 additions and 139 deletions

View File

@@ -92,34 +92,6 @@ impl BackendType {
}
}
}
pub fn is_composite(&self) -> bool {
match self {
#[cfg(feature = "halo2")]
BackendType::Halo2 => false,
#[cfg(feature = "halo2")]
BackendType::Halo2Composite => true,
#[cfg(feature = "halo2")]
BackendType::Halo2Mock => false,
#[cfg(feature = "halo2")]
BackendType::Halo2MockComposite => true,
#[cfg(feature = "estark-polygon")]
BackendType::EStarkPolygon => false,
#[cfg(feature = "estark-polygon")]
BackendType::EStarkPolygonComposite => true,
BackendType::EStarkStarky => false,
BackendType::EStarkStarkyComposite => true,
BackendType::EStarkDump => false,
BackendType::EStarkDumpComposite => true,
#[cfg(feature = "plonky3")]
BackendType::Plonky3 => false,
#[cfg(feature = "plonky3")]
BackendType::Plonky3Composite => true,
// We explicitly do not use a wildcard here
// so that a new composite backend needs to be
// added here too.
}
}
}
#[derive(thiserror::Error, Debug)]

View File

@@ -30,19 +30,35 @@ pub fn execute_test_file(
.map(|_| ())
}
pub fn verify_test_file(
/// Makes a new pipeline for the given file and inputs. All steps until witness generation are
/// already computed, so that the test can branch off from there, without having to re-compute
/// these steps.
pub fn make_prepared_pipeline<T: FieldElement>(
file_name: &str,
inputs: Vec<T>,
external_witness_values: Vec<(String, Vec<T>)>,
) -> Pipeline<T> {
let mut pipeline = Pipeline::default()
.with_tmp_output()
.from_file(resolve_test_file(file_name))
.with_prover_inputs(inputs)
.add_external_witness_values(external_witness_values);
pipeline.compute_witness().unwrap();
pipeline
}
pub fn run_pilcom_test_file(
file_name: &str,
inputs: Vec<GoldilocksField>,
external_witness_values: Vec<(String, Vec<GoldilocksField>)>,
) -> Result<(), String> {
let pipeline = Pipeline::default()
.from_file(resolve_test_file(file_name))
.with_prover_inputs(inputs)
.add_external_witness_values(external_witness_values);
verify_pipeline(pipeline, BackendType::EStarkDump)
let pipeline = make_prepared_pipeline(file_name, inputs, external_witness_values);
run_pilcom_with_backend_variant(pipeline.clone(), BackendVariant::Monolithic)?;
run_pilcom_with_backend_variant(pipeline, BackendVariant::Composite)?;
Ok(())
}
pub fn verify_asm_string<S: serde::Serialize + Send + Sync + 'static>(
pub fn run_pilcom_asm_string<S: serde::Serialize + Send + Sync + 'static>(
file_name: &str,
contents: &str,
inputs: Vec<GoldilocksField>,
@@ -57,14 +73,20 @@ pub fn verify_asm_string<S: serde::Serialize + Send + Sync + 'static>(
if let Some(data) = data {
pipeline = pipeline.add_data_vec(&data);
}
pipeline.compute_witness().unwrap();
verify_pipeline(pipeline, BackendType::EStarkDump).unwrap();
run_pilcom_with_backend_variant(pipeline.clone(), BackendVariant::Monolithic).unwrap();
run_pilcom_with_backend_variant(pipeline, BackendVariant::Composite).unwrap();
}
pub fn verify_pipeline(
pub fn run_pilcom_with_backend_variant(
pipeline: Pipeline<GoldilocksField>,
backend: BackendType,
backend_variant: BackendVariant,
) -> Result<(), String> {
let backend = match backend_variant {
BackendVariant::Monolithic => BackendType::EStarkDump,
BackendVariant::Composite => BackendType::EStarkDumpComposite,
};
let mut pipeline = pipeline.with_backend(backend, None);
if pipeline.output_dir().is_none() {
@@ -74,35 +96,24 @@ pub fn verify_pipeline(
pipeline.compute_proof().unwrap();
let out_dir = pipeline.output_dir().as_ref().unwrap();
if backend.is_composite() {
// traverse all subdirs of the given output dir and verify each subproof
for entry in fs::read_dir(out_dir).unwrap() {
let entry = entry.unwrap();
let path = entry.path();
if path.is_dir() {
verify(&path)?;
match backend_variant {
BackendVariant::Composite => {
// traverse all subdirs of the given output dir and verify each subproof
for entry in fs::read_dir(out_dir).unwrap() {
let entry = entry.unwrap();
let path = entry.path();
if path.is_dir() {
verify(&path)?;
}
}
Ok(())
}
Ok(())
} else {
verify(out_dir)
BackendVariant::Monolithic => verify(out_dir),
}
}
/// Makes a new pipeline for the given file and inputs. All steps until witness generation are
/// already computed, so that the test can branch off from there, without having to re-compute
/// these steps.
pub fn make_prepared_pipeline<T: FieldElement>(file_name: &str, inputs: Vec<T>) -> Pipeline<T> {
let mut pipeline = Pipeline::default()
.with_tmp_output()
.from_file(resolve_test_file(file_name))
.with_prover_inputs(inputs);
pipeline.compute_witness().unwrap();
pipeline
}
pub fn gen_estark_proof(file_name: &str, inputs: Vec<GoldilocksField>) {
let pipeline = make_prepared_pipeline(file_name, inputs);
let pipeline = make_prepared_pipeline(file_name, inputs, Vec::new());
gen_estark_proof_with_backend_variant(pipeline.clone(), BackendVariant::Monolithic);
gen_estark_proof_with_backend_variant(pipeline, BackendVariant::Composite);
}
@@ -146,7 +157,7 @@ pub fn gen_estark_proof_with_backend_variant(
}
pub fn test_halo2(file_name: &str, inputs: Vec<Bn254Field>) {
let pipeline = make_prepared_pipeline(file_name, inputs);
let pipeline = make_prepared_pipeline(file_name, inputs, Vec::new());
test_halo2_with_backend_variant(pipeline.clone(), BackendVariant::Monolithic);
test_halo2_with_backend_variant(pipeline, BackendVariant::Composite);
}
@@ -338,12 +349,14 @@ pub fn assert_proofs_fail_for_invalid_witnesses_pilcom(
file_name: &str,
witness: &[(String, Vec<u64>)],
) {
let pipeline = Pipeline::<GoldilocksField>::default()
let mut pipeline = Pipeline::<GoldilocksField>::default()
.with_tmp_output()
.from_file(resolve_test_file(file_name))
.set_witness(convert_witness(witness));
pipeline.compute_witness().unwrap();
assert!(verify_pipeline(pipeline.clone(), BackendType::EStarkDump).is_err());
assert!(run_pilcom_with_backend_variant(pipeline.clone(), BackendVariant::Monolithic).is_err());
assert!(run_pilcom_with_backend_variant(pipeline, BackendVariant::Composite).is_err());
}
pub fn assert_proofs_fail_for_invalid_witnesses_estark(

View File

@@ -3,8 +3,8 @@ use powdr_number::{Bn254Field, FieldElement, GoldilocksField};
use powdr_pipeline::{
test_util::{
gen_estark_proof, gen_estark_proof_with_backend_variant, make_prepared_pipeline,
resolve_test_file, test_halo2, test_halo2_with_backend_variant, verify_test_file,
BackendVariant,
resolve_test_file, run_pilcom_test_file, run_pilcom_with_backend_variant, test_halo2,
test_halo2_with_backend_variant, BackendVariant,
},
util::{read_poly_set, FixedPolySet, WitnessPolySet},
Pipeline,
@@ -12,7 +12,7 @@ use powdr_pipeline::{
use test_log::test;
fn verify_asm(file_name: &str, inputs: Vec<GoldilocksField>) {
verify_test_file(file_name, inputs, vec![]).unwrap();
run_pilcom_test_file(file_name, inputs, vec![]).unwrap();
}
fn slice_to_vec<T: FieldElement>(arr: &[i32]) -> Vec<T> {
@@ -82,7 +82,7 @@ fn mem_write_once_external_write() {
mem[17] = GoldilocksField::from(42);
mem[62] = GoldilocksField::from(123);
mem[255] = GoldilocksField::from(-1);
verify_test_file(
run_pilcom_test_file(
f,
Default::default(),
vec![("main_memory.value".to_string(), mem)],
@@ -228,9 +228,17 @@ fn vm_to_block_different_length() {
let f = "asm/vm_to_block_different_length.asm";
// Because machines have different lengths, this can only be proven
// with a composite proof.
test_halo2_with_backend_variant(make_prepared_pipeline(f, vec![]), BackendVariant::Composite);
run_pilcom_with_backend_variant(
make_prepared_pipeline(f, vec![], vec![]),
BackendVariant::Composite,
)
.unwrap();
test_halo2_with_backend_variant(
make_prepared_pipeline(f, vec![], vec![]),
BackendVariant::Composite,
);
gen_estark_proof_with_backend_variant(
make_prepared_pipeline(f, vec![]),
make_prepared_pipeline(f, vec![], vec![]),
BackendVariant::Composite,
);
}

View File

@@ -5,14 +5,15 @@ use powdr_pipeline::test_util::{
assert_proofs_fail_for_invalid_witnesses, assert_proofs_fail_for_invalid_witnesses_estark,
assert_proofs_fail_for_invalid_witnesses_halo2,
assert_proofs_fail_for_invalid_witnesses_pilcom, gen_estark_proof,
gen_estark_proof_with_backend_variant, make_prepared_pipeline, test_halo2,
test_halo2_with_backend_variant, test_plonky3, verify_test_file, BackendVariant,
gen_estark_proof_with_backend_variant, make_prepared_pipeline, run_pilcom_test_file,
run_pilcom_with_backend_variant, test_halo2, test_halo2_with_backend_variant, test_plonky3,
BackendVariant,
};
use test_log::test;
pub fn verify_pil(file_name: &str, inputs: Vec<GoldilocksField>) {
verify_test_file(file_name, inputs, vec![]).unwrap();
run_pilcom_test_file(file_name, inputs, vec![]).unwrap();
}
#[test]
@@ -145,14 +146,14 @@ fn external_witgen_fails_if_none_provided() {
fn external_witgen_a_provided() {
let f = "pil/external_witgen.pil";
let external_witness = vec![("main.a".to_string(), vec![GoldilocksField::from(3); 16])];
verify_test_file(f, Default::default(), external_witness).unwrap();
run_pilcom_test_file(f, Default::default(), external_witness).unwrap();
}
#[test]
fn external_witgen_b_provided() {
let f = "pil/external_witgen.pil";
let external_witness = vec![("main.b".to_string(), vec![GoldilocksField::from(4); 16])];
verify_test_file(f, Default::default(), external_witness).unwrap();
run_pilcom_test_file(f, Default::default(), external_witness).unwrap();
}
#[test]
@@ -162,7 +163,7 @@ fn external_witgen_both_provided() {
("main.a".to_string(), vec![GoldilocksField::from(3); 16]),
("main.b".to_string(), vec![GoldilocksField::from(4); 16]),
];
verify_test_file(f, Default::default(), external_witness).unwrap();
run_pilcom_test_file(f, Default::default(), external_witness).unwrap();
}
#[test]
@@ -174,7 +175,7 @@ fn external_witgen_fails_on_conflicting_external_witness() {
// Does not satisfy b = a + 1
("main.b".to_string(), vec![GoldilocksField::from(3); 16]),
];
verify_test_file(f, Default::default(), external_witness).unwrap();
run_pilcom_test_file(f, Default::default(), external_witness).unwrap();
}
#[test]
@@ -313,9 +314,17 @@ fn different_degrees() {
let f = "pil/different_degrees.pil";
// Because machines have different lengths, this can only be proven
// with a composite proof.
test_halo2_with_backend_variant(make_prepared_pipeline(f, vec![]), BackendVariant::Composite);
run_pilcom_with_backend_variant(
make_prepared_pipeline(f, vec![], vec![]),
BackendVariant::Composite,
)
.unwrap();
test_halo2_with_backend_variant(
make_prepared_pipeline(f, vec![], vec![]),
BackendVariant::Composite,
);
gen_estark_proof_with_backend_variant(
make_prepared_pipeline(f, vec![]),
make_prepared_pipeline(f, vec![], vec![]),
BackendVariant::Composite,
);
}

View File

@@ -6,8 +6,8 @@ use powdr_pil_analyzer::evaluator::Value;
use powdr_pipeline::{
test_util::{
evaluate_function, evaluate_integer_function, execute_test_file, gen_estark_proof,
gen_halo2_proof, make_prepared_pipeline, resolve_test_file, std_analyzed, test_halo2,
verify_test_file, BackendVariant,
gen_halo2_proof, make_prepared_pipeline, resolve_test_file, run_pilcom_test_file,
std_analyzed, test_halo2, BackendVariant,
},
Pipeline,
};
@@ -22,23 +22,26 @@ fn poseidon_bn254_test() {
// This makes sure we test the whole proof generation for one example
// file even in the PR tests.
gen_halo2_proof(
make_prepared_pipeline(f, vec![]),
make_prepared_pipeline(f, vec![], vec![]),
BackendVariant::Monolithic,
);
gen_halo2_proof(make_prepared_pipeline(f, vec![]), BackendVariant::Composite);
gen_halo2_proof(
make_prepared_pipeline(f, vec![], vec![]),
BackendVariant::Composite,
);
}
#[test]
fn poseidon_gl_test() {
let f = "std/poseidon_gl_test.asm";
verify_test_file(f, Default::default(), vec![]).unwrap();
run_pilcom_test_file(f, Default::default(), vec![]).unwrap();
gen_estark_proof(f, Default::default());
}
#[test]
fn poseidon_gl_memory_test() {
let f = "std/poseidon_gl_memory_test.asm";
verify_test_file(f, Default::default(), vec![]).unwrap();
run_pilcom_test_file(f, Default::default(), vec![]).unwrap();
gen_estark_proof(f, Default::default());
}
@@ -51,7 +54,7 @@ fn split_bn254_test() {
#[test]
fn split_gl_test() {
let f = "std/split_gl_test.asm";
verify_test_file(f, Default::default(), vec![]).unwrap();
run_pilcom_test_file(f, Default::default(), vec![]).unwrap();
gen_estark_proof(f, Default::default());
}
@@ -59,7 +62,7 @@ fn split_gl_test() {
#[ignore = "Too slow"]
fn arith_test() {
let f = "std/arith_test.asm";
verify_test_file(f, Default::default(), vec![]).unwrap();
run_pilcom_test_file(f, Default::default(), vec![]).unwrap();
// Running gen_estark_proof(f, Default::default())
// is too slow for the PR tests. This will only create a single
@@ -76,7 +79,7 @@ fn arith_test() {
#[test]
fn memory_test() {
let f = "std/memory_test.asm";
verify_test_file(f, Default::default(), vec![]).unwrap();
run_pilcom_test_file(f, Default::default(), vec![]).unwrap();
gen_estark_proof(f, Default::default());
test_halo2(f, Default::default());
}
@@ -84,7 +87,7 @@ fn memory_test() {
#[test]
fn memory_with_bootloader_write_test() {
let f = "std/memory_with_bootloader_write_test.asm";
verify_test_file(f, Default::default(), vec![]).unwrap();
run_pilcom_test_file(f, Default::default(), vec![]).unwrap();
gen_estark_proof(f, Default::default());
test_halo2(f, Default::default());
}
@@ -92,7 +95,7 @@ fn memory_with_bootloader_write_test() {
#[test]
fn memory_test_parallel_accesses() {
let f = "std/memory_test_parallel_accesses.asm";
verify_test_file(f, Default::default(), vec![]).unwrap();
run_pilcom_test_file(f, Default::default(), vec![]).unwrap();
gen_estark_proof(f, Default::default());
test_halo2(f, Default::default());
}
@@ -171,7 +174,7 @@ fn bus_permutation_via_challenges_ext_bn() {
#[test]
fn write_once_memory_test() {
let f = "std/write_once_memory_test.asm";
verify_test_file(f, Default::default(), vec![]).unwrap();
run_pilcom_test_file(f, Default::default(), vec![]).unwrap();
gen_estark_proof(f, Default::default());
test_halo2(f, Default::default());
}
@@ -179,14 +182,14 @@ fn write_once_memory_test() {
#[test]
fn binary_test() {
let f = "std/binary_test.asm";
verify_test_file(f, Default::default(), vec![]).unwrap();
run_pilcom_test_file(f, Default::default(), vec![]).unwrap();
test_halo2(f, Default::default());
}
#[test]
fn shift_test() {
let f = "std/shift_test.asm";
verify_test_file(f, Default::default(), vec![]).unwrap();
run_pilcom_test_file(f, Default::default(), vec![]).unwrap();
test_halo2(f, Default::default());
}

View File

@@ -1,20 +1,21 @@
use mktemp::Temp;
use powdr_backend::BackendType;
use powdr_number::GoldilocksField;
use powdr_pipeline::{test_util::verify_pipeline, Pipeline};
use powdr_pipeline::{
test_util::{run_pilcom_with_backend_variant, BackendVariant},
Pipeline,
};
use powdr_riscv::Runtime;
use std::{
path::{Path, PathBuf},
process::Command,
};
/// Like compiler::test_util::verify_asm_string, but also runs RISCV executor.
/// Like compiler::test_util::run_pilcom_asm_string, but also runs RISCV executor.
pub fn verify_riscv_asm_string<S: serde::Serialize + Send + Sync + 'static>(
file_name: &str,
contents: &str,
inputs: &[GoldilocksField],
data: Option<&[(u32, S)]>,
backend: BackendType,
) {
let temp_dir = mktemp::Temp::new_dir().unwrap().release();
@@ -38,7 +39,7 @@ pub fn verify_riscv_asm_string<S: serde::Serialize + Send + Sync + 'static>(
powdr_riscv_executor::ExecMode::Fast,
Default::default(),
);
verify_pipeline(pipeline, backend).unwrap();
run_pilcom_with_backend_variant(pipeline, BackendVariant::Composite).unwrap();
}
fn find_assembler() -> &'static str {
@@ -88,11 +89,5 @@ pub fn verify_riscv_asm_file(asm_file: &Path, runtime: &Runtime, use_pie: bool)
let case_name = asm_file.file_stem().unwrap().to_str().unwrap();
let powdr_asm = powdr_riscv::elf::translate::<GoldilocksField>(&executable, runtime, false);
verify_riscv_asm_string::<()>(
&format!("{case_name}.asm"),
&powdr_asm,
&[],
None,
BackendType::EStarkDumpComposite,
);
verify_riscv_asm_string::<()>(&format!("{case_name}.asm"), &powdr_asm, &[], None);
}

View File

@@ -4,7 +4,6 @@ mod instruction_tests {
use std::path::Path;
use crate::common::{verify_riscv_asm_file, verify_riscv_asm_string};
use powdr_backend::BackendType;
use powdr_number::GoldilocksField;
use powdr_riscv::asm::compile;
use powdr_riscv::Runtime;
@@ -28,13 +27,7 @@ mod instruction_tests {
false,
);
verify_riscv_asm_string::<()>(
&format!("{name}.asm"),
&powdr_asm,
Default::default(),
None,
BackendType::EStarkDumpComposite,
);
verify_riscv_asm_string::<()>(&format!("{name}.asm"), &powdr_asm, Default::default(), None);
}
include!(concat!(env!("OUT_DIR"), "/instruction_tests.rs"));

View File

@@ -2,9 +2,11 @@ mod common;
use common::{verify_riscv_asm_file, verify_riscv_asm_string};
use mktemp::Temp;
use powdr_backend::BackendType;
use powdr_number::GoldilocksField;
use powdr_pipeline::{test_util::verify_pipeline, Pipeline};
use powdr_pipeline::{
test_util::{run_pilcom_with_backend_variant, BackendVariant},
Pipeline,
};
use std::path::{Path, PathBuf};
use test_log::test;
@@ -47,7 +49,7 @@ fn run_continuations_test(case: &str, powdr_asm: String) {
.with_prover_inputs(Default::default())
.with_output(tmp_dir.to_path_buf(), false);
let pipeline_callback = |pipeline: Pipeline<GoldilocksField>| -> Result<(), ()> {
verify_pipeline(pipeline, BackendType::EStarkDumpComposite).unwrap();
run_pilcom_with_backend_variant(pipeline, BackendVariant::Composite).unwrap();
Ok(())
};
@@ -143,14 +145,13 @@ fn keccak() {
#[ignore = "Too slow"]
fn vec_median_estark_polygon() {
let case = "vec_median";
verify_riscv_crate_with_backend(
verify_riscv_crate(
case,
[5, 11, 15, 75, 6, 5, 1, 4, 7, 3, 2, 9, 2]
.into_iter()
.map(|x| x.into())
.collect(),
&Runtime::base(),
BackendType::EStarkPolygonComposite,
);
}
@@ -364,17 +365,7 @@ fn many_chunks_memory() {
}
fn verify_riscv_crate(case: &str, inputs: Vec<GoldilocksField>, runtime: &Runtime) {
verify_riscv_crate_with_backend(case, inputs.clone(), runtime, BackendType::EStarkDump);
verify_riscv_crate_with_backend(case, inputs, runtime, BackendType::EStarkDumpComposite);
}
fn verify_riscv_crate_with_backend(
case: &str,
inputs: Vec<GoldilocksField>,
runtime: &Runtime,
backend: BackendType,
) {
verify_riscv_crate_from_both_paths::<()>(case, inputs, runtime, None, backend)
verify_riscv_crate_from_both_paths::<()>(case, inputs, runtime, None)
}
fn verify_riscv_crate_with_data<S: serde::Serialize + Send + Sync + 'static>(
@@ -383,13 +374,7 @@ fn verify_riscv_crate_with_data<S: serde::Serialize + Send + Sync + 'static>(
runtime: &Runtime,
data: Vec<(u32, S)>,
) {
verify_riscv_crate_from_both_paths(
case,
inputs,
runtime,
Some(data),
BackendType::EStarkDumpComposite,
)
verify_riscv_crate_from_both_paths(case, inputs, runtime, Some(data))
}
fn verify_riscv_crate_from_both_paths<S: serde::Serialize + Send + Sync + 'static>(
@@ -397,7 +382,6 @@ fn verify_riscv_crate_from_both_paths<S: serde::Serialize + Send + Sync + 'stati
inputs: Vec<GoldilocksField>,
runtime: &Runtime,
data: Option<Vec<(u32, S)>>,
backend: BackendType,
) {
let temp_dir = Temp::new_dir().unwrap();
let compiled = powdr_riscv::compile_rust_crate_to_riscv(
@@ -416,7 +400,6 @@ fn verify_riscv_crate_from_both_paths<S: serde::Serialize + Send + Sync + 'stati
&from_elf,
&inputs,
data.as_deref(),
backend,
);
log::info!("Verifying {case} converted from assembly files");
@@ -427,6 +410,5 @@ fn verify_riscv_crate_from_both_paths<S: serde::Serialize + Send + Sync + 'stati
&from_asm,
&inputs,
data.as_deref(),
backend,
);
}