Add tests with invalid witnesses

2026-04-20 03:03:25 -04:00 · 2024-02-28 17:33:57 +01:00
parent 41d856c698
commit 50a181dab3
10 changed files with 174 additions and 110 deletions
--- a/backend/src/pilstark/estark.rs
+++ b/backend/src/pilstark/estark.rs
@@ -147,16 +147,20 @@ impl<'a, F: FieldElement> Backend<'a, F> for EStark<'a, F> {

        log::info!("Proof done in: {:?}", duration);

-        assert!(stark_verify::<MerkleTreeGL, TranscriptGL>(
+        let valid = stark_verify::<MerkleTreeGL, TranscriptGL>(
            &starkproof,
            &setup.const_root,
            &setup.starkinfo,
            &self.params,
            &mut setup.program,
        )
-        .unwrap());
+        .map_err(|e| Error::BackendError(e.to_string()))?;

-        Ok(serde_json::to_vec(&starkproof).unwrap())
+        if valid {
+            Ok(serde_json::to_vec(&starkproof).unwrap())
+        } else {
+            Err(Error::BackendError("Proof verification failed".to_string()))
+        }
    }
 }

--- a/pipeline/src/pipeline.rs
+++ b/pipeline/src/pipeline.rs
@@ -390,7 +390,12 @@ impl<T: FieldElement> Pipeline<T> {
    }

    /// Sets the witness to the provided value.
-    pub fn set_witness(self, witness: Vec<(String, Vec<T>)>) -> Self {
+    pub fn set_witness(mut self, witness: Vec<(String, Vec<T>)>) -> Self {
+        if self.output_dir.is_some() {
+            // Some future steps (e.g. Pilcom verification) require the witness to be persisted.
+            let fixed_cols = self.compute_fixed_cols().unwrap();
+            self.maybe_write_witness(&fixed_cols, &witness).unwrap();
+        }
        Pipeline {
            artifact: Artifacts {
                witness: Some(Rc::new(witness)),
--- a/pipeline/src/test_util.rs
+++ b/pipeline/src/test_util.rs
@@ -18,11 +18,11 @@ pub fn resolve_test_file(file_name: &str) -> PathBuf {
    ))
 }

-pub fn verify_test_file<T: FieldElement>(
+pub fn verify_test_file(
    file_name: &str,
-    inputs: Vec<T>,
-    external_witness_values: Vec<(String, Vec<T>)>,
-) {
+    inputs: Vec<GoldilocksField>,
+    external_witness_values: Vec<(String, Vec<GoldilocksField>)>,
+) -> Result<(), String> {
    let pipeline = Pipeline::default()
        .from_file(resolve_test_file(file_name))
        .with_prover_inputs(inputs)
@@ -30,20 +30,20 @@ pub fn verify_test_file<T: FieldElement>(
    verify_pipeline(pipeline)
 }

-pub fn verify_asm_string<T: FieldElement>(
+pub fn verify_asm_string(
    file_name: &str,
    contents: &str,
-    inputs: Vec<T>,
-    external_witness_values: Vec<(String, Vec<T>)>,
+    inputs: Vec<GoldilocksField>,
+    external_witness_values: Vec<(String, Vec<GoldilocksField>)>,
 ) {
    let pipeline = Pipeline::default()
        .from_asm_string(contents.to_string(), Some(PathBuf::from(file_name)))
        .with_prover_inputs(inputs)
        .add_external_witness_values(external_witness_values);
-    verify_pipeline(pipeline)
+    verify_pipeline(pipeline).unwrap();
 }

-pub fn verify_pipeline<T: FieldElement>(pipeline: Pipeline<T>) {
+pub fn verify_pipeline(pipeline: Pipeline<GoldilocksField>) -> Result<(), String> {
    let mut pipeline = pipeline.with_backend(BackendType::PilStarkCli);

    let tmp_dir = mktemp::Temp::new_dir().unwrap();
@@ -53,7 +53,7 @@ pub fn verify_pipeline<T: FieldElement>(pipeline: Pipeline<T>) {

    pipeline.compute_proof().unwrap();

-    verify(pipeline.output_dir().unwrap(), pipeline.name(), None);
+    verify(pipeline.output_dir().unwrap(), pipeline.name(), None)
 }

 pub fn gen_estark_proof(file_name: &str, inputs: Vec<GoldilocksField>) {
@@ -183,3 +183,67 @@ pub fn evaluate_integer_function<T: FieldElement>(
        panic!("Expected integer.");
    }
 }
+
+fn convert_witness<T: FieldElement>(witness: &[(String, Vec<u64>)]) -> Vec<(String, Vec<T>)> {
+    witness
+        .iter()
+        .map(|(k, v)| (k.clone(), v.iter().cloned().map(T::from).collect()))
+        .collect()
+}
+
+fn assert_proofs_fail_for_invalid_witnesses_gl(file_name: &str, witness: &[(String, Vec<u64>)]) {
+    let file_name = format!("{}/../test_data/{file_name}", env!("CARGO_MANIFEST_DIR"));
+
+    let tmp_dir = mktemp::Temp::new_dir().unwrap();
+    let pipeline = Pipeline::<GoldilocksField>::default()
+        .with_tmp_output(&tmp_dir)
+        .from_file(PathBuf::from(file_name))
+        .set_witness(convert_witness(witness));
+
+    assert!(pipeline
+        .clone()
+        .with_backend(powdr_backend::BackendType::EStark)
+        .compute_proof()
+        .is_err());
+    assert!(verify_pipeline(pipeline.clone()).is_err());
+}
+
+#[cfg(feature = "halo2")]
+fn assert_proofs_fail_for_invalid_witnesses_bn254(file_name: &str, witness: &[(String, Vec<u64>)]) {
+    let file_name = format!("{}/../test_data/{file_name}", env!("CARGO_MANIFEST_DIR"));
+
+    let tmp_dir = mktemp::Temp::new_dir().unwrap();
+    let pipeline = Pipeline::<Bn254Field>::default()
+        .with_tmp_output(&tmp_dir)
+        .from_file(PathBuf::from(file_name))
+        .set_witness(convert_witness(witness));
+
+    // This will panic, because Halo2's MockProver::assert_satisfied() panics if it is not.
+    // We could use MockProver::verify() instead in our backend implementation to get a Result,
+    // but assert_satisfied() is the only way to print a helpful error message using the public API...
+    // It can still be helpful to uncomment this line to make sure the constraint that's failing
+    // is the one you'd expect.
+    // assert!(pipeline
+    //     .clone()
+    //     .with_backend(powdr_backend::BackendType::Halo2Mock)
+    //     .compute_proof()
+    //     .is_err());
+
+    assert!(pipeline
+        .clone()
+        .with_backend(powdr_backend::BackendType::Halo2)
+        .compute_proof()
+        .is_err());
+}
+
+#[cfg(not(feature = "halo2"))]
+fn assert_proofs_fail_for_invalid_witnesses_bn254(
+    _file_name: &str,
+    _witness: &[(String, Vec<u64>)],
+) {
+}
+
+pub fn assert_proofs_fail_for_invalid_witnesses(file_name: &str, witness: &[(String, Vec<u64>)]) {
+    assert_proofs_fail_for_invalid_witnesses_gl(file_name, witness);
+    assert_proofs_fail_for_invalid_witnesses_bn254(file_name, witness);
+}
--- a/pipeline/src/verify.rs
+++ b/pipeline/src/verify.rs
@@ -1,6 +1,6 @@
 use std::{path::Path, process::Command};

-pub fn verify(temp_dir: &Path, name: &str, constants_name: Option<&str>) {
+pub fn verify(temp_dir: &Path, name: &str, constants_name: Option<&str>) -> Result<(), String> {
    let pilcom = std::env::var("PILCOM")
        .expect("Please set the PILCOM environment variable to the path to the pilcom repository.");

@@ -31,12 +31,13 @@ pub fn verify(temp_dir: &Path, name: &str, constants_name: Option<&str>) {
            String::from_utf8_lossy(&verifier_output.stdout),
            String::from_utf8_lossy(&verifier_output.stderr)
        );
-        panic!("Pil verifier run was unsuccessful.");
+        return Err("Pil verifier run was unsuccessful.".to_string());
    } else {
        let output = String::from_utf8(verifier_output.stdout).unwrap();
        log::error!("PIL verifier output: {}", output);
        if !output.trim().ends_with("PIL OK!!") {
-            panic!("Verified did not say 'PIL OK' for {name}.");
+            return Err("Verified did not say 'PIL OK' for {name}.".to_string());
        }
    }
+    Ok(())
 }
--- a/pipeline/tests/asm.rs
+++ b/pipeline/tests/asm.rs
@@ -2,8 +2,8 @@ use powdr_number::{FieldElement, GoldilocksField};
 use powdr_pipeline::test_util::{gen_estark_proof, test_halo2, verify_test_file};
 use test_log::test;

-fn verify_asm<T: FieldElement>(file_name: &str, inputs: Vec<T>) {
-    verify_test_file(file_name, inputs, vec![]);
+fn verify_asm(file_name: &str, inputs: Vec<GoldilocksField>) {
+    verify_test_file(file_name, inputs, vec![]).unwrap();
 }

 fn slice_to_vec<T: FieldElement>(arr: &[i32]) -> Vec<T> {
@@ -14,7 +14,7 @@ fn slice_to_vec<T: FieldElement>(arr: &[i32]) -> Vec<T> {
 fn simple_sum_asm() {
    let f = "asm/simple_sum.asm";
    let i = [16, 4, 1, 2, 8, 5];
-    verify_asm::<GoldilocksField>(f, slice_to_vec(&i));
+    verify_asm(f, slice_to_vec(&i));
    test_halo2(f, slice_to_vec(&i));
    gen_estark_proof(f, slice_to_vec(&i));
 }
@@ -22,7 +22,7 @@ fn simple_sum_asm() {
 #[test]
 fn secondary_block_machine_add2() {
    let f = "asm/secondary_block_machine_add2.asm";
-    verify_asm::<GoldilocksField>(f, Default::default());
+    verify_asm(f, Default::default());
    test_halo2(f, Default::default());
    gen_estark_proof(f, Default::default());
 }
@@ -30,7 +30,7 @@ fn secondary_block_machine_add2() {
 #[test]
 fn mem_write_once() {
    let f = "asm/mem_write_once.asm";
-    verify_asm::<GoldilocksField>(f, Default::default());
+    verify_asm(f, Default::default());
    test_halo2(f, Default::default());
    gen_estark_proof(f, Default::default());
 }
@@ -42,13 +42,13 @@ fn mem_write_once_external_write() {
    mem[17] = GoldilocksField::from(42);
    mem[62] = GoldilocksField::from(123);
    mem[255] = GoldilocksField::from(-1);
-    verify_test_file::<GoldilocksField>(f, Default::default(), vec![("main.v".to_string(), mem)]);
+    verify_test_file(f, Default::default(), vec![("main.v".to_string(), mem)]).unwrap();
 }

 #[test]
 fn block_machine_cache_miss() {
    let f = "asm/block_machine_cache_miss.asm";
-    verify_asm::<GoldilocksField>(f, Default::default());
+    verify_asm(f, Default::default());
    test_halo2(f, Default::default());
    gen_estark_proof(f, Default::default());
 }
@@ -57,7 +57,7 @@ fn block_machine_cache_miss() {
 fn palindrome() {
    let f = "asm/palindrome.asm";
    let i = [7, 1, 7, 3, 9, 3, 7, 1];
-    verify_asm::<GoldilocksField>(f, slice_to_vec(&i));
+    verify_asm(f, slice_to_vec(&i));
    test_halo2(f, slice_to_vec(&i));
    // currently starky leads to
    // thread 'functional_instructions' has overflowed its stack
@@ -69,7 +69,7 @@ fn palindrome() {
 fn single_function_vm() {
    let f = "asm/single_function_vm.asm";
    let i = [];
-    verify_asm::<GoldilocksField>(f, slice_to_vec(&i));
+    verify_asm(f, slice_to_vec(&i));
    test_halo2(f, slice_to_vec(&i));
    gen_estark_proof(f, slice_to_vec(&i));
 }
@@ -78,7 +78,7 @@ fn single_function_vm() {
 fn empty() {
    let f = "asm/empty.asm";
    let i = [];
-    verify_asm::<GoldilocksField>(f, slice_to_vec(&i));
+    verify_asm(f, slice_to_vec(&i));
    test_halo2(f, slice_to_vec(&i));
    gen_estark_proof(f, slice_to_vec(&i));
 }
@@ -87,7 +87,7 @@ fn empty() {
 fn single_operation() {
    let f = "asm/single_operation.asm";
    let i = [];
-    verify_asm::<GoldilocksField>(f, slice_to_vec(&i));
+    verify_asm(f, slice_to_vec(&i));
    test_halo2(f, slice_to_vec(&i));
    gen_estark_proof(f, slice_to_vec(&i));
 }
@@ -96,7 +96,7 @@ fn single_operation() {
 fn empty_vm() {
    let f = "asm/empty_vm.asm";
    let i = [];
-    verify_asm::<GoldilocksField>(f, slice_to_vec(&i));
+    verify_asm(f, slice_to_vec(&i));
    test_halo2(f, slice_to_vec(&i));
    gen_estark_proof(f, slice_to_vec(&i));
 }
@@ -105,7 +105,7 @@ fn empty_vm() {
 fn vm_to_block_unique_interface() {
    let f = "asm/vm_to_block_unique_interface.asm";
    let i = [];
-    verify_asm::<GoldilocksField>(f, slice_to_vec(&i));
+    verify_asm(f, slice_to_vec(&i));
    test_halo2(f, slice_to_vec(&i));
    // currently starky leads to
    // thread 'functional_instructions' has overflowed its stack
@@ -117,7 +117,7 @@ fn vm_to_block_unique_interface() {
 fn vm_to_block_to_block() {
    let f = "asm/vm_to_block_to_block.asm";
    let i = [];
-    verify_asm::<GoldilocksField>(f, slice_to_vec(&i));
+    verify_asm(f, slice_to_vec(&i));
    test_halo2(f, slice_to_vec(&i));
 }

@@ -125,7 +125,7 @@ fn vm_to_block_to_block() {
 fn block_to_block() {
    let f = "asm/block_to_block.asm";
    let i = [];
-    verify_asm::<GoldilocksField>(f, slice_to_vec(&i));
+    verify_asm(f, slice_to_vec(&i));
    test_halo2(f, slice_to_vec(&i));
    gen_estark_proof(f, slice_to_vec(&i));
 }
@@ -134,7 +134,7 @@ fn block_to_block() {
 fn vm_instr_param_mapping() {
    let f = "asm/vm_instr_param_mapping.asm";
    let i = [];
-    verify_asm::<GoldilocksField>(f, slice_to_vec(&i));
+    verify_asm(f, slice_to_vec(&i));
    test_halo2(f, slice_to_vec(&i));
    gen_estark_proof(f, slice_to_vec(&i));
 }
@@ -143,7 +143,7 @@ fn vm_instr_param_mapping() {
 fn vm_to_block_multiple_interfaces() {
    let f = "asm/vm_to_block_multiple_interfaces.asm";
    let i = [];
-    verify_asm::<GoldilocksField>(f, slice_to_vec(&i));
+    verify_asm(f, slice_to_vec(&i));
    test_halo2(f, slice_to_vec(&i));
    gen_estark_proof(f, slice_to_vec(&i));
 }
@@ -152,7 +152,7 @@ fn vm_to_block_multiple_interfaces() {
 fn vm_to_vm() {
    let f = "asm/vm_to_vm.asm";
    let i = [];
-    verify_asm::<GoldilocksField>(f, slice_to_vec(&i));
+    verify_asm(f, slice_to_vec(&i));
    test_halo2(f, slice_to_vec(&i));
    gen_estark_proof(f, slice_to_vec(&i));
 }
@@ -161,7 +161,7 @@ fn vm_to_vm() {
 fn vm_to_vm_dynamic_trace_length() {
    let f = "asm/vm_to_vm_dynamic_trace_length.asm";
    let i = [];
-    verify_asm::<GoldilocksField>(f, slice_to_vec(&i));
+    verify_asm(f, slice_to_vec(&i));
    test_halo2(f, slice_to_vec(&i));
    gen_estark_proof(f, slice_to_vec(&i));
 }
@@ -170,7 +170,7 @@ fn vm_to_vm_dynamic_trace_length() {
 fn vm_to_vm_to_block() {
    let f = "asm/vm_to_vm_to_block.asm";
    let i = [];
-    verify_asm::<GoldilocksField>(f, slice_to_vec(&i));
+    verify_asm(f, slice_to_vec(&i));
    test_halo2(f, slice_to_vec(&i));
    gen_estark_proof(f, slice_to_vec(&i));
 }
@@ -179,7 +179,7 @@ fn vm_to_vm_to_block() {
 fn vm_to_block_array() {
    let f = "asm/vm_to_block_array.asm";
    let i = [];
-    verify_asm::<GoldilocksField>(f, slice_to_vec(&i));
+    verify_asm(f, slice_to_vec(&i));
    test_halo2(f, slice_to_vec(&i));
    gen_estark_proof(f, slice_to_vec(&i));
 }
@@ -188,7 +188,7 @@ fn vm_to_block_array() {
 fn vm_to_vm_to_vm() {
    let f = "asm/vm_to_vm_to_vm.asm";
    let i = [];
-    verify_asm::<GoldilocksField>(f, slice_to_vec(&i));
+    verify_asm(f, slice_to_vec(&i));
    test_halo2(f, slice_to_vec(&i));
    gen_estark_proof(f, slice_to_vec(&i));
 }
@@ -196,7 +196,7 @@ fn vm_to_vm_to_vm() {
 #[test]
 fn test_mem_read_write() {
    let f = "asm/mem_read_write.asm";
-    verify_asm::<GoldilocksField>(f, Default::default());
+    verify_asm(f, Default::default());
    test_halo2(f, Default::default());
    gen_estark_proof(f, Default::default());
 }
@@ -204,7 +204,7 @@ fn test_mem_read_write() {
 #[test]
 fn test_mem_read_write_no_memory_accesses() {
    let f = "asm/mem_read_write_no_memory_accesses.asm";
-    verify_asm::<GoldilocksField>(f, Default::default());
+    verify_asm(f, Default::default());
    test_halo2(f, Default::default());
    gen_estark_proof(f, Default::default());
 }
@@ -212,7 +212,7 @@ fn test_mem_read_write_no_memory_accesses() {
 #[test]
 fn test_mem_read_write_with_bootloader() {
    let f = "asm/mem_read_write_with_bootloader.asm";
-    verify_asm::<GoldilocksField>(f, Default::default());
+    verify_asm(f, Default::default());
    test_halo2(f, Default::default());
    gen_estark_proof(f, Default::default());
 }
@@ -220,7 +220,7 @@ fn test_mem_read_write_with_bootloader() {
 #[test]
 fn test_mem_read_write_large_diffs() {
    let f = "asm/mem_read_write_large_diffs.asm";
-    verify_asm::<GoldilocksField>(f, Default::default());
+    verify_asm(f, Default::default());
    test_halo2(f, Default::default());
    gen_estark_proof(f, Default::default());
 }
@@ -229,7 +229,7 @@ fn test_mem_read_write_large_diffs() {
 fn test_multi_assign() {
    let f = "asm/multi_assign.asm";
    let i = [7];
-    verify_asm::<GoldilocksField>(f, slice_to_vec(&i));
+    verify_asm(f, slice_to_vec(&i));
    test_halo2(f, slice_to_vec(&i));
    gen_estark_proof(f, slice_to_vec(&i));
 }
@@ -238,7 +238,7 @@ fn test_multi_assign() {
 fn test_multi_return() {
    let f = "asm/multi_return.asm";
    let i = [];
-    verify_asm::<GoldilocksField>(f, slice_to_vec(&i));
+    verify_asm(f, slice_to_vec(&i));
    test_halo2(f, slice_to_vec(&i));
    gen_estark_proof(f, Default::default());
 }
@@ -248,7 +248,7 @@ fn test_multi_return() {
 fn test_multi_return_wrong_assignment_registers() {
    let f = "asm/multi_return_wrong_assignment_registers.asm";
    let i = [];
-    verify_asm::<GoldilocksField>(f, slice_to_vec(&i));
+    verify_asm(f, slice_to_vec(&i));
 }

 #[test]
@@ -256,14 +256,14 @@ fn test_multi_return_wrong_assignment_registers() {
 fn test_multi_return_wrong_assignment_register_length() {
    let f = "asm/multi_return_wrong_assignment_register_length.asm";
    let i = [];
-    verify_asm::<GoldilocksField>(f, slice_to_vec(&i));
+    verify_asm(f, slice_to_vec(&i));
 }

 #[test]
 fn test_bit_access() {
    let f = "asm/bit_access.asm";
    let i = [20];
-    verify_asm::<GoldilocksField>(f, slice_to_vec(&i));
+    verify_asm(f, slice_to_vec(&i));
    test_halo2(f, slice_to_vec(&i));
    // currently starky leads to
    // thread 'functional_instructions' has overflowed its stack
@@ -274,7 +274,7 @@ fn test_bit_access() {
 #[test]
 fn test_sqrt() {
    let f = "asm/sqrt.asm";
-    verify_asm::<GoldilocksField>(f, Default::default());
+    verify_asm(f, Default::default());
    test_halo2(f, Default::default());
    gen_estark_proof(f, Default::default());
 }
@@ -283,7 +283,7 @@ fn test_sqrt() {
 fn functional_instructions() {
    let f = "asm/functional_instructions.asm";
    let i = [20];
-    verify_asm::<GoldilocksField>(f, slice_to_vec(&i));
+    verify_asm(f, slice_to_vec(&i));
    test_halo2(f, slice_to_vec(&i));
    // currently starky leads to
    // thread 'functional_instructions' has overflowed its stack
@@ -294,7 +294,7 @@ fn functional_instructions() {
 #[test]
 fn full_pil_constant() {
    let f = "asm/full_pil_constant.asm";
-    verify_asm::<GoldilocksField>(f, Default::default());
+    verify_asm(f, Default::default());
    test_halo2(f, Default::default());
    gen_estark_proof(f, Default::default());
 }
@@ -302,7 +302,7 @@ fn full_pil_constant() {
 #[test]
 fn intermediate() {
    let f = "asm/intermediate.asm";
-    verify_asm::<GoldilocksField>(f, Default::default());
+    verify_asm(f, Default::default());
    test_halo2(f, Default::default());
    gen_estark_proof(f, Default::default());
 }
@@ -310,7 +310,7 @@ fn intermediate() {
 #[test]
 fn intermediate_nested() {
    let f = "asm/intermediate_nested.asm";
-    verify_asm::<GoldilocksField>(f, Default::default());
+    verify_asm(f, Default::default());
    test_halo2(f, Default::default());
    gen_estark_proof(f, Default::default());
 }
@@ -318,21 +318,20 @@ fn intermediate_nested() {
 #[test]
 fn pil_at_module_level() {
    let f = "asm/pil_at_module_level.asm";
-    verify_asm::<GoldilocksField>(f, Default::default());
+    verify_asm(f, Default::default());
    test_halo2(f, Default::default());
    gen_estark_proof(f, Default::default());
 }

 mod book {
    use super::*;
-    use powdr_number::GoldilocksField;
    use test_log::test;

    fn run_book_test(file: &str) {
        // passing 0 to all tests currently works as they either take no prover input or 0 works
        let i = [0];

-        verify_asm::<GoldilocksField>(file, slice_to_vec(&i));
+        verify_asm(file, slice_to_vec(&i));
        test_halo2(file, slice_to_vec(&i));
        gen_estark_proof(file, slice_to_vec(&i));
    }
@@ -345,7 +344,7 @@ mod book {
 fn hello_world_asm_fail() {
    let f = "asm/book/hello_world.asm";
    let i = [1];
-    verify_asm::<GoldilocksField>(f, slice_to_vec(&i));
+    verify_asm(f, slice_to_vec(&i));
 }

 #[test]
@@ -353,5 +352,5 @@ fn hello_world_asm_fail() {
 fn test_failing_assertion() {
    let f = "asm/failing_assertion.asm";
    let i = [];
-    verify_asm::<GoldilocksField>(f, slice_to_vec(&i));
+    verify_asm(f, slice_to_vec(&i));
 }
--- a/pipeline/tests/pil.rs
+++ b/pipeline/tests/pil.rs
@@ -3,48 +3,29 @@ use powdr_number::Bn254Field;
 use powdr_number::GoldilocksField;
 use powdr_pipeline::{
    test_util::{
-        gen_estark_proof, resolve_test_file, test_halo2, verify_pipeline, verify_test_file,
+        assert_proofs_fail_for_invalid_witnesses, gen_estark_proof, resolve_test_file, test_halo2,
+        verify_test_file,
    },
    Pipeline,
 };
 use test_log::test;

 pub fn verify_pil(file_name: &str, inputs: Vec<GoldilocksField>) {
-    verify_test_file(file_name, inputs, vec![]);
+    verify_test_file(file_name, inputs, vec![]).unwrap();
 }

 #[test]
-#[should_panic = "Pil verifier run was unsuccessful."]
-fn test_invalid_witness_pilcom() {
+fn test_invalid_witness() {
    let f = "pil/trivial.pil";
-    let pipeline = Pipeline::default()
-        .from_file(resolve_test_file(f))
-        .set_witness(vec![(
-            "main.w".to_string(),
-            vec![GoldilocksField::from(0); 4],
-        )]);
-    verify_pipeline(pipeline);
-}
-
-#[test]
-#[should_panic = "assertion failed: stark_verify::<MerkleTreeGL,\\n            TranscriptGL>(&starkproof, &setup.const_root, &setup.starkinfo,\\n        &self.params, &mut setup.program).unwrap()"]
-fn test_invalid_witness_estark() {
-    let f = "pil/trivial.pil";
-    Pipeline::default()
-        .from_file(resolve_test_file(f))
-        .set_witness(vec![(
-            "main.w".to_string(),
-            vec![GoldilocksField::from(0); 4],
-        )])
-        .with_backend(powdr_backend::BackendType::EStark)
-        .compute_proof()
-        .unwrap();
+    let witness = vec![("main.w".to_string(), vec![0; 4])];
+    assert_proofs_fail_for_invalid_witnesses(f, &witness);
 }

 #[test]
 #[should_panic = "circuit was not satisfied"]
 #[cfg(feature = "halo2")]
 fn test_invalid_witness_halo2mock() {
+    // assert_proofs_fail_for_invalid_witnesses() doesn't assert that Halo2Mock fails, so this is a separate test using should_panic.
    let f = "pil/trivial.pil";
    Pipeline::default()
        .from_file(resolve_test_file(f))
@@ -54,19 +35,6 @@ fn test_invalid_witness_halo2mock() {
        .unwrap();
 }

-#[test]
-#[should_panic = "called `Result::unwrap()` on an `Err` value: [\"Proof is invalid\"]"]
-#[cfg(feature = "halo2")]
-fn test_invalid_witness_halo2() {
-    let f = "pil/trivial.pil";
-    Pipeline::default()
-        .from_file(resolve_test_file(f))
-        .set_witness(vec![("main.w".to_string(), vec![Bn254Field::from(0); 4])])
-        .with_backend(powdr_backend::BackendType::Halo2)
-        .compute_proof()
-        .unwrap();
-}
-
 #[test]
 fn test_fibonacci() {
    let f = "pil/fibonacci.pil";
@@ -75,6 +43,29 @@ fn test_fibonacci() {
    gen_estark_proof(f, Default::default());
 }

+#[test]
+fn test_fibonacci_invalid_witness() {
+    let f = "pil/fibonacci.pil";
+
+    // Changed one value and then continued.
+    // The following constraint should fail in row 1:
+    //     (1-ISLAST) * (x' - y) = 0;
+    let witness = vec![
+        ("Fibonacci.x".to_string(), vec![1, 1, 10, 3]),
+        ("Fibonacci.y".to_string(), vec![1, 2, 3, 13]),
+    ];
+    assert_proofs_fail_for_invalid_witnesses(f, &witness);
+
+    // All constraints are valid, except the initial row.
+    // The following constraint should fail in row 3:
+    //     ISLAST * (y' - 1) = 0;
+    let witness = vec![
+        ("Fibonacci.x".to_string(), vec![1, 2, 3, 5]),
+        ("Fibonacci.y".to_string(), vec![2, 3, 5, 8]),
+    ];
+    assert_proofs_fail_for_invalid_witnesses(f, &witness);
+}
+
 #[test]
 fn test_constant_in_identity() {
    let f = "pil/constant_in_identity.pil";
@@ -102,14 +93,14 @@ fn test_external_witgen_fails_if_none_provided() {
 fn test_external_witgen_a_provided() {
    let f = "pil/external_witgen.pil";
    let external_witness = vec![("main.a".to_string(), vec![GoldilocksField::from(3); 16])];
-    verify_test_file(f, Default::default(), external_witness);
+    verify_test_file(f, Default::default(), external_witness).unwrap();
 }

 #[test]
 fn test_external_witgen_b_provided() {
    let f = "pil/external_witgen.pil";
    let external_witness = vec![("main.b".to_string(), vec![GoldilocksField::from(4); 16])];
-    verify_test_file(f, Default::default(), external_witness);
+    verify_test_file(f, Default::default(), external_witness).unwrap();
 }

 #[test]
@@ -119,7 +110,7 @@ fn test_external_witgen_both_provided() {
        ("main.a".to_string(), vec![GoldilocksField::from(3); 16]),
        ("main.b".to_string(), vec![GoldilocksField::from(4); 16]),
    ];
-    verify_test_file(f, Default::default(), external_witness);
+    verify_test_file(f, Default::default(), external_witness).unwrap();
 }

 #[test]
@@ -131,7 +122,7 @@ fn test_external_witgen_fails_on_conflicting_external_witness() {
        // Does not satisfy b = a + 1
        ("main.b".to_string(), vec![GoldilocksField::from(3); 16]),
    ];
-    verify_test_file(f, Default::default(), external_witness);
+    verify_test_file(f, Default::default(), external_witness).unwrap();
 }

 #[test]
--- a/pipeline/tests/powdr_std.rs
+++ b/pipeline/tests/powdr_std.rs
@@ -22,7 +22,7 @@ fn poseidon_bn254_test() {
 #[test]
 fn poseidon_gl_test() {
    let f = "std/poseidon_gl_test.asm";
-    verify_test_file::<GoldilocksField>(f, Default::default(), vec![]);
+    verify_test_file(f, Default::default(), vec![]).unwrap();
    gen_estark_proof(f, Default::default());
 }

@@ -35,7 +35,7 @@ fn split_bn254_test() {
 #[test]
 fn split_gl_test() {
    let f = "std/split_gl_test.asm";
-    verify_test_file::<GoldilocksField>(f, Default::default(), vec![]);
+    verify_test_file(f, Default::default(), vec![]).unwrap();
    gen_estark_proof(f, Default::default());
 }

@@ -43,7 +43,7 @@ fn split_gl_test() {
 #[ignore = "Too slow"]
 fn arith_test() {
    let f = "std/arith_test.asm";
-    verify_test_file::<GoldilocksField>(f, Default::default(), vec![]);
+    verify_test_file(f, Default::default(), vec![]).unwrap();
    gen_estark_proof(f, Default::default());
    test_halo2(f, Default::default());
 }
@@ -51,14 +51,14 @@ fn arith_test() {
 #[test]
 fn binary_test() {
    let f = "std/binary_test.asm";
-    verify_test_file::<GoldilocksField>(f, Default::default(), vec![]);
+    verify_test_file(f, Default::default(), vec![]).unwrap();
    test_halo2(f, Default::default());
 }

 #[test]
 fn shift_test() {
    let f = "std/shift_test.asm";
-    verify_test_file::<GoldilocksField>(f, Default::default(), vec![]);
+    verify_test_file(f, Default::default(), vec![]).unwrap();
    test_halo2(f, Default::default());
 }

--- a/riscv/tests/common/mod.rs
+++ b/riscv/tests/common/mod.rs
@@ -19,5 +19,5 @@ pub fn verify_riscv_asm_string(file_name: &str, contents: &str, inputs: Vec<Gold
        usize::MAX,
        powdr_riscv_executor::ExecMode::Fast,
    );
-    verify_pipeline(pipeline);
+    verify_pipeline(pipeline).unwrap();
 }
--- a/riscv/tests/riscv.rs
+++ b/riscv/tests/riscv.rs
@@ -35,7 +35,7 @@ pub fn test_continuations(case: &str) {
        // computing the constants file.
        let mut pipeline = pipeline.with_backend(BackendType::PilStarkCli);
        pipeline.compute_proof().unwrap();
-        verify(pipeline.output_dir().unwrap(), pipeline.name(), Some(case));
+        verify(pipeline.output_dir().unwrap(), pipeline.name(), Some(case)).unwrap();
        Ok(())
    };
    let bootloader_inputs = rust_continuations_dry_run(&mut pipeline);
@@ -182,7 +182,7 @@ fn test_evm() {
        .from_asm_string(powdr_asm, None)
        .add_data(666, &bytes);

-    powdr_pipeline::test_util::verify_pipeline(pipeline);
+    powdr_pipeline::test_util::verify_pipeline(pipeline).unwrap();
 }

 #[test]
--- a/test_data/pil/fibonacci.pil
+++ b/test_data/pil/fibonacci.pil
@@ -1,4 +1,4 @@
-constant %N = 16;
+constant %N = 4;

 // This uses the alternative nomenclature as well.