From 2c699ce6a10eb91086c5e42c0855ce4a1d7a373f Mon Sep 17 00:00:00 2001 From: Georg Wiese Date: Thu, 26 Oct 2023 13:27:58 +0000 Subject: [PATCH] Improve witgen memory consumption by introducing FinalizableData --- executor/Cargo.toml | 1 + .../{ => data_structures}/column_map.rs | 0 .../data_structures/finalizable_data.rs | 185 ++++++++++++++++++ executor/src/witgen/data_structures/mod.rs | 2 + executor/src/witgen/generator.rs | 48 ++--- executor/src/witgen/global_constraints.rs | 3 +- executor/src/witgen/machines/block_machine.rs | 47 +++-- executor/src/witgen/mod.rs | 4 +- executor/src/witgen/processor.rs | 40 ++-- executor/src/witgen/rows.rs | 32 +-- executor/src/witgen/vm_processor.rs | 32 ++- 11 files changed, 292 insertions(+), 102 deletions(-) rename executor/src/witgen/{ => data_structures}/column_map.rs (100%) create mode 100644 executor/src/witgen/data_structures/finalizable_data.rs create mode 100644 executor/src/witgen/data_structures/mod.rs diff --git a/executor/Cargo.toml b/executor/Cargo.toml index f6088fee9..81e21e598 100644 --- a/executor/Cargo.toml +++ b/executor/Cargo.toml @@ -10,6 +10,7 @@ number = { path = "../number" } parser_util = { path = "../parser_util" } pil_analyzer = { path = "../pil_analyzer" } rayon = "1.7.0" +bit-vec = "0.6.3" num-traits = "0.2.15" ast = { version = "0.1.0", path = "../ast" } lazy_static = "1.4.0" diff --git a/executor/src/witgen/column_map.rs b/executor/src/witgen/data_structures/column_map.rs similarity index 100% rename from executor/src/witgen/column_map.rs rename to executor/src/witgen/data_structures/column_map.rs diff --git a/executor/src/witgen/data_structures/finalizable_data.rs b/executor/src/witgen/data_structures/finalizable_data.rs new file mode 100644 index 000000000..6e1de6eaa --- /dev/null +++ b/executor/src/witgen/data_structures/finalizable_data.rs @@ -0,0 +1,185 @@ +use std::{ + collections::HashSet, + ops::{Index, IndexMut}, +}; + +use ast::analyzed::PolyID; +use bit_vec::BitVec; +use number::FieldElement; + +use crate::witgen::rows::Row; + +/// A row entry in [FinalizableData]. +#[derive(Clone)] +enum Entry<'a, T: FieldElement> { + /// The row is still in progress, and range constraints are still available. + InProgress(Row<'a, T>), + /// A finalized row, represented as a vector of values (corresponding to the columns + /// stored in [FinalizableData]) and a bit vector indicating which cells are known. + /// The value of unknown cells should be ignored. + Finalized(Vec, BitVec), +} + +/// A data structure that stores rows of a witness table, and behaves much like a `Vec>`. +/// However, it also allows to finalize rows, which means that memory for things like range +/// constraints is freed. The information which cells are known is preserved, though. +/// Once a row has been finalized, any operation trying to access it again will fail at runtime. +/// [FinalizableData::take_transposed] can be used to access the final cells. +#[derive(Clone)] +pub struct FinalizableData<'a, T: FieldElement> { + /// The list of rows (either in progress or finalized) + data: Vec>, + /// The list of column IDs (in sorted order), used to index finalized rows. + column_ids: Vec, +} + +impl<'a, T: FieldElement> FinalizableData<'a, T> { + pub fn new(column_ids: &HashSet) -> Self { + Self::with_initial_rows_in_progress(column_ids, [].into_iter()) + } + + pub fn with_initial_rows_in_progress( + column_ids: &HashSet, + rows: impl Iterator>, + ) -> Self { + let mut column_ids = column_ids.iter().cloned().collect::>(); + column_ids.sort(); + let data = rows.map(Entry::InProgress).collect::>(); + Self { data, column_ids } + } + + pub fn len(&self) -> usize { + self.data.len() + } + + pub fn push(&mut self, row: Row<'a, T>) { + self.data.push(Entry::InProgress(row)); + } + + pub fn pop(&mut self) -> Option> { + match self.data.pop() { + Some(Entry::InProgress(row)) => Some(row), + Some(Entry::Finalized(_, _)) => panic!("Row already finalized."), + None => None, + } + } + + pub fn extend(&mut self, other: Self) { + self.data.extend(other.data); + } + + pub fn remove(&mut self, i: usize) -> Row<'a, T> { + match self.data.remove(i) { + Entry::InProgress(row) => row, + Entry::Finalized(_, _) => panic!("Row {} already finalized.", i), + } + } + + pub fn truncate(&mut self, len: usize) { + self.data.truncate(len); + } + + pub fn get_mut(&mut self, i: usize) -> Option<&mut Row<'a, T>> { + match &mut self.data[i] { + Entry::InProgress(row) => Some(row), + Entry::Finalized(_, _) => panic!("Row {} already finalized.", i), + } + } + + pub fn mutable_row_pair(&mut self, i: usize) -> (&mut Row<'a, T>, &mut Row<'a, T>) { + let (before, after) = self.data.split_at_mut(i + 1); + let current = before.last_mut().unwrap(); + let next = after.first_mut().unwrap(); + match (current, next) { + (Entry::InProgress(current), Entry::InProgress(next)) => (current, next), + _ => panic!("Row {} or {} (or both) already finalized.", i, i + 1), + } + } + + pub fn finalize(&mut self, i: usize) -> bool { + if let Entry::InProgress(row) = &self.data[i] { + let (values, known_cells) = self + .column_ids + .iter() + .map(|c| (row[c].value.unwrap_or_default(), row[c].value.is_known())) + .unzip(); + self.data[i] = Entry::Finalized(values, known_cells); + true + } else { + false + } + } + + pub fn finalize_range(&mut self, range: impl Iterator) { + for i in range { + self.finalize(i); + } + } + + /// Takes all data out of the [FinalizableData] and returns it as a list of columns. + /// Columns are represented as a tuple of: + /// - A list of values + /// - A bit vector indicating which cells are known. Values of unknown cells should be ignored. + pub fn take_transposed(&mut self) -> impl Iterator, BitVec))> { + log::info!( + "Transposing {} rows with {} columns...", + self.data.len(), + self.column_ids.len() + ); + log::info!("Finalizing remaining rows..."); + let mut counter = 0; + for i in 0..self.data.len() { + if self.finalize(i) { + counter += 1; + } + } + log::info!("Needed to finalize {} / {} rows.", counter, self.data.len()); + + // Store transposed columns in vectors for performance reasons + let mut columns = vec![vec![]; self.column_ids.len()]; + let mut known_cells_col = vec![BitVec::new(); self.column_ids.len()]; + for row in std::mem::take(&mut self.data) { + match row { + Entry::InProgress(_) => unreachable!(), + Entry::Finalized(row, known_cells) => { + for (col_index, (value, is_known)) in + row.into_iter().zip(known_cells).enumerate() + { + known_cells_col[col_index].push(is_known); + columns[col_index].push(value); + } + } + } + } + + log::info!("Done transposing."); + + // Pair columns with their IDs + let column_ids = std::mem::take(&mut self.column_ids); + columns.into_iter().zip(known_cells_col).enumerate().map( + move |(col_index, (column, known_cells))| { + (column_ids[col_index], (column, known_cells)) + }, + ) + } +} + +impl<'a, T: FieldElement> Index for FinalizableData<'a, T> { + type Output = Row<'a, T>; + + fn index(&self, index: usize) -> &Self::Output { + match &self.data[index] { + Entry::InProgress(row) => row, + Entry::Finalized(_, _) => panic!("Row {} already finalized.", index), + } + } +} + +impl<'a, T: FieldElement> IndexMut for FinalizableData<'a, T> { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + match &mut self.data[index] { + Entry::InProgress(row) => row, + Entry::Finalized(_, _) => panic!("Row {} already finalized.", index), + } + } +} diff --git a/executor/src/witgen/data_structures/mod.rs b/executor/src/witgen/data_structures/mod.rs new file mode 100644 index 000000000..caf5b0d39 --- /dev/null +++ b/executor/src/witgen/data_structures/mod.rs @@ -0,0 +1,2 @@ +pub mod column_map; +pub mod finalizable_data; diff --git a/executor/src/witgen/generator.rs b/executor/src/witgen/generator.rs index 149928050..f64c3880d 100644 --- a/executor/src/witgen/generator.rs +++ b/executor/src/witgen/generator.rs @@ -3,15 +3,16 @@ use ast::parsed::SelectedExpressions; use number::{DegreeType, FieldElement}; use std::collections::{HashMap, HashSet}; +use crate::witgen::data_structures::finalizable_data::FinalizableData; use crate::witgen::rows::CellValue; use super::affine_expression::AffineExpression; -use super::column_map::WitnessColumnMap; +use super::data_structures::column_map::WitnessColumnMap; use super::global_constraints::GlobalConstraints; use super::machines::Machine; use super::processor::Processor; -use super::rows::{transpose_rows, Row, RowFactory}; +use super::rows::{Row, RowFactory}; use super::sequence_iterator::{DefaultSequenceIterator, ProcessingSequenceIterator}; use super::vm_processor::VmProcessor; use super::{EvalResult, FixedData, MutableState, QueryCallback}; @@ -21,7 +22,7 @@ pub struct Generator<'a, T: FieldElement> { identities: Vec<&'a Identity>>, witnesses: HashSet, global_range_constraints: GlobalConstraints, - data: Vec>, + data: FinalizableData<'a, T>, latch: Option>, } @@ -37,14 +38,9 @@ impl<'a, T: FieldElement> Machine<'a, T> for Generator<'a, T> { } fn take_witness_col_values(&mut self) -> HashMap> { - transpose_rows(std::mem::take(&mut self.data), &self.witnesses) - .into_iter() - .map(|(id, values)| { - ( - self.fixed_data.column_name(&id).to_string(), - values.into_iter().map(|v| v.unwrap_or_default()).collect(), - ) - }) + self.data + .take_transposed() + .map(|(id, (values, _))| (self.fixed_data.column_name(&id).to_string(), values)) .collect() } } @@ -57,12 +53,13 @@ impl<'a, T: FieldElement> Generator<'a, T> { global_range_constraints: &GlobalConstraints, latch: Option>, ) -> Self { + let data = FinalizableData::new(&witnesses); Self { fixed_data, identities: identities.to_vec(), witnesses, global_range_constraints: global_range_constraints.clone(), - data: vec![], + data, latch, } } @@ -83,11 +80,8 @@ impl<'a, T: FieldElement> Generator<'a, T> { let first_row = self.data.pop().unwrap(); - self.data.append(&mut self.process( - first_row, - self.data.len() as DegreeType, - mutable_state, - )); + self.data + .extend(self.process(first_row, self.data.len() as DegreeType, mutable_state)); } } @@ -104,10 +98,14 @@ impl<'a, T: FieldElement> Generator<'a, T> { // it does not assert that the row is "complete" afterwards (i.e., that all identities // are satisfied assuming 0 for unknown values). let row_factory = RowFactory::new(self.fixed_data, self.global_range_constraints.clone()); - let data = vec![ - row_factory.fresh_row(self.fixed_data.degree - 1), - row_factory.fresh_row(0), - ]; + let data = FinalizableData::with_initial_rows_in_progress( + &self.witnesses, + [ + row_factory.fresh_row(self.fixed_data.degree - 1), + row_factory.fresh_row(0), + ] + .into_iter(), + ); let mut processor = Processor::new( self.fixed_data.degree - 1, data, @@ -131,17 +129,21 @@ impl<'a, T: FieldElement> Generator<'a, T> { first_row: Row<'a, T>, row_offset: DegreeType, mutable_state: &mut MutableState<'a, '_, T, Q>, - ) -> Vec> { + ) -> FinalizableData<'a, T> { log::trace!( "Running main machine from row {row_offset} with the following initial values in the first row:\n{}", first_row.render_values(false, None) ); let row_factory = RowFactory::new(self.fixed_data, self.global_range_constraints.clone()); + let data = FinalizableData::with_initial_rows_in_progress( + &self.witnesses, + [first_row].into_iter(), + ); let mut processor = VmProcessor::new( row_offset, self.fixed_data, &self.identities, self.witnesses.clone(), - vec![first_row], + data, row_factory, self.latch.clone(), ); diff --git a/executor/src/witgen/global_constraints.rs b/executor/src/witgen/global_constraints.rs index fed9454f4..f00ba35c3 100644 --- a/executor/src/witgen/global_constraints.rs +++ b/executor/src/witgen/global_constraints.rs @@ -8,9 +8,8 @@ use ast::analyzed::{ use ast::parsed::BinaryOperator; use number::FieldElement; -use crate::witgen::column_map::FixedColumnMap; +use crate::witgen::data_structures::column_map::{FixedColumnMap, WitnessColumnMap}; -use super::column_map::WitnessColumnMap; use super::expression_evaluator::ExpressionEvaluator; use super::range_constraints::RangeConstraint; use super::symbolic_evaluator::SymbolicEvaluator; diff --git a/executor/src/witgen/machines/block_machine.rs b/executor/src/witgen/machines/block_machine.rs index 21e141e28..a927f054a 100644 --- a/executor/src/witgen/machines/block_machine.rs +++ b/executor/src/witgen/machines/block_machine.rs @@ -3,10 +3,11 @@ use std::collections::{HashMap, HashSet}; use super::{EvalResult, FixedData}; use crate::witgen::affine_expression::AffineExpression; +use crate::witgen::data_structures::finalizable_data::FinalizableData; use crate::witgen::global_constraints::GlobalConstraints; use crate::witgen::identity_processor::IdentityProcessor; use crate::witgen::processor::{OuterQuery, Processor}; -use crate::witgen::rows::{transpose_rows, CellValue, Row, RowFactory, RowPair, UnknownStrategy}; +use crate::witgen::rows::{CellValue, RowFactory, RowPair, UnknownStrategy}; use crate::witgen::sequence_iterator::{ProcessingSequenceCache, ProcessingSequenceIterator}; use crate::witgen::util::try_to_simple_poly; use crate::witgen::{machines::Machine, EvalError, EvalValue, IncompleteCause}; @@ -16,7 +17,10 @@ use ast::parsed::SelectedExpressions; use number::{DegreeType, FieldElement}; enum ProcessResult<'a, T: FieldElement> { - Success(Vec>, Constraints<&'a PolynomialReference, T>), + Success( + FinalizableData<'a, T>, + Constraints<&'a PolynomialReference, T>, + ), Incomplete, } @@ -43,7 +47,7 @@ pub struct BlockMachine<'a, T: FieldElement> { /// The row factory row_factory: RowFactory<'a, T>, /// The data of the machine. - data: Vec>, + data: FinalizableData<'a, T>, /// The set of witness columns that are actually part of this machine. witness_cols: HashSet, /// Cache that states the order in which to evaluate identities @@ -68,9 +72,10 @@ impl<'a, T: FieldElement> BlockMachine<'a, T> { // Start out with a block filled with unknown values so that we do not have to deal with wrap-around // when storing machine witness data. // This will be filled with the default block in `take_witness_col_values` - let data = (0..block_size) - .map(|i| row_factory.fresh_row(i as DegreeType)) - .collect(); + let data = FinalizableData::with_initial_rows_in_progress( + witness_cols, + (0..block_size).map(|i| row_factory.fresh_row(i as DegreeType)), + ); return Some(BlockMachine { block_size, selected_expressions: id.right.clone(), @@ -164,9 +169,17 @@ impl<'a, T: FieldElement> Machine<'a, T> for BlockMachine<'a, T> { This might violate some internal constraints." ); } - let mut data = transpose_rows(std::mem::take(&mut self.data), &self.witness_cols) - .into_iter() - .map(|(id, mut values)| { + let mut data = self + .data + .take_transposed() + .map(|(id, (values, known_cells))| { + // Materialize column as Vec> + let mut values = values + .into_iter() + .zip(known_cells) + .map(|(v, known)| known.then_some(v)) + .collect::>(); + // For all constraints to be satisfied, unused cells have to be filled with valid values. // We do this, we construct a default block, by repeating the first input to the block machine. values.resize(self.fixed_data.degree as usize, None); @@ -353,9 +366,11 @@ impl<'a, T: FieldElement> BlockMachine<'a, T> { let row_offset = self.rows() - 1; // Make the block two rows larger than the block size, it includes the last row of the previous block // and the first row of the next block. - let block = (0..(self.block_size + 2)) - .map(|i| self.row_factory.fresh_row(i as DegreeType + row_offset)) - .collect(); + let block = FinalizableData::with_initial_rows_in_progress( + &self.witness_cols, + (0..(self.block_size + 2)) + .map(|i| self.row_factory.fresh_row(i as DegreeType + row_offset)), + ); let mut processor = Processor::new( row_offset, block, @@ -385,7 +400,7 @@ impl<'a, T: FieldElement> BlockMachine<'a, T> { /// the last row of its previous block is merged with the one we have already. /// This is necessary to handle non-rectangular block machines, which already use /// unused cells in the previous block. - fn append_block(&mut self, mut new_block: Vec>) -> Result<(), EvalError> { + fn append_block(&mut self, mut new_block: FinalizableData<'a, T>) -> Result<(), EvalError> { if self.rows() + self.block_size as DegreeType >= self.fixed_data.degree { return Err(EvalError::RowsExhausted); } @@ -413,7 +428,11 @@ impl<'a, T: FieldElement> BlockMachine<'a, T> { // 3. Remove the last row of the previous block from data self.data.pop(); - // 4. Append the new block (including the merged last row of the previous block) + // 4. Finalize most of the block + // The last row might be needed later, so we do not finalize it yet. + new_block.finalize_range(0..self.block_size); + + // 5. Append the new block (including the merged last row of the previous block) self.data.extend(new_block); Ok(()) diff --git a/executor/src/witgen/mod.rs b/executor/src/witgen/mod.rs index 395d626aa..a4563d072 100644 --- a/executor/src/witgen/mod.rs +++ b/executor/src/witgen/mod.rs @@ -6,7 +6,7 @@ use ast::analyzed::{ use num_traits::Zero; use number::{DegreeType, FieldElement}; -use self::column_map::{FixedColumnMap, WitnessColumnMap}; +use self::data_structures::column_map::{FixedColumnMap, WitnessColumnMap}; pub use self::eval_result::{ Constraint, Constraints, EvalError, EvalResult, EvalStatus, EvalValue, IncompleteCause, }; @@ -19,7 +19,7 @@ use self::machines::{FixedLookup, Machine}; use pil_analyzer::pil_analyzer::inline_intermediate_polynomials; mod affine_expression; -mod column_map; +mod data_structures; mod eval_result; mod expression_evaluator; pub mod fixed_evaluator; diff --git a/executor/src/witgen/processor.rs b/executor/src/witgen/processor.rs index 1bd89a201..a9c0d6a32 100644 --- a/executor/src/witgen/processor.rs +++ b/executor/src/witgen/processor.rs @@ -10,9 +10,9 @@ use crate::witgen::{query_processor::QueryProcessor, Constraint}; use super::{ affine_expression::AffineExpression, - column_map::WitnessColumnMap, + data_structures::{column_map::WitnessColumnMap, finalizable_data::FinalizableData}, identity_processor::IdentityProcessor, - rows::{Row, RowFactory, RowPair, RowUpdater, UnknownStrategy}, + rows::{RowFactory, RowPair, RowUpdater, UnknownStrategy}, sequence_iterator::{Action, ProcessingSequenceIterator, SequenceStep}, Constraints, EvalError, EvalValue, FixedData, MutableState, QueryCallback, }; @@ -48,7 +48,7 @@ pub struct Processor<'a, 'b, 'c, T: FieldElement, Q: QueryCallback, CalldataA /// The global index of the first row of [Processor::data]. row_offset: u64, /// The rows that are being processed. - data: Vec>, + data: FinalizableData<'a, T>, /// The list of identities identities: &'c [&'a Identity>], /// The mutable state @@ -71,7 +71,7 @@ impl<'a, 'b, 'c, T: FieldElement, Q: QueryCallback> { pub fn new( row_offset: u64, - data: Vec>, + data: FinalizableData<'a, T>, mutable_state: &'c mut MutableState<'a, 'b, T, Q>, identities: &'c [&'a Identity>], fixed_data: &'a FixedData<'a, T>, @@ -116,14 +116,14 @@ impl<'a, 'b, 'c, T: FieldElement, Q: QueryCallback> } } - pub fn finish(self) -> Vec> { + pub fn finish(self) -> FinalizableData<'a, T> { self.data } } impl<'a, 'b, T: FieldElement, Q: QueryCallback> Processor<'a, 'b, '_, T, Q, WithCalldata> { /// Destroys itself, returns the data and updated left-hand side of the outer query (if available). - pub fn finish(self) -> (Vec>, Left<'a, T>) { + pub fn finish(self) -> (FinalizableData<'a, T>, Left<'a, T>) { (self.data, self.outer_query.unwrap().left) } } @@ -297,9 +297,7 @@ impl<'a, 'b, T: FieldElement, Q: QueryCallback, CalldataAvailable> // Build RowUpdater // (a bit complicated, because we need two mutable // references to elements of the same vector) - let (before, after) = self.data.split_at_mut(row_index + 1); - let current = before.last_mut().unwrap(); - let next = after.first_mut().unwrap(); + let (current, next) = self.data.mutable_row_pair(row_index); let mut row_updater = RowUpdater::new(current, next, self.row_offset + row_index as u64); for (poly, c) in &updates.constraints { @@ -322,14 +320,15 @@ impl<'a, 'b, T: FieldElement, Q: QueryCallback, CalldataAvailable> mod tests { use std::collections::BTreeMap; - use ast::analyzed::PolyID; + use ast::analyzed::{PolyID, PolynomialType}; use number::{FieldElement, GoldilocksField}; use pil_analyzer::analyze_string; use crate::{ constant_evaluator::generate, witgen::{ - column_map::FixedColumnMap, + data_structures::column_map::FixedColumnMap, + data_structures::finalizable_data::FinalizableData, global_constraints::GlobalConstraints, identity_processor::Machines, machines::FixedLookup, @@ -373,9 +372,16 @@ mod tests { let mut machines = vec![]; let row_factory = RowFactory::new(&fixed_data, global_range_constraints); - let data = (0..fixed_data.degree) - .map(|i| row_factory.fresh_row(i)) + let columns = (0..fixed_data.witness_cols.len()) + .map(move |i| PolyID { + id: i as u64, + ptype: PolynomialType::Committed, + }) .collect(); + let data = FinalizableData::with_initial_rows_in_progress( + &columns, + (0..fixed_data.degree).map(|i| row_factory.fresh_row(i)), + ); let mut mutable_state = MutableState { fixed_lookup: &mut fixed_lookup, @@ -414,14 +420,6 @@ mod tests { // Can't use processor.finish(), because we don't own it... let data = processor.data.clone(); - // In case of any error, this will be useful - for (i, row) in data.iter().enumerate() { - println!( - "{}", - row.render(&format!("Row {i}"), true, processor.witness_cols) - ); - } - for &(i, name, expected) in asserted_values.iter() { let poly_id = poly_ids[name]; let row = &data[i]; diff --git a/executor/src/witgen/rows.rs b/executor/src/witgen/rows.rs index 5e9a5aefd..ad29863cd 100644 --- a/executor/src/witgen/rows.rs +++ b/executor/src/witgen/rows.rs @@ -1,7 +1,4 @@ -use std::{ - collections::{BTreeMap, HashSet}, - fmt::Debug, -}; +use std::{collections::HashSet, fmt::Debug}; use ast::analyzed::{Expression, PolyID, PolynomialReference}; use itertools::Itertools; @@ -11,7 +8,7 @@ use crate::witgen::Constraint; use super::{ affine_expression::{AffineExpression, AffineResult}, - column_map::WitnessColumnMap, + data_structures::column_map::WitnessColumnMap, expression_evaluator::ExpressionEvaluator, global_constraints::{GlobalConstraints, RangeConstraintSet}, range_constraints::RangeConstraint, @@ -150,31 +147,6 @@ impl Row<'_, T> { } } -/// Transposes a list of rows into a map from column to a list of values. -pub fn transpose_rows( - rows: Vec>, - column_set: &HashSet, -) -> BTreeMap>> { - // Use column maps for efficiency - let mut columns: WitnessColumnMap>> = - WitnessColumnMap::from(rows[0].iter().map(|_| Vec::new())); - let is_relevant_column = - WitnessColumnMap::from(rows[0].keys().map(|poly_id| column_set.contains(&poly_id))); - - for row in rows.into_iter() { - for (poly_id, cell) in row.into_iter() { - if is_relevant_column[&poly_id] { - columns[&poly_id].push(cell.value.into()); - } - } - } - // Convert to BTreeMap and filter out columns outside column set - columns - .into_iter() - .filter(|(poly_id, _)| is_relevant_column[poly_id]) - .collect() -} - /// A factory for rows, which knows the global range constraints and has pointers to column names. #[derive(Clone)] pub struct RowFactory<'a, T: FieldElement> { diff --git a/executor/src/witgen/vm_processor.rs b/executor/src/witgen/vm_processor.rs index 3a80c52a0..feb52d1d1 100644 --- a/executor/src/witgen/vm_processor.rs +++ b/executor/src/witgen/vm_processor.rs @@ -10,12 +10,16 @@ use crate::witgen::identity_processor::{self, IdentityProcessor}; use crate::witgen::rows::RowUpdater; use crate::witgen::IncompleteCause; -use super::column_map::WitnessColumnMap; +use super::data_structures::column_map::WitnessColumnMap; +use super::data_structures::finalizable_data::FinalizableData; use super::query_processor::QueryProcessor; use super::rows::{Row, RowFactory, RowPair, UnknownStrategy}; use super::{EvalError, EvalResult, EvalValue, FixedData, MutableState, QueryCallback}; +/// Maximal period checked during loop detection. +const MAX_PERIOD: usize = 4; + /// A list of identities with a flag whether it is complete. struct CompletableIdentities<'a, T: FieldElement> { identities_with_complete: Vec<(&'a Identity>, bool)>, @@ -50,7 +54,7 @@ pub struct VmProcessor<'a, T: FieldElement> { /// The subset of identities that does not contain a reference to the next row /// (precomputed once for performance reasons) identities_without_next_ref: Vec<&'a Identity>>, - data: Vec>, + data: FinalizableData<'a, T>, last_report: DegreeType, last_report_time: Instant, row_factory: RowFactory<'a, T>, @@ -63,7 +67,7 @@ impl<'a, T: FieldElement> VmProcessor<'a, T> { fixed_data: &'a FixedData<'a, T>, identities: &[&'a Identity>], witnesses: HashSet, - data: Vec>, + data: FinalizableData<'a, T>, row_factory: RowFactory<'a, T>, latch: Option>, ) -> Self { @@ -93,7 +97,7 @@ impl<'a, T: FieldElement> VmProcessor<'a, T> { } } - pub fn finish(self) -> Vec> { + pub fn finish(self) -> FinalizableData<'a, T> { self.data } @@ -109,8 +113,18 @@ impl<'a, T: FieldElement> VmProcessor<'a, T> { let mut looping_period = None; let mut loop_detection_log_level = log::Level::Info; let rows_left = self.fixed_data.degree - self.row_offset + 1; + let mut finalize_start = 1; for row_index in 0..rows_left { self.maybe_log_performance(row_index); + + if (row_index + 1) % 10000 == 0 { + // Periodically make sure most rows are finalized. + // Row 0 and the last MAX_PERIOD rows might be needed later, so they are not finalized. + let finalize_end = row_index as usize - MAX_PERIOD; + self.data.finalize_range(finalize_start..finalize_end); + finalize_start = finalize_end; + } + // Check if we are in a loop. if looping_period.is_none() && row_index % 100 == 0 && row_index > 0 { looping_period = self.rows_are_repeating(row_index); @@ -175,14 +189,14 @@ impl<'a, T: FieldElement> VmProcessor<'a, T> { } /// Checks if the last rows are repeating and returns the period. - /// Only checks for periods of 1, 2, 3 and 4. + /// Only checks for periods of 1, ..., MAX_PERIOD. fn rows_are_repeating(&self, row_index: DegreeType) -> Option { - if row_index < 4 { + if row_index < MAX_PERIOD as DegreeType { return None; } let row = row_index as usize; - (1..=3).find(|&period| { + (1..MAX_PERIOD).find(|&period| { (1..=period).all(|i| { self.data[row - i - period] .values() @@ -376,9 +390,7 @@ impl<'a, T: FieldElement> VmProcessor<'a, T> { if updates.constraints.is_empty() { return false; } - let (before, after) = self.data.split_at_mut(row_index as usize + 1); - let current = before.last_mut().unwrap(); - let next = after.first_mut().unwrap(); + let (current, next) = self.data.mutable_row_pair(row_index as usize); let mut row_updater = RowUpdater::new(current, next, row_index + self.row_offset); row_updater.apply_updates(updates, source_name) }