From db92a636e565a248cde392d4463481c7fa6a7e9b Mon Sep 17 00:00:00 2001 From: schaeff Date: Wed, 19 Jun 2024 19:56:36 +0200 Subject: [PATCH 01/39] wip --- ast/src/analyzed/mod.rs | 1 + executor/src/constant_evaluator/mod.rs | 2 +- number/Cargo.toml | 1 + number/src/serialize.rs | 81 ++++--------------------- pil-analyzer/src/condenser.rs | 2 + pil-analyzer/src/pil_analyzer.rs | 9 +-- pil-analyzer/src/statement_processor.rs | 3 + pipeline/src/pipeline.rs | 14 +---- pipeline/src/util.rs | 23 +++---- pipeline/tests/asm.rs | 8 +-- 10 files changed, 32 insertions(+), 112 deletions(-) diff --git a/ast/src/analyzed/mod.rs b/ast/src/analyzed/mod.rs index 4c89ab7dd..203857325 100644 --- a/ast/src/analyzed/mod.rs +++ b/ast/src/analyzed/mod.rs @@ -446,6 +446,7 @@ pub struct Symbol { pub stage: Option, pub kind: SymbolKind, pub length: Option, + pub degree: Option, } impl Symbol { diff --git a/executor/src/constant_evaluator/mod.rs b/executor/src/constant_evaluator/mod.rs index b811e3c9e..0ca3ded9c 100644 --- a/executor/src/constant_evaluator/mod.rs +++ b/executor/src/constant_evaluator/mod.rs @@ -28,7 +28,7 @@ pub fn generate(analyzed: &Analyzed) -> Vec<(String, Vec) // for non-arrays, set index to None. for (index, (name, id)) in poly.array_elements().enumerate() { let index = poly.is_array().then_some(index as u64); - let values = generate_values(analyzed, analyzed.degree(), &name, value, index); + let values = generate_values(analyzed, poly.degree.unwrap(), &name, value, index); assert!(fixed_cols.insert(name, (id, values)).is_none()); } } diff --git a/number/Cargo.toml b/number/Cargo.toml index 0de87dbf3..7c6bc97b0 100644 --- a/number/Cargo.toml +++ b/number/Cargo.toml @@ -20,6 +20,7 @@ serde = { version = "1.0", default-features = false, features = ["alloc", "deriv serde_with = "3.6.1" schemars = { version = "0.8.16", features = ["preserve_order"]} ibig = { version = "0.3.6", features = ["serde"]} +serde_json = "1.0.117" [dev-dependencies] test-log = "0.2.12" diff --git a/number/src/serialize.rs b/number/src/serialize.rs index a08a48877..22f48ce65 100644 --- a/number/src/serialize.rs +++ b/number/src/serialize.rs @@ -8,7 +8,7 @@ use ark_serialize::{CanonicalDeserialize, CanonicalSerialize, Compress, Validate use csv::{Reader, Writer}; use serde_with::{DeserializeAs, SerializeAs}; -use crate::{DegreeType, FieldElement}; +use crate::FieldElement; #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum CsvRenderMode { @@ -89,10 +89,6 @@ pub fn read_polys_csv_file(file: impl Read) -> Vec<(String, Vec .collect() } -fn ceil_div(num: usize, div: usize) -> usize { - (num + div - 1) / div -} - pub fn buffered_write_file( path: &Path, do_write: impl FnOnce(&mut BufWriter) -> R, @@ -117,59 +113,14 @@ fn write_polys_stream( file: &mut impl Write, polys: &[(String, Vec)], ) -> Result<(), io::Error> { - let width = ceil_div(T::BITS as usize, 64) * 8; - - if polys.is_empty() { - return Ok(()); - } - - // TODO maybe the witness should have a proper type that - // explicitly has a degree or length? - let degree = polys[0].1.len(); - for (_, values) in polys { - assert_eq!(values.len(), degree); - } - - for i in 0..degree { - for (_name, constant) in polys { - let bytes = constant[i].to_bytes_le(); - assert_eq!(bytes.len(), width); - file.write_all(&bytes)?; - } - } - - Ok(()) + Ok(serde_json::to_writer(file, polys)?) } pub fn read_polys_file( file: &mut impl Read, - columns: &[String], -) -> (Vec<(String, Vec)>, DegreeType) { - assert!(!columns.is_empty()); - let width = ceil_div(T::BITS as usize, 64) * 8; - - let bytes_to_read = width * columns.len(); - - let mut result: Vec<(_, Vec)> = columns - .iter() - .map(|name| (name.to_string(), vec![])) - .collect(); - let mut degree = 0; - - loop { - let mut buf = vec![0u8; bytes_to_read]; - match file.read_exact(&mut buf) { - Ok(()) => {} - Err(_) => return (result, degree), - } - degree += 1; - result - .iter_mut() - .zip(buf.chunks(width)) - .for_each(|((_, values), bytes)| { - values.push(T::from_bytes_le(bytes)); - }); - } + // columns: &[String], +) -> Vec<(String, Vec)> { + serde_json::from_reader(file).unwrap() } // Serde wrappers for serialize/deserialize @@ -201,36 +152,28 @@ mod tests { use super::*; use test_log::test; - fn test_polys() -> (Vec<(String, Vec)>, u64) { - ( - vec![ - ("a".to_string(), (0..16).map(Bn254Field::from).collect()), - ("b".to_string(), (-16..0).map(Bn254Field::from).collect()), - ], - 16, - ) + fn test_polys() -> Vec<(String, Vec)> { + vec![ + ("a".to_string(), (0..16).map(Bn254Field::from).collect()), + ("b".to_string(), (-16..0).map(Bn254Field::from).collect()), + ] } #[test] fn write_read() { let mut buf: Vec = vec![]; - let (polys, degree) = test_polys(); + let polys = test_polys(); write_polys_stream(&mut buf, &polys).unwrap(); - let (read_polys, read_degree) = read_polys_file::( - &mut Cursor::new(buf), - &["a".to_string(), "b".to_string()], - ); + let read_polys = read_polys_file::(&mut Cursor::new(buf)); assert_eq!(read_polys, polys); - assert_eq!(read_degree, degree); } #[test] fn write_read_csv() { let polys = test_polys() - .0 .into_iter() .map(|(name, values)| (name.to_string(), values)) .collect::>(); diff --git a/pil-analyzer/src/condenser.rs b/pil-analyzer/src/condenser.rs index 4600ee64c..d5b23d50d 100644 --- a/pil-analyzer/src/condenser.rs +++ b/pil-analyzer/src/condenser.rs @@ -369,6 +369,8 @@ impl<'a, T: FieldElement> SymbolLookup<'a, T> for Condenser<'a, T> { stage: None, kind: SymbolKind::Poly(PolynomialType::Committed), length: None, + // TODO: what is the actual degree? + degree: None, }; self.next_witness_id += 1; self.all_new_witness_names.insert(name.clone()); diff --git a/pil-analyzer/src/pil_analyzer.rs b/pil-analyzer/src/pil_analyzer.rs index 476f64826..b5ef71ca1 100644 --- a/pil-analyzer/src/pil_analyzer.rs +++ b/pil-analyzer/src/pil_analyzer.rs @@ -421,14 +421,7 @@ impl PILAnalyzer { .unwrap(), ) .unwrap(); - if let Some(degree) = self.polynomial_degree { - assert_eq!( - degree, namespace_degree, - "all namespaces must have the same degree" - ); - } else { - self.polynomial_degree = Some(namespace_degree); - } + self.polynomial_degree = Some(namespace_degree); } self.current_namespace = AbsoluteSymbolPath::default().join(name); } diff --git a/pil-analyzer/src/statement_processor.rs b/pil-analyzer/src/statement_processor.rs index 7e3dcd3c4..e8b378962 100644 --- a/pil-analyzer/src/statement_processor.rs +++ b/pil-analyzer/src/statement_processor.rs @@ -416,6 +416,7 @@ where let id = self.counters.dispense_symbol_id(symbol_kind, length); let absolute_name = self.driver.resolve_decl(&name); + let symbol = Symbol { id, source: source.clone(), @@ -423,6 +424,7 @@ where absolute_name: absolute_name.clone(), kind: symbol_kind, length, + degree: self.degree, }; if let Some(FunctionDefinition::TypeDeclaration(enum_decl)) = value { @@ -442,6 +444,7 @@ where stage: None, kind: SymbolKind::Other(), length: None, + degree: None, }; let value = FunctionValueDefinition::TypeConstructor( shared_enum_decl.clone(), diff --git a/pipeline/src/pipeline.rs b/pipeline/src/pipeline.rs index 14cc4e96a..0b5385764 100644 --- a/pipeline/src/pipeline.rs +++ b/pipeline/src/pipeline.rs @@ -385,12 +385,7 @@ impl Pipeline { pub fn read_constants(mut self, directory: &Path) -> Self { let pil = self.compute_optimized_pil().unwrap(); - let fixed = try_read_poly_set::(&pil, directory) - .map(|(fixed, degree_fixed)| { - assert_eq!(pil.degree.unwrap(), degree_fixed); - fixed - }) - .unwrap_or_default(); + let fixed = try_read_poly_set::(&pil, directory).unwrap_or_default(); Pipeline { artifact: Artifacts { @@ -405,12 +400,7 @@ impl Pipeline { pub fn read_witness(mut self, directory: &Path) -> Self { let pil = self.compute_optimized_pil().unwrap(); - let witness = try_read_poly_set::(&pil, directory) - .map(|(witness, degree_witness)| { - assert_eq!(pil.degree.unwrap(), degree_witness); - witness - }) - .unwrap_or_default(); + let witness = try_read_poly_set::(&pil, directory).unwrap_or_default(); Pipeline { artifact: Artifacts { diff --git a/pipeline/src/util.rs b/pipeline/src/util.rs index 0a7502a29..6c56295af 100644 --- a/pipeline/src/util.rs +++ b/pipeline/src/util.rs @@ -1,5 +1,5 @@ use powdr_ast::analyzed::{Analyzed, FunctionValueDefinition, Symbol}; -use powdr_number::{read_polys_file, DegreeType, FieldElement}; +use powdr_number::{read_polys_file, FieldElement}; use std::{fs::File, io::BufReader, path::Path}; pub trait PolySet { @@ -33,20 +33,11 @@ impl PolySet for WitnessPolySet { #[allow(clippy::type_complexity)] pub fn try_read_poly_set( - pil: &Analyzed, + _pil: &Analyzed, dir: &Path, -) -> Option<(Vec<(String, Vec)>, DegreeType)> { - let column_names: Vec = P::get_polys(pil) - .iter() - .flat_map(|(poly, _)| poly.array_elements()) - .map(|(name, _id)| name) - .collect(); - - (!column_names.is_empty()).then(|| { - let path = dir.join(P::FILE_NAME); - read_polys_file( - &mut BufReader::new(File::open(path).unwrap()), - &column_names, - ) - }) +) -> Option)>> { + let path = dir.join(P::FILE_NAME); + Some(read_polys_file(&mut BufReader::new( + File::open(path).unwrap(), + ))) } diff --git a/pipeline/tests/asm.rs b/pipeline/tests/asm.rs index d3449e733..512d93b94 100644 --- a/pipeline/tests/asm.rs +++ b/pipeline/tests/asm.rs @@ -368,16 +368,12 @@ fn read_poly_files() { pipeline.compute_proof().unwrap(); // check fixed cols (may have no fixed cols) - if let Some((fixed, degree)) = try_read_poly_set::(&pil, tmp_dir.as_path()) - { - assert_eq!(pil.degree(), degree); + if let Some(fixed) = try_read_poly_set::(&pil, tmp_dir.as_path()) { assert_eq!(pil.degree(), fixed[0].1.len() as u64); } // check witness cols (examples assumed to have at least one witness col) - let (witness, degree) = - try_read_poly_set::(&pil, tmp_dir.as_path()).unwrap(); - assert_eq!(pil.degree(), degree); + let witness = try_read_poly_set::(&pil, tmp_dir.as_path()).unwrap(); assert_eq!(pil.degree(), witness[0].1.len() as u64); } } From 3285424f3e6ec0433fc8e939b8f64990603861bf Mon Sep 17 00:00:00 2001 From: schaeff Date: Thu, 20 Jun 2024 17:54:09 +0200 Subject: [PATCH 02/39] make tests pass --- ast/src/analyzed/display.rs | 17 +++-- ast/src/analyzed/mod.rs | 15 +++-- backend/src/estark/json_exporter/mod.rs | 6 +- backend/src/estark/mod.rs | 4 +- backend/src/estark/starky_wrapper.rs | 2 +- backend/src/halo2/circuit_builder.rs | 4 +- backend/src/halo2/mock_prover.rs | 2 +- backend/src/halo2/prover.rs | 8 +-- executor/src/constant_evaluator/mod.rs | 36 +++++------ executor/src/witgen/block_processor.rs | 12 ++-- .../src/witgen/data_structures/column_map.rs | 20 +++++- executor/src/witgen/generator.rs | 37 +++++++---- executor/src/witgen/global_constraints.rs | 64 ++++++++++++++----- executor/src/witgen/machines/block_machine.rs | 43 +++++++++---- .../machines/double_sorted_witness_machine.rs | 18 +++++- .../witgen/machines/fixed_lookup_machine.rs | 15 ++++- executor/src/witgen/machines/mod.rs | 14 ++++ .../witgen/machines/sorted_witness_machine.rs | 28 ++++++-- .../src/witgen/machines/write_once_memory.rs | 22 ++++++- executor/src/witgen/mod.rs | 33 +++++----- executor/src/witgen/processor.rs | 12 ++++ executor/src/witgen/rows.rs | 5 ++ executor/src/witgen/vm_processor.rs | 23 +++++-- number/src/serialize.rs | 18 +++--- pil-analyzer/src/condenser.rs | 1 - pil-analyzer/tests/condenser.rs | 2 +- pil-analyzer/tests/parse_display.rs | 2 - pipeline/src/test_util.rs | 2 +- pipeline/tests/asm.rs | 4 +- pipeline/tests/pil.rs | 6 ++ riscv/src/continuations.rs | 2 +- test_data/pil/single_line_blocks.pil | 2 +- 32 files changed, 337 insertions(+), 142 deletions(-) diff --git a/ast/src/analyzed/display.rs b/ast/src/analyzed/display.rs index cbc75c5ee..f1de91556 100644 --- a/ast/src/analyzed/display.rs +++ b/ast/src/analyzed/display.rs @@ -21,9 +21,8 @@ use super::*; impl Display for Analyzed { fn fmt(&self, f: &mut Formatter<'_>) -> Result { - let degree = self.degree.unwrap_or_default(); let mut current_namespace = AbsoluteSymbolPath::default(); - let mut update_namespace = |name: &str, f: &mut Formatter<'_>| { + let mut update_namespace = |name: &str, degree: Option, f: &mut Formatter<'_>| { let mut namespace = AbsoluteSymbolPath::default().join(SymbolPath::from_str(name).unwrap()); let name = namespace.pop().unwrap(); @@ -31,8 +30,9 @@ impl Display for Analyzed { current_namespace = namespace; writeln!( f, - "namespace {}({degree});", - current_namespace.relative_to(&Default::default()) + "namespace {}({});", + current_namespace.relative_to(&Default::default()), + degree.map(|d| d.to_string()).unwrap_or_default() )?; }; Ok((name, !current_namespace.is_empty())) @@ -53,7 +53,7 @@ impl Display for Analyzed { // These are printed as part of the enum. continue; } - let (name, is_local) = update_namespace(name, f)?; + let (name, is_local) = update_namespace(name, symbol.degree, f)?; match symbol.kind { SymbolKind::Poly(_) => { writeln_indented(f, format_poly(&name, symbol, definition))?; @@ -110,7 +110,7 @@ impl Display for Analyzed { } } else if let Some((symbol, definition)) = self.intermediate_columns.get(name) { assert!(symbol.stage.is_none()); - let (name, _) = update_namespace(name, f)?; + let (name, _) = update_namespace(name, symbol.degree, f)?; assert_eq!(symbol.kind, SymbolKind::Poly(PolynomialType::Intermediate)); if let Some(length) = symbol.length { writeln_indented( @@ -130,7 +130,7 @@ impl Display for Analyzed { } StatementIdentifier::PublicDeclaration(name) => { let decl = &self.public_declarations[name]; - let (name, is_local) = update_namespace(&decl.name, f)?; + let (name, is_local) = update_namespace(&decl.name, None, f)?; writeln_indented_by( f, format_public_declaration(&name, decl), @@ -485,6 +485,7 @@ mod test { poly_id: super::PolyID { id: 0, ptype: super::PolynomialType::Committed, + degree: None, }, next: false, }); @@ -493,6 +494,7 @@ mod test { poly_id: super::PolyID { id: 1, ptype: super::PolynomialType::Committed, + degree: None, }, next: false, }); @@ -501,6 +503,7 @@ mod test { poly_id: super::PolyID { id: 2, ptype: super::PolynomialType::Committed, + degree: None, }, next: false, }); diff --git a/ast/src/analyzed/mod.rs b/ast/src/analyzed/mod.rs index 203857325..86c55b557 100644 --- a/ast/src/analyzed/mod.rs +++ b/ast/src/analyzed/mod.rs @@ -31,8 +31,6 @@ pub enum StatementIdentifier { #[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema)] pub struct Analyzed { - /// The degree of all namespaces, which must match if provided. If no degrees are given, then `None`. - pub degree: Option, pub definitions: HashMap)>, pub public_declarations: HashMap, pub intermediate_columns: HashMap>)>, @@ -45,10 +43,14 @@ pub struct Analyzed { } impl Analyzed { - /// @returns the degree if any. Panics if there is none. - pub fn degree(&self) -> DegreeType { - self.degree.unwrap() + pub fn max_degree(&self) -> DegreeType { + self.definitions + .values() + .filter_map(|(symbol, _)| symbol.degree) + .max() + .unwrap() } + /// @returns the number of committed polynomials (with multiplicities for arrays) pub fn commitment_count(&self) -> usize { self.declaration_type_count(PolynomialType::Committed) @@ -466,6 +468,7 @@ impl Symbol { PolyID { id: self.id + i, ptype, + degree: self.degree, }, ) }) @@ -1152,6 +1155,7 @@ pub struct PolynomialReference { pub struct PolyID { pub id: u64, pub ptype: PolynomialType, + pub degree: Option, } impl From<&Symbol> for PolyID { @@ -1162,6 +1166,7 @@ impl From<&Symbol> for PolyID { PolyID { id: symbol.id, ptype, + degree: symbol.degree, } } } diff --git a/backend/src/estark/json_exporter/mod.rs b/backend/src/estark/json_exporter/mod.rs index e70324f3f..314c20afe 100644 --- a/backend/src/estark/json_exporter/mod.rs +++ b/backend/src/estark/json_exporter/mod.rs @@ -232,7 +232,7 @@ impl<'a, T: FieldElement> Exporter<'a, T> { polType: None, type_: symbol_kind_to_json_string(symbol.kind).to_string(), id: id as usize, - polDeg: self.analyzed.degree() as usize, + polDeg: self.analyzed.max_degree() as usize, isArray: symbol.is_array(), elementType: None, len: symbol.length.map(|l| l as usize), @@ -251,7 +251,7 @@ impl<'a, T: FieldElement> Exporter<'a, T> { polType: None, type_: symbol_kind_to_json_string(symbol.kind).to_string(), id: id as usize, - polDeg: self.analyzed.degree() as usize, + polDeg: self.analyzed.max_degree() as usize, isArray: symbol.is_array(), elementType: None, len: symbol.length.map(|l| l as usize), @@ -369,7 +369,7 @@ impl<'a, T: FieldElement> Exporter<'a, T> { fn polynomial_reference_to_json( &self, - PolyID { id, ptype }: PolyID, + PolyID { id, ptype, .. }: PolyID, next: bool, ) -> (u32, StarkyExpr) { let id = if ptype == PolynomialType::Intermediate { diff --git a/backend/src/estark/mod.rs b/backend/src/estark/mod.rs index 29168a708..21110804b 100644 --- a/backend/src/estark/mod.rs +++ b/backend/src/estark/mod.rs @@ -76,7 +76,7 @@ fn first_step_fixup<'a, F: FieldElement>( pil: &'a Analyzed, fixed: &'a [(String, Vec)], ) -> (PIL, Option>) { - let degree = pil.degree(); + let degree = pil.max_degree(); let mut pil: PIL = json_exporter::export(pil); @@ -161,7 +161,7 @@ impl<'a, F: FieldElement> EStarkFilesCommon<'a, F> { let proof_type: ProofType = ProofType::from(options); Ok(EStarkFilesCommon { - degree: analyzed.degree(), + degree: analyzed.max_degree(), pil, patched_constants, output_dir, diff --git a/backend/src/estark/starky_wrapper.rs b/backend/src/estark/starky_wrapper.rs index 5527c8c6c..4a9d85826 100644 --- a/backend/src/estark/starky_wrapper.rs +++ b/backend/src/estark/starky_wrapper.rs @@ -47,7 +47,7 @@ impl BackendFactory for Factory { let proof_type: ProofType = ProofType::from(options); - let params = create_stark_struct(pil.degree(), proof_type.hash_type()); + let params = create_stark_struct(pil.max_degree(), proof_type.hash_type()); let (pil_json, patched_fixed) = first_step_fixup(pil, fixed); diff --git a/backend/src/halo2/circuit_builder.rs b/backend/src/halo2/circuit_builder.rs index f5023feed..91730e43f 100644 --- a/backend/src/halo2/circuit_builder.rs +++ b/backend/src/halo2/circuit_builder.rs @@ -272,7 +272,7 @@ impl<'a, T: FieldElement, F: PrimeField> Circuit for PowdrCi let first_row = meta.query_advice(config.advice[name], Rotation::cur()); let last_row = meta.query_advice( config.advice[name], - Rotation(analyzed.degree().try_into().unwrap()), + Rotation(analyzed.max_degree().try_into().unwrap()), ); let expr = first_step.clone() * (first_row - last_row); (format!("enforce wrapping ({name})"), expr) @@ -411,7 +411,7 @@ impl<'a, T: FieldElement, F: PrimeField> Circuit for PowdrCi )?; } } - let degree = self.analyzed.degree() as usize; + let degree = self.analyzed.max_degree() as usize; for i in 0..(2 * degree) { let value = F::from((i < degree) as u64); region.assign_fixed( diff --git a/backend/src/halo2/mock_prover.rs b/backend/src/halo2/mock_prover.rs index 65e2e206d..ed1afe2d2 100644 --- a/backend/src/halo2/mock_prover.rs +++ b/backend/src/halo2/mock_prover.rs @@ -20,7 +20,7 @@ pub fn mock_prove( // double the row count in order to make space for the cells introduced by the backend // TODO: use a precise count of the extra rows needed to avoid using so many rows - let circuit_row_count_log = usize::BITS - pil.degree().leading_zeros(); + let circuit_row_count_log = usize::BITS - pil.max_degree().leading_zeros(); let expanded_row_count_log = circuit_row_count_log + 1; let circuit = PowdrCircuit::new(pil, constants) diff --git a/backend/src/halo2/prover.rs b/backend/src/halo2/prover.rs index 2830c3f62..4c5971fb0 100644 --- a/backend/src/halo2/prover.rs +++ b/backend/src/halo2/prover.rs @@ -94,10 +94,10 @@ impl<'a, F: FieldElement> Halo2Prover<'a, F> { let mut params = setup .map(|mut setup| ParamsKZG::::read(&mut setup)) .transpose()? - .unwrap_or_else(|| generate_setup(analyzed.degree())); + .unwrap_or_else(|| generate_setup(analyzed.max_degree())); if matches!(proof_type, ProofType::Poseidon | ProofType::SnarkSingle) { - params.downsize(degree_bits(analyzed.degree())); + params.downsize(degree_bits(analyzed.max_degree())); } Ok(Self { @@ -250,7 +250,7 @@ impl<'a, F: FieldElement> Halo2Prover<'a, F> { log::info!("Generating VK for app snark..."); let mut params_app = self.params.clone(); - params_app.downsize(degree_bits(self.analyzed.degree())); + params_app.downsize(degree_bits(self.analyzed.max_degree())); log::info!("Generating circuit for compression snark..."); let protocol_app = compile( @@ -375,7 +375,7 @@ impl<'a, F: FieldElement> Halo2Prover<'a, F> { }; let mut params_app = self.params.clone(); - params_app.downsize(degree_bits(self.analyzed.degree())); + params_app.downsize(degree_bits(self.analyzed.max_degree())); let protocol_app = compile( ¶ms_app, diff --git a/executor/src/constant_evaluator/mod.rs b/executor/src/constant_evaluator/mod.rs index 0ca3ded9c..4a8b16e34 100644 --- a/executor/src/constant_evaluator/mod.rs +++ b/executor/src/constant_evaluator/mod.rs @@ -186,7 +186,7 @@ mod test { col fixed LAST(i) { if i == N - 1 { 1 } else { 0 } }; "#; let analyzed = analyze_string(src); - assert_eq!(analyzed.degree(), 8); + assert_eq!(analyzed.max_degree(), 8); let constants = generate(&analyzed); assert_eq!( constants, @@ -202,7 +202,7 @@ mod test { pol constant EVEN(i) { 2 * (i - 1) + 4 }; "#; let analyzed = analyze_string(src); - assert_eq!(analyzed.degree(), 8); + assert_eq!(analyzed.max_degree(), 8); let constants = generate(&analyzed); assert_eq!( constants, @@ -221,7 +221,7 @@ mod test { pol constant X(i) { i ^ (i + 17) | 3 }; "#; let analyzed = analyze_string(src); - assert_eq!(analyzed.degree(), 8); + assert_eq!(analyzed.max_degree(), 8); let constants = generate(&analyzed); assert_eq!( constants, @@ -245,7 +245,7 @@ mod test { } + 1 }; "#; let analyzed = analyze_string(src); - assert_eq!(analyzed.degree(), 8); + assert_eq!(analyzed.max_degree(), 8); let constants = generate(&analyzed); assert_eq!( constants, @@ -261,7 +261,7 @@ mod test { let X: col = |i| if i < 3 { 7 } else { 9 }; "#; let analyzed = analyze_string(src); - assert_eq!(analyzed.degree(), 8); + assert_eq!(analyzed.max_degree(), 8); let constants = generate(&analyzed); assert_eq!( constants, @@ -278,7 +278,7 @@ mod test { pol constant EVEN(i) { 2 * minus_one(i) + 2 }; "#; let analyzed = analyze_string(src); - assert_eq!(analyzed.degree(), 8); + assert_eq!(analyzed.max_degree(), 8); let constants = generate(&analyzed); assert_eq!( constants, @@ -304,7 +304,7 @@ mod test { col fixed doubled_half_nibble(i) { half_nibble_f(i / 2) }; "#; let analyzed = analyze_string(src); - assert_eq!(analyzed.degree(), 10); + assert_eq!(analyzed.max_degree(), 10); let constants = generate(&analyzed); assert_eq!(constants.len(), 4); assert_eq!( @@ -346,7 +346,7 @@ mod test { col fixed ref_other = [n-1, f(1), 8] + [0]*; "#; let analyzed = analyze_string(src); - assert_eq!(analyzed.degree(), 10); + assert_eq!(analyzed.max_degree(), 10); let constants = generate(&analyzed); assert_eq!(constants.len(), 3); assert_eq!( @@ -377,7 +377,7 @@ mod test { col fixed arr = [0, 1, 2]* + [7]; "#; let analyzed = analyze_string(src); - assert_eq!(analyzed.degree(), 10); + assert_eq!(analyzed.max_degree(), 10); let constants = generate(&analyzed); assert_eq!(constants.len(), 1); assert_eq!( @@ -412,7 +412,7 @@ mod test { col fixed greater_eq(i) { if id(i) >= inv(i) { 1 } else { 0 } }; "#; let analyzed = analyze_string(src); - assert_eq!(analyzed.degree(), 6); + assert_eq!(analyzed.max_degree(), 6); let constants = generate(&analyzed); assert_eq!( constants[0], @@ -471,7 +471,7 @@ mod test { let x: col = |i| w(i) + 1; "#; let analyzed = analyze_string::(src); - assert_eq!(analyzed.degree(), 10); + assert_eq!(analyzed.max_degree(), 10); generate(&analyzed); } @@ -484,7 +484,7 @@ mod test { let x = |i| w(i) + 1; "#; let analyzed = analyze_string::(src); - assert_eq!(analyzed.degree(), 10); + assert_eq!(analyzed.max_degree(), 10); generate(&analyzed); } @@ -498,7 +498,7 @@ mod test { col fixed y = [1, 2, 3]*; "#; let analyzed = analyze_string::(src); - assert_eq!(analyzed.degree(), 10); + assert_eq!(analyzed.max_degree(), 10); generate(&analyzed); } @@ -513,7 +513,7 @@ mod test { let Y: col = y; "#; let analyzed = analyze_string::(src); - assert_eq!(analyzed.degree(), 4); + assert_eq!(analyzed.max_degree(), 4); let constants = generate(&analyzed); assert_eq!( constants[0], @@ -536,7 +536,7 @@ mod test { let x: col = |i| (1 << (2000 + i)) >> 2000; "#; let analyzed = analyze_string::(src); - assert_eq!(analyzed.degree(), 4); + assert_eq!(analyzed.max_degree(), 4); let constants = generate(&analyzed); assert_eq!( constants[0], @@ -556,7 +556,7 @@ mod test { let x: col = |i| 100 + x_arr[i]; "#; let analyzed = analyze_string::(src); - assert_eq!(analyzed.degree(), 4); + assert_eq!(analyzed.max_degree(), 4); let constants = generate(&analyzed); // Semantics of p % q involving negative numbers: // sgn(p) * |p| % |q| @@ -576,7 +576,7 @@ mod test { let fe = || fe(); "#; let analyzed = analyze_string::(src); - assert_eq!(analyzed.degree(), 4); + assert_eq!(analyzed.max_degree(), 4); let constants = generate(&analyzed); assert_eq!( constants[0], @@ -600,7 +600,7 @@ mod test { let a: col = |i| std::convert::fe(i + seven) + seven; "#; let analyzed = analyze_string::(src); - assert_eq!(analyzed.degree(), 4); + assert_eq!(analyzed.max_degree(), 4); let constants = generate(&analyzed); assert_eq!( constants[0], diff --git a/executor/src/witgen/block_processor.rs b/executor/src/witgen/block_processor.rs index fb8e13076..eaef36cd0 100644 --- a/executor/src/witgen/block_processor.rs +++ b/executor/src/witgen/block_processor.rs @@ -154,16 +154,20 @@ mod tests { let mut fixed_lookup = FixedLookup::new(fixed_data.global_range_constraints().clone()); let mut machines = []; + // The degree is the max degree as we have a single machine + let degree = fixed_data.analyzed.max_degree(); + let columns = (0..fixed_data.witness_cols.len()) .map(move |i| PolyID { id: i as u64, ptype: PolynomialType::Committed, + // we only have one machine, so its degree is also the max + degree: Some(degree), }) .collect(); let data = FinalizableData::with_initial_rows_in_progress( &columns, - (0..fixed_data.degree) - .map(|i| Row::fresh(&fixed_data, RowIndex::from_degree(i, fixed_data.degree))), + (0..degree).map(|i| Row::fresh(&fixed_data, RowIndex::from_degree(i, degree))), ); let mut mutable_state = MutableState { @@ -171,7 +175,7 @@ mod tests { machines: Machines::from(machines.iter_mut()), query_callback: &mut query_callback, }; - let row_offset = RowIndex::from_degree(0, fixed_data.degree); + let row_offset = RowIndex::from_degree(0, degree); let identities = analyzed.identities.iter().collect::>(); let witness_cols = fixed_data.witness_cols.keys().collect(); @@ -187,7 +191,7 @@ mod tests { f( processor, name_to_poly_id(&fixed_data), - analyzed.degree(), + degree, identities.len(), ) } diff --git a/executor/src/witgen/data_structures/column_map.rs b/executor/src/witgen/data_structures/column_map.rs index e90aedb2b..36ff7e2ef 100644 --- a/executor/src/witgen/data_structures/column_map.rs +++ b/executor/src/witgen/data_structures/column_map.rs @@ -4,6 +4,7 @@ use std::{ }; use powdr_ast::analyzed::{PolyID, PolynomialType}; +use powdr_number::DegreeType; // Marker types for each PolynomialType #[derive(Clone, Copy)] @@ -29,17 +30,19 @@ pub type FixedColumnMap = ColumnMap; /// A Map indexed by polynomial ID, for a specific polynomial type (e.g. fixed or witness). /// For performance reasons, it uses a Vec internally and assumes that the polynomial IDs -/// are contiguous. +/// are contiguous. It also assumes that all polynomials have the same degree. #[derive(Clone)] pub struct ColumnMap { + pub degree: Option, values: Vec, _ptype: PhantomData, } impl ColumnMap { /// Create a new ColumnMap with the given initial value and size. - pub fn new(initial_value: V, size: usize) -> Self { + pub fn new(initial_value: V, size: usize, degree: Option) -> Self { ColumnMap { + degree, values: vec![initial_value; size], _ptype: PhantomData, } @@ -47,8 +50,9 @@ impl ColumnMap { } impl ColumnMap { - pub fn from(values: impl Iterator) -> Self { + pub fn from(values: impl Iterator, degree: Option) -> Self { ColumnMap { + degree, values: values.collect(), _ptype: PhantomData, } @@ -60,21 +64,31 @@ impl ColumnMap { V: Default, { let mut values: Vec = (0..len).map(|_| V::default()).collect(); + let mut degree: Option = None; for (poly, value) in items { values[poly.id as usize] = value; assert_eq!(poly.ptype, T::P_TYPE); + if degree.is_none() { + degree = Some(poly.degree.unwrap()); + } + assert_eq!(poly.degree, degree); } ColumnMap { + degree, values, _ptype: PhantomData, } } pub fn keys(&self) -> impl Iterator { + + let degree = self.degree; + (0..self.values.len()).map(move |i| PolyID { id: i as u64, ptype: T::P_TYPE, + degree, }) } diff --git a/executor/src/witgen/generator.rs b/executor/src/witgen/generator.rs index b6a3d2b92..6b43a7e2a 100644 --- a/executor/src/witgen/generator.rs +++ b/executor/src/witgen/generator.rs @@ -24,6 +24,7 @@ struct ProcessResult<'a, T: FieldElement> { } pub struct Generator<'a, T: FieldElement> { + degree: DegreeType, connecting_identities: BTreeMap>>, fixed_data: &'a FixedData<'a, T>, identities: Vec<&'a Identity>>, @@ -38,6 +39,10 @@ impl<'a, T: FieldElement> Machine<'a, T> for Generator<'a, T> { self.connecting_identities.keys().cloned().collect() } + fn degree(&self) -> DegreeType { + self.degree + } + fn name(&self) -> &str { &self.name } @@ -115,7 +120,19 @@ impl<'a, T: FieldElement> Generator<'a, T> { latch: Option>, ) -> Self { let data = FinalizableData::new(&witnesses); + + // get the degree of all witnesses, which must match + let degree = witnesses + .iter() + .map(|p| p.degree.unwrap()) + .reduce(|acc, degree| { + assert_eq!(acc, degree); + acc + }) + .unwrap(); + Self { + degree, connecting_identities: connecting_identities.clone(), name, fixed_data, @@ -139,7 +156,7 @@ impl<'a, T: FieldElement> Generator<'a, T> { &mut self, mutable_state: &mut MutableState<'a, '_, T, Q>, ) { - if self.data.len() < self.fixed_data.degree as usize + 1 { + if self.data.len() < self.degree() as usize + 1 { assert!(self.latch.is_some()); let first_row = self.data.pop().unwrap(); @@ -171,14 +188,8 @@ impl<'a, T: FieldElement> Generator<'a, T> { let data = FinalizableData::with_initial_rows_in_progress( &self.witnesses, [ - Row::fresh( - self.fixed_data, - RowIndex::from_i64(-1, self.fixed_data.degree), - ), - Row::fresh( - self.fixed_data, - RowIndex::from_i64(0, self.fixed_data.degree), - ), + Row::fresh(self.fixed_data, RowIndex::from_i64(-1, self.degree())), + Row::fresh(self.fixed_data, RowIndex::from_i64(0, self.degree())), ] .into_iter(), ); @@ -193,7 +204,7 @@ impl<'a, T: FieldElement> Generator<'a, T> { .filter_map(|identity| identity.contains_next_ref().then_some(*identity)) .collect::>(); let mut processor = BlockProcessor::new( - RowIndex::from_i64(-1, self.fixed_data.degree), + RowIndex::from_i64(-1, self.degree()), data, mutable_state, &identities_with_next_reference, @@ -225,7 +236,7 @@ impl<'a, T: FieldElement> Generator<'a, T> { [first_row].into_iter(), ); let mut processor = VmProcessor::new( - RowIndex::from_degree(row_offset, self.fixed_data.degree), + RowIndex::from_degree(row_offset, self.degree()), self.fixed_data, &self.identities, &self.witnesses, @@ -243,7 +254,7 @@ impl<'a, T: FieldElement> Generator<'a, T> { /// At the end of the solving algorithm, we'll have computed the first row twice /// (as row 0 and as row ). This function merges the two versions. fn fix_first_row(&mut self) { - assert_eq!(self.data.len() as DegreeType, self.fixed_data.degree + 1); + assert_eq!(self.data.len() as DegreeType, self.degree() + 1); let last_row = self.data.pop().unwrap(); self.data[0] = WitnessColumnMap::from(self.data[0].values().zip(last_row.values()).map( @@ -255,6 +266,6 @@ impl<'a, T: FieldElement> Generator<'a, T> { (CellValue::Known(_), _) => cell1.clone(), _ => cell2.clone(), }, - )); + ), Some(self.degree())); } } diff --git a/executor/src/witgen/global_constraints.rs b/executor/src/witgen/global_constraints.rs index 1b540fcf6..27b2002c4 100644 --- a/executor/src/witgen/global_constraints.rs +++ b/executor/src/witgen/global_constraints.rs @@ -360,7 +360,7 @@ mod test { use std::collections::BTreeMap; use powdr_ast::analyzed::{PolyID, PolynomialType}; - use powdr_number::GoldilocksField; + use powdr_number::{DegreeType, GoldilocksField}; use pretty_assertions::assert_eq; use test_log::test; @@ -402,17 +402,19 @@ mod test { ); } - fn constant_poly_id(i: u64) -> PolyID { + fn constant_poly_id(i: u64, degree: DegreeType) -> PolyID { PolyID { ptype: PolynomialType::Constant, id: i, + degree: Some(degree), } } - fn witness_poly_id(i: u64) -> PolyID { + fn witness_poly_id(i: u64, degree: DegreeType) -> PolyID { PolyID { ptype: PolynomialType::Committed, id: i, + degree: Some(degree), } } @@ -435,9 +437,10 @@ namespace Global(2**20); { D } in { SHIFTED }; "; let analyzed = powdr_pil_analyzer::analyze_string::(pil_source); + let degree = analyzed.max_degree(); let constants = crate::constant_evaluator::generate(&analyzed); let fixed_polys = (0..constants.len()) - .map(|i| constant_poly_id(i as u64)) + .map(|i| constant_poly_id(i as u64, degree)) .collect::>(); let mut known_constraints = fixed_polys .iter() @@ -450,11 +453,20 @@ namespace Global(2**20); known_constraints, vec![ // Global.BYTE - (constant_poly_id(0), RangeConstraint::from_max_bit(7)), + ( + constant_poly_id(0, degree), + RangeConstraint::from_max_bit(7) + ), // Global.BYTE2 - (constant_poly_id(1), RangeConstraint::from_max_bit(15)), + ( + constant_poly_id(1, degree), + RangeConstraint::from_max_bit(15) + ), // Global.SHIFTED - (constant_poly_id(2), RangeConstraint::from_mask(0xff0_u32)), + ( + constant_poly_id(2, degree), + RangeConstraint::from_mask(0xff0_u32) + ), ] .into_iter() .collect() @@ -467,19 +479,34 @@ namespace Global(2**20); known_constraints, vec![ // Global.A - (witness_poly_id(0), RangeConstraint::from_max_bit(0)), + (witness_poly_id(0, degree), RangeConstraint::from_max_bit(0)), // Global.B - (witness_poly_id(1), RangeConstraint::from_max_bit(7)), + (witness_poly_id(1, degree), RangeConstraint::from_max_bit(7)), // Global.C - (witness_poly_id(2), RangeConstraint::from_mask(0x2ff_u32)), + ( + witness_poly_id(2, degree), + RangeConstraint::from_mask(0x2ff_u32) + ), // Global.D - (witness_poly_id(3), RangeConstraint::from_mask(0xf0_u32)), + ( + witness_poly_id(3, degree), + RangeConstraint::from_mask(0xf0_u32) + ), // Global.BYTE - (constant_poly_id(0), RangeConstraint::from_max_bit(7)), + ( + constant_poly_id(0, degree), + RangeConstraint::from_max_bit(7) + ), // Global.BYTE2 - (constant_poly_id(1), RangeConstraint::from_max_bit(15)), + ( + constant_poly_id(1, degree), + RangeConstraint::from_max_bit(15) + ), // Global.SHIFTED - (constant_poly_id(2), RangeConstraint::from_mask(0xff0_u32)), + ( + constant_poly_id(2, degree), + RangeConstraint::from_mask(0xff0_u32) + ), ] .into_iter() .collect::>() @@ -498,9 +525,12 @@ namespace Global(1024); { X * 4 } in { bytes }; "; let analyzed = powdr_pil_analyzer::analyze_string::(pil_source); - let known_constraints = vec![(constant_poly_id(0), RangeConstraint::from_max_bit(7))] - .into_iter() - .collect(); + let known_constraints = vec![( + constant_poly_id(0, analyzed.max_degree()), + RangeConstraint::from_max_bit(7), + )] + .into_iter() + .collect(); assert_eq!(analyzed.identities.len(), 1); let (_, removed) = propagate_constraints( known_constraints, diff --git a/executor/src/witgen/machines/block_machine.rs b/executor/src/witgen/machines/block_machine.rs index 24d71970b..ff07b8ddc 100644 --- a/executor/src/witgen/machines/block_machine.rs +++ b/executor/src/witgen/machines/block_machine.rs @@ -98,6 +98,8 @@ impl<'a, T: FieldElement> Display for BlockMachine<'a, T> { /// TODO we do not actually "detect" the machine yet, we just check if /// the lookup has a binary selector that is 1 every k rows for some k pub struct BlockMachine<'a, T: FieldElement> { + /// The unique degree of all columns in this machine + degree: DegreeType, /// Block size, the period of the selector. block_size: usize, /// The row index (within the block) of the latch row @@ -127,8 +129,18 @@ impl<'a, T: FieldElement> BlockMachine<'a, T> { identities: &[&'a Identity>], witness_cols: &HashSet, ) -> Option { + // get the degree of all witnesses, which must match + let degree = witness_cols + .iter() + .map(|p| p.degree.unwrap()) + .reduce(|acc, degree| { + assert_eq!(acc, degree); + acc + }) + .unwrap(); + let (is_permutation, block_size, latch_row) = - detect_connection_type_and_block_size(fixed_data, connecting_identities)?; + detect_connection_type_and_block_size(fixed_data, connecting_identities, degree)?; for id in connecting_identities.values() { for r in id.right.expressions.iter() { @@ -143,19 +155,20 @@ impl<'a, T: FieldElement> BlockMachine<'a, T> { } } - assert!(block_size <= fixed_data.degree as usize); + assert!(block_size <= degree as usize); // Because block shapes are not always rectangular, we add the last block to the data at the // beginning. It starts out with unknown values. Should the first block decide to write to // rows < 0, they will be written to this block. // In `take_witness_col_values()`, this block will be removed and its values will be used to // construct the "default" block used to fill up unused rows. - let start_index = RowIndex::from_i64(-(block_size as i64), fixed_data.degree); + let start_index = RowIndex::from_i64(-(block_size as i64), degree); let data = FinalizableData::with_initial_rows_in_progress( witness_cols, (0..block_size).map(|i| Row::fresh(fixed_data, start_index + i)), ); Some(BlockMachine { name, + degree, block_size, latch_row, connecting_identities: connecting_identities.clone(), @@ -176,6 +189,7 @@ impl<'a, T: FieldElement> BlockMachine<'a, T> { fn detect_connection_type_and_block_size<'a, T: FieldElement>( fixed_data: &'a FixedData<'a, T>, connecting_identities: &BTreeMap>>, + degree: DegreeType, ) -> Option<(ConnectionType, usize, usize)> { // TODO we should check that the other constraints/fixed columns are also periodic. @@ -194,7 +208,7 @@ fn detect_connection_type_and_block_size<'a, T: FieldElement>( // We'd expect all RHS selectors to be fixed columns of the same period. connecting_identities .values() - .map(|id| try_to_period(&id.right.selector, fixed_data)) + .map(|id| try_to_period(&id.right.selector, fixed_data, degree)) .unique() .exactly_one() .ok()?? @@ -205,7 +219,7 @@ fn detect_connection_type_and_block_size<'a, T: FieldElement>( let find_max_period = |latch_candidates: BTreeSet>>| { latch_candidates .iter() - .filter_map(|e| try_to_period(e, fixed_data)) + .filter_map(|e| try_to_period(e, fixed_data, degree)) // If there is more than one period, the block size is the maximum period. .max_by_key(|&(_, period)| period) }; @@ -232,6 +246,7 @@ fn detect_connection_type_and_block_size<'a, T: FieldElement>( fn try_to_period( expr: &Option>, fixed_data: &FixedData, + degree: DegreeType, ) -> Option<(usize, usize)> { match expr { Some(expr) => { @@ -250,7 +265,7 @@ fn try_to_period( let offset = values.iter().position(|v| v.is_one())?; let period = 1 + values.iter().skip(offset + 1).position(|v| v.is_one())?; - if period > fixed_data.degree as usize / 2 { + if period > degree as usize / 2 { // This filters out columns like [0]* + [1], which might appear in a block machine // but shouldn't be detected as the latch. return None; @@ -277,6 +292,10 @@ impl<'a, T: FieldElement> Machine<'a, T> for BlockMachine<'a, T> { self.connecting_identities.keys().copied().collect() } + fn degree(&self) -> DegreeType { + self.degree + } + fn process_plookup<'b, Q: QueryCallback>( &mut self, mutable_state: &'b mut MutableState<'a, 'b, T, Q>, @@ -324,7 +343,7 @@ impl<'a, T: FieldElement> Machine<'a, T> for BlockMachine<'a, T> { ); // Instantiate a processor - let row_offset = RowIndex::from_i64(-1, self.fixed_data.degree); + let row_offset = RowIndex::from_i64(-1, self.degree); let mut mutable_state = MutableState { fixed_lookup, machines: vec![].into_iter().into(), @@ -382,7 +401,7 @@ impl<'a, T: FieldElement> Machine<'a, T> for BlockMachine<'a, T> { // For all constraints to be satisfied, unused cells have to be filled with valid values. // We do this, we construct a default block, by repeating the first input to the block machine. - values.resize(self.fixed_data.degree as usize, None); + values.resize(self.degree as usize, None); // Use the block as the default block. However, it needs to be merged with the dummy block, // to handle blocks of non-rectangular shape. @@ -445,7 +464,7 @@ impl<'a, T: FieldElement> BlockMachine<'a, T> { .ends_with("_operation_id_no_change") { log::trace!("Setting _operation_id_no_change to 0."); - col[self.fixed_data.degree as usize - 1] = T::zero(); + col[self.degree as usize - 1] = T::zero(); } } } @@ -457,7 +476,7 @@ impl<'a, T: FieldElement> BlockMachine<'a, T> { } fn last_row_index(&self) -> RowIndex { - RowIndex::from_i64(self.rows() as i64 - 1, self.fixed_data.degree) + RowIndex::from_i64(self.rows() as i64 - 1, self.degree) } fn get_row(&self, row: RowIndex) -> &Row<'a, T> { @@ -528,7 +547,7 @@ impl<'a, T: FieldElement> BlockMachine<'a, T> { )); } - if self.rows() + self.block_size as DegreeType >= self.fixed_data.degree { + if self.rows() + self.block_size as DegreeType >= self.degree { return Err(EvalError::RowsExhausted(self.name.clone())); } @@ -614,7 +633,7 @@ impl<'a, T: FieldElement> BlockMachine<'a, T> { /// unused cells in the previous block. fn append_block(&mut self, mut new_block: FinalizableData<'a, T>) -> Result<(), EvalError> { assert!( - (self.rows() + self.block_size as DegreeType) < self.fixed_data.degree, + (self.rows() + self.block_size as DegreeType) < self.degree, "Block machine is full (this should have been checked before)" ); diff --git a/executor/src/witgen/machines/double_sorted_witness_machine.rs b/executor/src/witgen/machines/double_sorted_witness_machine.rs index 9da0c7c10..9a36c4370 100644 --- a/executor/src/witgen/machines/double_sorted_witness_machine.rs +++ b/executor/src/witgen/machines/double_sorted_witness_machine.rs @@ -82,6 +82,16 @@ impl<'a, T: FieldElement> DoubleSortedWitnesses<'a, T> { connecting_identities: &BTreeMap>>, witness_cols: &HashSet, ) -> Option { + // get the degree of all witnesses, which must match + let degree = witness_cols + .iter() + .map(|p| p.degree.unwrap()) + .reduce(|acc, degree| { + assert_eq!(acc, degree); + acc + }) + .unwrap(); + // get the namespaces and column names let (mut namespaces, columns): (HashSet<_>, HashSet<_>) = witness_cols .iter() @@ -151,7 +161,7 @@ impl<'a, T: FieldElement> DoubleSortedWitnesses<'a, T> { name, namespace, fixed: fixed_data, - degree: fixed_data.degree, + degree, diff_columns_base, has_bootloader_write_column, trace: Default::default(), @@ -167,7 +177,7 @@ impl<'a, T: FieldElement> DoubleSortedWitnesses<'a, T> { name, namespace, fixed: fixed_data, - degree: fixed_data.degree, + degree, diff_columns_base: None, has_bootloader_write_column, trace: Default::default(), @@ -184,6 +194,10 @@ impl<'a, T: FieldElement> Machine<'a, T> for DoubleSortedWitnesses<'a, T> { self.selector_ids.keys().cloned().collect() } + fn degree(&self) -> DegreeType { + self.degree + } + fn name(&self) -> &str { &self.name } diff --git a/executor/src/witgen/machines/fixed_lookup_machine.rs b/executor/src/witgen/machines/fixed_lookup_machine.rs index 1fe744575..8a461067c 100644 --- a/executor/src/witgen/machines/fixed_lookup_machine.rs +++ b/executor/src/witgen/machines/fixed_lookup_machine.rs @@ -109,7 +109,20 @@ impl IndexedColumns { .map(|id| fixed_data.fixed_cols[id].values) .collect::>(); - let index: BTreeMap, IndexValue> = (0..fixed_data.degree as usize) + let degree = input_column_values + .iter() + .chain(output_column_values.iter()) + .map(|values| values.len()) + .reduce(|acc, l| { + assert_eq!( + acc, l, + "all columns in a given lookup are expected to have the same degree" + ); + acc + }) + .unwrap(); + + let index: BTreeMap, IndexValue> = (0..degree as usize) .fold( ( BTreeMap::, IndexValue>::default(), diff --git a/executor/src/witgen/machines/mod.rs b/executor/src/witgen/machines/mod.rs index 8ee9a3f83..d628a7bb1 100644 --- a/executor/src/witgen/machines/mod.rs +++ b/executor/src/witgen/machines/mod.rs @@ -1,5 +1,6 @@ use std::collections::HashMap; +use powdr_number::DegreeType; use powdr_number::FieldElement; use self::block_machine::BlockMachine; @@ -44,6 +45,9 @@ pub trait Machine<'a, T: FieldElement>: Send + Sync { /// Returns a unique name for this machine. fn name(&self) -> &str; + /// Return the unique degree of all columns in this machine + fn degree(&self) -> DegreeType; + /// Processes a connecting identity of a given ID (which must be known to the callee). /// Returns an error if the query leads to a constraint failure. /// Otherwise, it computes any updates to the caller row pair and returns them. @@ -100,6 +104,16 @@ impl<'a, T: FieldElement> Machine<'a, T> for KnownMachine<'a, T> { } } + fn degree(&self) -> DegreeType { + match self { + KnownMachine::SortedWitnesses(m) => m.degree(), + KnownMachine::DoubleSortedWitnesses(m) => m.degree(), + KnownMachine::WriteOnceMemory(m) => m.degree(), + KnownMachine::BlockMachine(m) => m.degree(), + KnownMachine::Vm(m) => m.degree(), + } + } + fn name(&self) -> &str { match self { KnownMachine::SortedWitnesses(m) => m.name(), diff --git a/executor/src/witgen/machines/sorted_witness_machine.rs b/executor/src/witgen/machines/sorted_witness_machine.rs index f17798e4a..7fde73702 100644 --- a/executor/src/witgen/machines/sorted_witness_machine.rs +++ b/executor/src/witgen/machines/sorted_witness_machine.rs @@ -14,7 +14,7 @@ use crate::witgen::{EvalValue, IncompleteCause, MutableState, QueryCallback}; use powdr_ast::analyzed::{ AlgebraicExpression as Expression, AlgebraicReference, Identity, IdentityKind, PolyID, }; -use powdr_number::FieldElement; +use powdr_number::{DegreeType, FieldElement}; /// A machine that can support a lookup in a set of columns that are sorted /// by one specific column and values in that column have to be unique. @@ -24,6 +24,7 @@ use powdr_number::FieldElement; /// - NOTLAST is zero only on the last row /// - POSITIVE has all values from 1 to half of the field size. pub struct SortedWitnesses<'a, T: FieldElement> { + degree: DegreeType, rhs_references: BTreeMap>, connecting_identities: BTreeMap>>, key_col: PolyID, @@ -45,7 +46,18 @@ impl<'a, T: FieldElement> SortedWitnesses<'a, T> { if identities.len() != 1 { return None; } - check_identity(fixed_data, identities.first().unwrap()).and_then(|key_col| { + + // get the degree of all witnesses, which must match + let degree = witnesses + .iter() + .map(|p| p.degree.unwrap()) + .reduce(|acc, degree| { + assert_eq!(acc, degree); + acc + }) + .unwrap(); + + check_identity(fixed_data, identities.first().unwrap(), degree).and_then(|key_col| { let witness_positions = witnesses .iter() .filter(|&w| *w != key_col) @@ -78,6 +90,7 @@ impl<'a, T: FieldElement> SortedWitnesses<'a, T> { } Some(SortedWitnesses { + degree, rhs_references, connecting_identities: connecting_identities.clone(), name, @@ -93,6 +106,7 @@ impl<'a, T: FieldElement> SortedWitnesses<'a, T> { fn check_identity( fixed_data: &FixedData, id: &Identity>, + degree: DegreeType, ) -> Option { // Looking for NOTLAST { A' - A } in { POSITIVE } if id.kind != IdentityKind::Plookup @@ -110,7 +124,7 @@ fn check_identity( // TODO this could be rather slow. We should check the code for identity instead // of evaluating it. - let degree = fixed_data.degree as usize; + let degree = degree as usize; for row in 0..(degree) { let ev = ExpressionEvaluator::new(FixedEvaluator::new(fixed_data, row)); let nl = ev.evaluate(not_last).ok()?.constant_value()?; @@ -168,6 +182,10 @@ impl<'a, T: FieldElement> Machine<'a, T> for SortedWitnesses<'a, T> { &self.name } + fn degree(&self) -> DegreeType { + self.degree + } + fn process_plookup>( &mut self, _mutable_state: &mut MutableState<'a, '_, T, Q>, @@ -188,7 +206,7 @@ impl<'a, T: FieldElement> Machine<'a, T> for SortedWitnesses<'a, T> { std::mem::take(&mut self.data).into_iter().unzip(); let mut last_key = keys.last().cloned().unwrap_or_default(); - while keys.len() < self.fixed_data.degree as usize { + while keys.len() < self.degree() as usize { last_key += 1u64.into(); keys.push(last_key); } @@ -199,7 +217,7 @@ impl<'a, T: FieldElement> Machine<'a, T> for SortedWitnesses<'a, T> { .iter_mut() .map(|row| std::mem::take(&mut row[i]).unwrap_or_default()) .collect::>(); - col_values.resize(self.fixed_data.degree as usize, 0.into()); + col_values.resize(self.degree() as usize, 0.into()); result.insert(self.fixed_data.column_name(col).to_string(), col_values); } diff --git a/executor/src/witgen/machines/write_once_memory.rs b/executor/src/witgen/machines/write_once_memory.rs index 24f0a6ba9..7a345b2c7 100644 --- a/executor/src/witgen/machines/write_once_memory.rs +++ b/executor/src/witgen/machines/write_once_memory.rs @@ -28,6 +28,7 @@ use super::{FixedLookup, Machine}; /// instr mload X -> Y { {X, Y} in {ADDR, v} } /// ``` pub struct WriteOnceMemory<'a, T: FieldElement> { + degree: DegreeType, connecting_identities: BTreeMap>>, /// The fixed data fixed_data: &'a FixedData<'a, T>, @@ -98,8 +99,18 @@ impl<'a, T: FieldElement> WriteOnceMemory<'a, T> { } }); + // get the degree of all witnesses, which must match + let degree = value_polys + .iter() + .map(|p| p.degree.unwrap()) + .reduce(|acc, degree| { + assert_eq!(acc, degree); + acc + }) + .unwrap(); + let mut key_to_index = BTreeMap::new(); - for row in 0..fixed_data.degree { + for row in 0..degree { let key = key_polys .iter() .map(|k| fixed_data.fixed_cols[k].values[row as usize]) @@ -111,6 +122,7 @@ impl<'a, T: FieldElement> WriteOnceMemory<'a, T> { } Some(Self { + degree, connecting_identities: connecting_identities.clone(), name, fixed_data, @@ -228,6 +240,10 @@ impl<'a, T: FieldElement> Machine<'a, T> for WriteOnceMemory<'a, T> { &self.name } + fn degree(&self) -> DegreeType { + self.degree + } + fn process_plookup<'b, Q: QueryCallback>( &mut self, _mutable_state: &'b mut MutableState<'a, 'b, T, Q>, @@ -251,11 +267,11 @@ impl<'a, T: FieldElement> Machine<'a, T> for WriteOnceMemory<'a, T> { .cloned() .map(|mut external_values| { // External witness values might only be provided partially. - external_values.resize(self.fixed_data.degree as usize, T::zero()); + external_values.resize(self.degree as usize, T::zero()); external_values }) .unwrap_or_else(|| { - let mut column = vec![T::zero(); self.fixed_data.degree as usize]; + let mut column = vec![T::zero(); self.degree as usize]; for (row, values) in self.data.iter() { column[*row as usize] = values[value_index].unwrap_or_default(); } diff --git a/executor/src/witgen/mod.rs b/executor/src/witgen/mod.rs index 7bf3b41f6..45714924c 100644 --- a/executor/src/witgen/mod.rs +++ b/executor/src/witgen/mod.rs @@ -271,7 +271,6 @@ pub fn extract_publics( /// Data that is fixed for witness generation. pub struct FixedData<'a, T: FieldElement> { analyzed: &'a Analyzed, - degree: DegreeType, fixed_cols: FixedColumnMap>, witness_cols: WitnessColumnMap>, column_by_name: HashMap, @@ -299,24 +298,24 @@ impl<'a, T: FieldElement> FixedData<'a, T> { .map(|(name, poly_id)| { let external_values = external_witness_values.remove(name.as_str()); if let Some(external_values) = &external_values { - if external_values.len() != analyzed.degree() as usize { + if external_values.len() != poly.degree.unwrap() as usize { log::debug!( "External witness values for column {} were only partially provided \ (length is {} but the degree is {})", name, external_values.len(), - analyzed.degree() + poly.degree.unwrap() ); } } // Remove any hint for witness columns of a later stage // (because it might reference a challenge that is not available yet) let value = if poly.stage.unwrap_or_default() <= stage.into() { value.as_ref() } else { None }; - WitnessColumn::new(poly_id.id as usize, &name, value, external_values) + WitnessColumn::new(poly_id, &name, value, external_values) }) .collect::>() }, - )); + ), Some(analyzed.max_degree())); if !external_witness_values.is_empty() { let available_columns = witness_cols @@ -331,17 +330,16 @@ impl<'a, T: FieldElement> FixedData<'a, T> { } let fixed_cols = - FixedColumnMap::from(fixed_col_values.iter().map(|(n, v)| FixedColumn::new(n, v))); + FixedColumnMap::from(fixed_col_values.iter().map(|(n, v)| FixedColumn::new(n, v)), Some(analyzed.max_degree())); // The global range constraints are not set yet. let global_range_constraints = GlobalConstraints { - witness_constraints: WitnessColumnMap::new(None, witness_cols.len()), - fixed_constraints: FixedColumnMap::new(None, fixed_cols.len()), + witness_constraints: WitnessColumnMap::new(None, witness_cols.len(), Some(analyzed.max_degree())), + fixed_constraints: FixedColumnMap::new(None, fixed_cols.len(), Some(analyzed.max_degree())), }; FixedData { analyzed, - degree: analyzed.degree(), fixed_cols, witness_cols, column_by_name: analyzed @@ -379,7 +377,7 @@ impl<'a, T: FieldElement> FixedData<'a, T> { } fn witness_map_with(&self, initial_value: V) -> WitnessColumnMap { - WitnessColumnMap::new(initial_value, self.witness_cols.len()) + WitnessColumnMap::new(initial_value, self.witness_cols.len(), None) } fn column_name(&self, poly_id: &PolyID) -> &str { @@ -395,11 +393,13 @@ impl<'a, T: FieldElement> FixedData<'a, T> { } fn external_witness(&self, row: DegreeType, column: &PolyID) -> Option { - let row = row % self.degree; self.witness_cols[column] .external_values .as_ref() - .and_then(|v| v.get(row as usize).cloned()) + .and_then(|v| { + let row = row % v.len() as u64; + v.get(row as usize).cloned() + }) } } @@ -433,11 +433,13 @@ pub struct WitnessColumn<'a, T> { impl<'a, T> WitnessColumn<'a, T> { pub fn new( - id: usize, + poly_id: PolyID, name: &str, value: Option<&'a FunctionValueDefinition>, external_values: Option<&'a Vec>, ) -> WitnessColumn<'a, T> { + assert_eq!(poly_id.ptype, PolynomialType::Committed); + let query = if let Some(FunctionValueDefinition::Expression(TypedExpression { e: query @ Expression::LambdaExpression( @@ -455,10 +457,7 @@ impl<'a, T> WitnessColumn<'a, T> { None }; let poly = AlgebraicReference { - poly_id: PolyID { - id: id as u64, - ptype: PolynomialType::Committed, - }, + poly_id, name: name.to_string(), next: false, }; diff --git a/executor/src/witgen/processor.rs b/executor/src/witgen/processor.rs index 1747c37c9..39a403835 100644 --- a/executor/src/witgen/processor.rs +++ b/executor/src/witgen/processor.rs @@ -97,11 +97,23 @@ impl<'a, 'b, 'c, T: FieldElement, Q: QueryCallback> Processor<'a, 'b, 'c, T, fixed_data: &'a FixedData<'a, T>, witness_cols: &'c HashSet, ) -> Self { + + // get the degree of all witnesses, which must match + let degree = witness_cols + .iter() + .map(|p| p.degree.unwrap()) + .reduce(|acc, degree| { + assert_eq!(acc, degree); + acc + }) + .unwrap(); + let is_relevant_witness = WitnessColumnMap::from( fixed_data .witness_cols .keys() .map(|poly_id| witness_cols.contains(&poly_id)), + Some(degree), ); Self { row_offset, diff --git a/executor/src/witgen/rows.rs b/executor/src/witgen/rows.rs index 98e03edd6..0b2c99adf 100644 --- a/executor/src/witgen/rows.rs +++ b/executor/src/witgen/rows.rs @@ -226,6 +226,7 @@ impl<'a, T: FieldElement> Row<'a, T> { }; Cell { name, value } }), + Some(row.num_rows) ) } @@ -269,9 +270,13 @@ impl<'a, T: FieldElement> Row<'a, T> { impl From> for WitnessColumnMap { /// Builds a map from polynomial ID to value. Unknown values are set to zero. fn from(val: Row) -> Self { + + let degree = val.degree; + WitnessColumnMap::from( val.into_iter() .map(|(_, cell)| cell.value.unwrap_or_default()), + degree ) } } diff --git a/executor/src/witgen/vm_processor.rs b/executor/src/witgen/vm_processor.rs index 7f2abeb9b..20c8d9287 100644 --- a/executor/src/witgen/vm_processor.rs +++ b/executor/src/witgen/vm_processor.rs @@ -44,6 +44,8 @@ impl<'a, T: FieldElement> CompletableIdentities<'a, T> { } pub struct VmProcessor<'a, 'b, 'c, T: FieldElement, Q: QueryCallback> { + /// The common degree of all referenced columns + degree: DegreeType, /// The global index of the first row of [VmProcessor::data]. row_offset: DegreeType, /// The witness columns belonging to this machine @@ -70,12 +72,22 @@ impl<'a, 'b, 'c, T: FieldElement, Q: QueryCallback> VmProcessor<'a, 'b, 'c, T data: FinalizableData<'a, T>, mutable_state: &'c mut MutableState<'a, 'b, T, Q>, ) -> Self { + // get the degree of all witnesses, which must match + let degree = witnesses + .iter() + .map(|p| p.degree.unwrap()) + .reduce(|acc, degree| { + assert_eq!(acc, degree); + acc + }) + .unwrap(); + let (identities_with_next, identities_without_next): (Vec<_>, Vec<_>) = identities .iter() .partition(|identity| identity.contains_next_ref()); let processor = Processor::new(row_offset, data, mutable_state, fixed_data, witnesses); - let progress_bar = ProgressBar::new(fixed_data.degree); + let progress_bar = ProgressBar::new(degree); progress_bar.set_style( ProgressStyle::with_template( "[{elapsed_precise} (ETA: {eta_precise})] {bar} {percent}% - {msg}", @@ -84,6 +96,7 @@ impl<'a, 'b, 'c, T: FieldElement, Q: QueryCallback> VmProcessor<'a, 'b, 'c, T ); VmProcessor { + degree, row_offset: row_offset.into(), witnesses: witnesses.clone(), fixed_data, @@ -111,7 +124,7 @@ impl<'a, 'b, 'c, T: FieldElement, Q: QueryCallback> VmProcessor<'a, 'b, 'c, T assert!(self.processor.len() == 1); if is_main_run { - log::info!("Running main machine for {} rows", self.fixed_data.degree); + log::info!("Running main machine for {} rows", self.degree); self.progress_bar.reset(); self.progress_bar.set_message("Starting..."); self.progress_bar.tick(); @@ -126,7 +139,7 @@ impl<'a, 'b, 'c, T: FieldElement, Q: QueryCallback> VmProcessor<'a, 'b, 'c, T } else { log::Level::Debug }; - let rows_left = self.fixed_data.degree - self.row_offset + 1; + let rows_left = self.degree - self.row_offset + 1; let mut finalize_start = 1; for row_index in 0..rows_left { if is_main_run { @@ -201,7 +214,7 @@ impl<'a, 'b, 'c, T: FieldElement, Q: QueryCallback> VmProcessor<'a, 'b, 'c, T assert_eq!( self.processor.len() as DegreeType + self.row_offset, - self.fixed_data.degree + 1 + self.degree + 1 ); if is_main_run { @@ -237,7 +250,7 @@ impl<'a, 'b, 'c, T: FieldElement, Q: QueryCallback> VmProcessor<'a, 'b, 'c, T self.processor.len(), Row::fresh( self.fixed_data, - RowIndex::from_degree(row_index, self.fixed_data.degree) + 1, + RowIndex::from_degree(row_index, self.degree) + 1, ), ); } diff --git a/number/src/serialize.rs b/number/src/serialize.rs index 22f48ce65..5a36f3d19 100644 --- a/number/src/serialize.rs +++ b/number/src/serialize.rs @@ -40,17 +40,19 @@ pub fn write_polys_csv_file( })); writer.write_record(&headers).unwrap(); - let len = polys[0].1.len(); - for row_index in 0..len { + let max_len = polys.iter().map(|p| p.1.len()).max().unwrap(); + for row_index in 0..max_len { let mut row = Vec::new(); row.push(format!("{row_index}")); for (_, values) in polys { - assert!(values.len() == len); - let value = match render_mode { - CsvRenderMode::SignedBase10 => format!("{}", values[row_index]), - CsvRenderMode::UnsignedBase10 => format!("{}", values[row_index].to_integer()), - CsvRenderMode::Hex => format!("0x{:x}", values[row_index].to_integer()), - }; + let value = values + .get(row_index) + .map(|v| match render_mode { + CsvRenderMode::SignedBase10 => format!("{v}"), + CsvRenderMode::UnsignedBase10 => format!("{}", v.to_integer()), + CsvRenderMode::Hex => format!("0x{:x}", v.to_integer()), + }) + .unwrap_or_default(); row.push(value); } writer.write_record(&row).unwrap(); diff --git a/pil-analyzer/src/condenser.rs b/pil-analyzer/src/condenser.rs index d5b23d50d..95a6f6cf1 100644 --- a/pil-analyzer/src/condenser.rs +++ b/pil-analyzer/src/condenser.rs @@ -144,7 +144,6 @@ pub fn condense( reference.poly_id = Some(symbol.into()); } Analyzed { - degree, definitions, public_declarations, intermediate_columns, diff --git a/pil-analyzer/tests/condenser.rs b/pil-analyzer/tests/condenser.rs index 24a9c2960..6890b89ff 100644 --- a/pil-analyzer/tests/condenser.rs +++ b/pil-analyzer/tests/condenser.rs @@ -124,7 +124,7 @@ pub fn degree() { let formatted = analyze_string::(input).to_string(); let expected = r#"namespace std::convert(8); let expr = []; -namespace std::prover(8); +namespace std::prover(); let degree = []; namespace Main(8); let d: int = std::prover::degree(); diff --git a/pil-analyzer/tests/parse_display.rs b/pil-analyzer/tests/parse_display.rs index 9535ebc39..d358ec865 100644 --- a/pil-analyzer/tests/parse_display.rs +++ b/pil-analyzer/tests/parse_display.rs @@ -701,7 +701,6 @@ namespace T(8); let k: int = X.y; "; let analyzed = analyze_string::(input); - assert_eq!(analyzed.degree, Some(8)); assert_eq!(expected, analyzed.to_string()); } @@ -718,6 +717,5 @@ namespace T(8); let k: int = std::prelude::y; "; let analyzed = analyze_string::(input); - assert_eq!(analyzed.degree, Some(8)); assert_eq!(expected, analyzed.to_string()); } diff --git a/pipeline/src/test_util.rs b/pipeline/src/test_util.rs index 1d39808b6..a084c55fe 100644 --- a/pipeline/src/test_util.rs +++ b/pipeline/src/test_util.rs @@ -143,7 +143,7 @@ pub fn gen_halo2_proof(file_name: &str, inputs: Vec) { buffered_write_file(&setup_file_path, |writer| { powdr_backend::BackendType::Halo2 .factory::() - .generate_setup(pil.degree(), writer) + .generate_setup(pil.max_degree(), writer) .unwrap() }) .unwrap(); diff --git a/pipeline/tests/asm.rs b/pipeline/tests/asm.rs index 512d93b94..c3f6a10a7 100644 --- a/pipeline/tests/asm.rs +++ b/pipeline/tests/asm.rs @@ -369,12 +369,12 @@ fn read_poly_files() { // check fixed cols (may have no fixed cols) if let Some(fixed) = try_read_poly_set::(&pil, tmp_dir.as_path()) { - assert_eq!(pil.degree(), fixed[0].1.len() as u64); + assert_eq!(pil.max_degree(), fixed[0].1.len() as u64); } // check witness cols (examples assumed to have at least one witness col) let witness = try_read_poly_set::(&pil, tmp_dir.as_path()).unwrap(); - assert_eq!(pil.degree(), witness[0].1.len() as u64); + assert_eq!(pil.max_degree(), witness[0].1.len() as u64); } } diff --git a/pipeline/tests/pil.rs b/pipeline/tests/pil.rs index c20082d9a..b3b048a57 100644 --- a/pipeline/tests/pil.rs +++ b/pipeline/tests/pil.rs @@ -299,6 +299,12 @@ fn naive_byte_decomposition_gl() { verify_pil(f, Default::default()); } +#[test] +fn different_degrees() { + let f = "pil/two_proofs.pil"; + verify_pil(f, Default::default()); +} + #[test] fn serialize_deserialize_optimized_pil() { let f = "pil/fibonacci.pil"; diff --git a/riscv/src/continuations.rs b/riscv/src/continuations.rs index d912d69e8..ee3a4fa85 100644 --- a/riscv/src/continuations.rs +++ b/riscv/src/continuations.rs @@ -71,7 +71,7 @@ where pipeline.compute_fixed_cols().unwrap(); // we can assume optimized_pil has been computed - let length = pipeline.compute_optimized_pil().unwrap().degree(); + let length = pipeline.compute_optimized_pil().unwrap().max_degree(); bootloader_inputs .into_iter() diff --git a/test_data/pil/single_line_blocks.pil b/test_data/pil/single_line_blocks.pil index 279b0b4b8..9f1ca0833 100644 --- a/test_data/pil/single_line_blocks.pil +++ b/test_data/pil/single_line_blocks.pil @@ -8,7 +8,7 @@ namespace Add(%N); A + B = C; // a machine which calls `Add` every other row on made up inputs -namespace Main(%N); +namespace Main(2 * %N); col fixed a(i) { (i + 13) & 0xffffffff }; col fixed b(i) { ((i + 19) * 17) & 0xffffffff }; col witness c; From 9507915f37333a0857d1cd80e6d5cb2a824b4852 Mon Sep 17 00:00:00 2001 From: schaeff Date: Thu, 20 Jun 2024 18:51:26 +0200 Subject: [PATCH 03/39] lint, fmt --- ast/src/analyzed/display.rs | 29 ++++++++++--------- .../src/witgen/data_structures/column_map.rs | 1 - executor/src/witgen/generator.rs | 24 ++++++++------- .../witgen/machines/fixed_lookup_machine.rs | 2 +- executor/src/witgen/mod.rs | 18 +++++++++--- executor/src/witgen/processor.rs | 3 +- executor/src/witgen/rows.rs | 5 ++-- 7 files changed, 47 insertions(+), 35 deletions(-) diff --git a/ast/src/analyzed/display.rs b/ast/src/analyzed/display.rs index f1de91556..d7fd05a87 100644 --- a/ast/src/analyzed/display.rs +++ b/ast/src/analyzed/display.rs @@ -22,21 +22,22 @@ use super::*; impl Display for Analyzed { fn fmt(&self, f: &mut Formatter<'_>) -> Result { let mut current_namespace = AbsoluteSymbolPath::default(); - let mut update_namespace = |name: &str, degree: Option, f: &mut Formatter<'_>| { - let mut namespace = - AbsoluteSymbolPath::default().join(SymbolPath::from_str(name).unwrap()); - let name = namespace.pop().unwrap(); - if namespace != current_namespace { - current_namespace = namespace; - writeln!( - f, - "namespace {}({});", - current_namespace.relative_to(&Default::default()), - degree.map(|d| d.to_string()).unwrap_or_default() - )?; + let mut update_namespace = + |name: &str, degree: Option, f: &mut Formatter<'_>| { + let mut namespace = + AbsoluteSymbolPath::default().join(SymbolPath::from_str(name).unwrap()); + let name = namespace.pop().unwrap(); + if namespace != current_namespace { + current_namespace = namespace; + writeln!( + f, + "namespace {}({});", + current_namespace.relative_to(&Default::default()), + degree.map(|d| d.to_string()).unwrap_or_default() + )?; + }; + Ok((name, !current_namespace.is_empty())) }; - Ok((name, !current_namespace.is_empty())) - }; for statement in &self.source_order { match statement { diff --git a/executor/src/witgen/data_structures/column_map.rs b/executor/src/witgen/data_structures/column_map.rs index 36ff7e2ef..faa48dd1b 100644 --- a/executor/src/witgen/data_structures/column_map.rs +++ b/executor/src/witgen/data_structures/column_map.rs @@ -82,7 +82,6 @@ impl ColumnMap { } pub fn keys(&self) -> impl Iterator { - let degree = self.degree; (0..self.values.len()).map(move |i| PolyID { diff --git a/executor/src/witgen/generator.rs b/executor/src/witgen/generator.rs index 6b43a7e2a..bbdce0ba8 100644 --- a/executor/src/witgen/generator.rs +++ b/executor/src/witgen/generator.rs @@ -257,15 +257,19 @@ impl<'a, T: FieldElement> Generator<'a, T> { assert_eq!(self.data.len() as DegreeType, self.degree() + 1); let last_row = self.data.pop().unwrap(); - self.data[0] = WitnessColumnMap::from(self.data[0].values().zip(last_row.values()).map( - |(cell1, cell2)| match (&cell1.value, &cell2.value) { - (CellValue::Known(v1), CellValue::Known(v2)) => { - assert_eq!(v1, v2); - cell1.clone() - } - (CellValue::Known(_), _) => cell1.clone(), - _ => cell2.clone(), - }, - ), Some(self.degree())); + self.data[0] = WitnessColumnMap::from( + self.data[0] + .values() + .zip(last_row.values()) + .map(|(cell1, cell2)| match (&cell1.value, &cell2.value) { + (CellValue::Known(v1), CellValue::Known(v2)) => { + assert_eq!(v1, v2); + cell1.clone() + } + (CellValue::Known(_), _) => cell1.clone(), + _ => cell2.clone(), + }), + Some(self.degree()), + ); } } diff --git a/executor/src/witgen/machines/fixed_lookup_machine.rs b/executor/src/witgen/machines/fixed_lookup_machine.rs index 8a461067c..1b174901b 100644 --- a/executor/src/witgen/machines/fixed_lookup_machine.rs +++ b/executor/src/witgen/machines/fixed_lookup_machine.rs @@ -122,7 +122,7 @@ impl IndexedColumns { }) .unwrap(); - let index: BTreeMap, IndexValue> = (0..degree as usize) + let index: BTreeMap, IndexValue> = (0..degree) .fold( ( BTreeMap::, IndexValue>::default(), diff --git a/executor/src/witgen/mod.rs b/executor/src/witgen/mod.rs index 45714924c..d395bf058 100644 --- a/executor/src/witgen/mod.rs +++ b/executor/src/witgen/mod.rs @@ -329,13 +329,23 @@ impl<'a, T: FieldElement> FixedData<'a, T> { ); } - let fixed_cols = - FixedColumnMap::from(fixed_col_values.iter().map(|(n, v)| FixedColumn::new(n, v)), Some(analyzed.max_degree())); + let fixed_cols = FixedColumnMap::from( + fixed_col_values.iter().map(|(n, v)| FixedColumn::new(n, v)), + Some(analyzed.max_degree()), + ); // The global range constraints are not set yet. let global_range_constraints = GlobalConstraints { - witness_constraints: WitnessColumnMap::new(None, witness_cols.len(), Some(analyzed.max_degree())), - fixed_constraints: FixedColumnMap::new(None, fixed_cols.len(), Some(analyzed.max_degree())), + witness_constraints: WitnessColumnMap::new( + None, + witness_cols.len(), + Some(analyzed.max_degree()), + ), + fixed_constraints: FixedColumnMap::new( + None, + fixed_cols.len(), + Some(analyzed.max_degree()), + ), }; FixedData { diff --git a/executor/src/witgen/processor.rs b/executor/src/witgen/processor.rs index 39a403835..61f4ee509 100644 --- a/executor/src/witgen/processor.rs +++ b/executor/src/witgen/processor.rs @@ -97,7 +97,6 @@ impl<'a, 'b, 'c, T: FieldElement, Q: QueryCallback> Processor<'a, 'b, 'c, T, fixed_data: &'a FixedData<'a, T>, witness_cols: &'c HashSet, ) -> Self { - // get the degree of all witnesses, which must match let degree = witness_cols .iter() @@ -113,7 +112,7 @@ impl<'a, 'b, 'c, T: FieldElement, Q: QueryCallback> Processor<'a, 'b, 'c, T, .witness_cols .keys() .map(|poly_id| witness_cols.contains(&poly_id)), - Some(degree), + Some(degree), ); Self { row_offset, diff --git a/executor/src/witgen/rows.rs b/executor/src/witgen/rows.rs index 0b2c99adf..380f8ed70 100644 --- a/executor/src/witgen/rows.rs +++ b/executor/src/witgen/rows.rs @@ -226,7 +226,7 @@ impl<'a, T: FieldElement> Row<'a, T> { }; Cell { name, value } }), - Some(row.num_rows) + Some(row.num_rows), ) } @@ -270,13 +270,12 @@ impl<'a, T: FieldElement> Row<'a, T> { impl From> for WitnessColumnMap { /// Builds a map from polynomial ID to value. Unknown values are set to zero. fn from(val: Row) -> Self { - let degree = val.degree; WitnessColumnMap::from( val.into_iter() .map(|(_, cell)| cell.value.unwrap_or_default()), - degree + degree, ) } } From e9a719927a0a7aec536704faf789b5bcaf653f3b Mon Sep 17 00:00:00 2001 From: schaeff Date: Thu, 20 Jun 2024 19:29:09 +0200 Subject: [PATCH 04/39] fix empty degree tests --- pil-analyzer/tests/condenser.rs | 2 +- pil-analyzer/tests/parse_display.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pil-analyzer/tests/condenser.rs b/pil-analyzer/tests/condenser.rs index 6890b89ff..fae7751da 100644 --- a/pil-analyzer/tests/condenser.rs +++ b/pil-analyzer/tests/condenser.rs @@ -122,7 +122,7 @@ pub fn degree() { w = std::convert::expr(d); "#; let formatted = analyze_string::(input).to_string(); - let expected = r#"namespace std::convert(8); + let expected = r#"namespace std::convert(); let expr = []; namespace std::prover(); let degree = []; diff --git a/pil-analyzer/tests/parse_display.rs b/pil-analyzer/tests/parse_display.rs index d358ec865..4435a3a7f 100644 --- a/pil-analyzer/tests/parse_display.rs +++ b/pil-analyzer/tests/parse_display.rs @@ -695,7 +695,7 @@ fn namespace_no_degree() { namespace T(8); let k = X::y; "; - let expected = "namespace X(8); + let expected = "namespace X(); let y: int = 7; namespace T(8); let k: int = X.y; From 591a0d878db27255200ded26127b0d94cd9c80a0 Mon Sep 17 00:00:00 2001 From: schaeff Date: Tue, 25 Jun 2024 15:40:57 +0200 Subject: [PATCH 05/39] use RawPolyID --- ast/src/analyzed/display.rs | 18 +-- ast/src/analyzed/mod.rs | 113 ++++++++++++++---- backend/src/estark/json_exporter/mod.rs | 12 +- executor/src/witgen/block_processor.rs | 6 +- .../src/witgen/data_structures/column_map.rs | 28 ++--- .../data_structures/finalizable_data.rs | 2 +- executor/src/witgen/fixed_evaluator.rs | 2 +- executor/src/witgen/generator.rs | 46 +++---- executor/src/witgen/global_constraints.rs | 84 +++++-------- executor/src/witgen/machines/block_machine.rs | 17 +-- .../machines/double_sorted_witness_machine.rs | 17 +-- .../witgen/machines/fixed_lookup_machine.rs | 6 +- .../src/witgen/machines/machine_extractor.rs | 60 ++++++++-- .../witgen/machines/sorted_witness_machine.rs | 21 ++-- .../src/witgen/machines/write_once_memory.rs | 21 +--- executor/src/witgen/mod.rs | 18 ++- executor/src/witgen/processor.rs | 22 +--- executor/src/witgen/query_processor.rs | 4 +- executor/src/witgen/rows.rs | 8 +- executor/src/witgen/vm_processor.rs | 14 +-- 20 files changed, 253 insertions(+), 266 deletions(-) diff --git a/ast/src/analyzed/display.rs b/ast/src/analyzed/display.rs index d7fd05a87..754dcc1db 100644 --- a/ast/src/analyzed/display.rs +++ b/ast/src/analyzed/display.rs @@ -483,29 +483,17 @@ mod test { let x = AlgebraicExpression::Reference(super::AlgebraicReference { name: "x".into(), - poly_id: super::PolyID { - id: 0, - ptype: super::PolynomialType::Committed, - degree: None, - }, + poly_id: super::PolyID::new(0, super::PolynomialType::Committed), next: false, }); let y = AlgebraicExpression::Reference(super::AlgebraicReference { name: "y".into(), - poly_id: super::PolyID { - id: 1, - ptype: super::PolynomialType::Committed, - degree: None, - }, + poly_id: super::PolyID::new(1, super::PolynomialType::Committed), next: false, }); let z = AlgebraicExpression::Reference(super::AlgebraicReference { name: "z".into(), - poly_id: super::PolyID { - id: 2, - ptype: super::PolynomialType::Committed, - degree: None, - }, + poly_id: super::PolyID::new(2, super::PolynomialType::Committed), next: false, }); diff --git a/ast/src/analyzed/mod.rs b/ast/src/analyzed/mod.rs index 86c55b557..0fe3d9cb9 100644 --- a/ast/src/analyzed/mod.rs +++ b/ast/src/analyzed/mod.rs @@ -216,14 +216,8 @@ impl Analyzed { let length = symbol.length.unwrap_or(1); // Empty arrays still need ID replacement for i in 0..max(length, 1) { - let old_poly_id = PolyID { - id: symbol.id + i, - ..PolyID::from(symbol) - }; - let new_poly_id = PolyID { - id: new_id + i, - ..PolyID::from(symbol) - }; + let old_poly_id = PolyID::from(symbol).with_id(symbol.id + i); + let new_poly_id = PolyID::from(symbol).with_id(new_id + i); replacements.insert(old_poly_id, new_poly_id); } new_id + length @@ -243,14 +237,14 @@ impl Analyzed { self.definitions.values_mut().for_each(|(poly, _def)| { if matches!(poly.kind, SymbolKind::Poly(_)) { let poly_id = PolyID::from(poly as &Symbol); - poly.id = replacements[&poly_id].id; + poly.id = replacements[&poly_id].id(); } }); self.intermediate_columns .values_mut() .for_each(|(poly, _def)| { let poly_id = PolyID::from(poly as &Symbol); - poly.id = replacements[&poly_id].id; + poly.id = replacements[&poly_id].id(); }); let visitor = &mut |expr: &mut Expression| { if let Expression::Reference(_, Reference::Poly(poly)) = expr { @@ -343,7 +337,7 @@ fn substitute_intermediate( .scan(HashMap::default(), |cache, mut identity| { identity.post_visit_expressions_mut(&mut |e| { if let AlgebraicExpression::Reference(poly) = e { - match poly.poly_id.ptype { + match poly.poly_id.ptype() { PolynomialType::Committed => {} PolynomialType::Constant => {} PolynomialType::Intermediate => { @@ -371,7 +365,10 @@ fn inlined_expression_from_intermediate_poly_id( intermediate_polynomials: &HashMap>, cache: &mut HashMap>, ) -> AlgebraicExpression { - assert_eq!(poly_to_replace.poly_id.ptype, PolynomialType::Intermediate); + assert_eq!( + poly_to_replace.poly_id.ptype(), + PolynomialType::Intermediate + ); if let Some(e) = cache.get(&poly_to_replace) { return e.clone(); } @@ -388,7 +385,7 @@ fn inlined_expression_from_intermediate_poly_id( ); } r.next = r.next || poly_to_replace.next; - match r.poly_id.ptype { + match r.poly_id.ptype() { PolynomialType::Committed | PolynomialType::Constant => {} PolynomialType::Intermediate => { *e = inlined_expression_from_intermediate_poly_id( @@ -466,8 +463,10 @@ impl Symbol { ( self.array_element_name(i), PolyID { - id: self.id + i, - ptype, + raw: RawPolyID { + id: self.id + i, + ptype, + }, degree: self.degree, }, ) @@ -768,11 +767,11 @@ pub struct AlgebraicReference { impl AlgebraicReference { #[inline] pub fn is_witness(&self) -> bool { - self.poly_id.ptype == PolynomialType::Committed + self.poly_id.ptype() == PolynomialType::Committed } #[inline] pub fn is_fixed(&self) -> bool { - self.poly_id.ptype == PolynomialType::Constant + self.poly_id.ptype() == PolynomialType::Constant } } @@ -1150,31 +1149,97 @@ pub struct PolynomialReference { } #[derive( - Debug, Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Serialize, Deserialize, JsonSchema, + Debug, Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Serialize, Deserialize, JsonSchema, Hash, )] pub struct PolyID { + pub raw: RawPolyID, + pub degree: Option, +} + +impl std::ops::Deref for PolyID { + type Target = RawPolyID; + + fn deref(&self) -> &Self::Target { + &self.raw + } +} + +impl PolyID { + pub fn with_id(mut self, id: u64) -> Self { + self.raw = self.raw.with_id(id); + self + } + + pub fn id(&self) -> u64 { + self.raw.id() + } + + pub fn ptype(&self) -> PolynomialType { + self.raw.ptype() + } + + pub fn new(id: u64, ptype: PolynomialType) -> Self { + Self { + raw: RawPolyID { id, ptype }, + degree: None, + } + } +} + +#[derive( + Debug, Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Serialize, Deserialize, JsonSchema, +)] +pub struct RawPolyID { pub id: u64, pub ptype: PolynomialType, - pub degree: Option, } -impl From<&Symbol> for PolyID { +impl From<&Symbol> for RawPolyID { fn from(symbol: &Symbol) -> Self { let SymbolKind::Poly(ptype) = symbol.kind else { panic!() }; - PolyID { + RawPolyID { id: symbol.id, ptype, - degree: symbol.degree, } } } -impl Hash for PolyID { +impl Hash for RawPolyID { fn hash(&self, state: &mut H) { // single call to hash is faster - ((self.id << 2) + self.ptype as u64).hash(state); + ((self.id() << 2) + self.ptype() as u64).hash(state); + } +} + +impl From for RawPolyID { + fn from(value: PolyID) -> Self { + value.raw + } +} + +impl RawPolyID { + fn with_id(mut self, id: u64) -> Self { + self.id = id; + self + } + + pub fn id(&self) -> u64 { + self.id + } + + pub fn ptype(&self) -> PolynomialType { + self.ptype + } +} + +impl From<&Symbol> for PolyID { + fn from(symbol: &Symbol) -> Self { + PolyID { + raw: symbol.into(), + degree: symbol.degree, + } } } diff --git a/backend/src/estark/json_exporter/mod.rs b/backend/src/estark/json_exporter/mod.rs index 314c20afe..be07c5661 100644 --- a/backend/src/estark/json_exporter/mod.rs +++ b/backend/src/estark/json_exporter/mod.rs @@ -5,7 +5,7 @@ use std::{cmp, path::PathBuf}; use powdr_ast::analyzed::{ AlgebraicBinaryOperation, AlgebraicBinaryOperator, AlgebraicExpression as Expression, AlgebraicUnaryOperation, AlgebraicUnaryOperator, Analyzed, IdentityKind, PolyID, - PolynomialType, StatementIdentifier, SymbolKind, + PolynomialType, RawPolyID, StatementIdentifier, SymbolKind, }; use powdr_parser_util::SourceRef; use starky::types::{ @@ -68,10 +68,7 @@ pub fn export(analyzed: &Analyzed) -> PIL { let pub_ref = &pub_def.polynomial; let poly_id = pub_ref.poly_id.unwrap(); let (_, expr) = exporter.polynomial_reference_to_json( - PolyID { - id: poly_id.id + pub_def.array_index.unwrap_or_default() as u64, - ..poly_id - }, + poly_id.with_id(poly_id.id() + pub_def.array_index.unwrap_or_default() as u64), false, ); let id = publics.len(); @@ -369,7 +366,10 @@ impl<'a, T: FieldElement> Exporter<'a, T> { fn polynomial_reference_to_json( &self, - PolyID { id, ptype, .. }: PolyID, + PolyID { + raw: RawPolyID { id, ptype }, + .. + }: PolyID, next: bool, ) -> (u32, StarkyExpr) { let id = if ptype == PolynomialType::Intermediate { diff --git a/executor/src/witgen/block_processor.rs b/executor/src/witgen/block_processor.rs index eaef36cd0..c7931ddf0 100644 --- a/executor/src/witgen/block_processor.rs +++ b/executor/src/witgen/block_processor.rs @@ -1,7 +1,7 @@ use std::collections::HashSet; use powdr_ast::analyzed::{ - AlgebraicExpression as Expression, AlgebraicReference, Identity, PolyID, + AlgebraicExpression as Expression, AlgebraicReference, Identity, RawPolyID as PolyID, }; use powdr_number::FieldElement; @@ -108,7 +108,7 @@ impl<'a, 'b, 'c, T: FieldElement, Q: QueryCallback> BlockProcessor<'a, 'b, 'c mod tests { use std::collections::BTreeMap; - use powdr_ast::analyzed::{PolyID, PolynomialType}; + use powdr_ast::analyzed::{PolynomialType, RawPolyID as PolyID}; use powdr_number::{FieldElement, GoldilocksField}; use powdr_pil_analyzer::analyze_string; @@ -161,8 +161,6 @@ mod tests { .map(move |i| PolyID { id: i as u64, ptype: PolynomialType::Committed, - // we only have one machine, so its degree is also the max - degree: Some(degree), }) .collect(); let data = FinalizableData::with_initial_rows_in_progress( diff --git a/executor/src/witgen/data_structures/column_map.rs b/executor/src/witgen/data_structures/column_map.rs index faa48dd1b..3e293b5f8 100644 --- a/executor/src/witgen/data_structures/column_map.rs +++ b/executor/src/witgen/data_structures/column_map.rs @@ -3,7 +3,7 @@ use std::{ ops::{Index, IndexMut}, }; -use powdr_ast::analyzed::{PolyID, PolynomialType}; +use powdr_ast::analyzed::{PolynomialType, RawPolyID as PolyID}; use powdr_number::DegreeType; // Marker types for each PolynomialType @@ -30,19 +30,17 @@ pub type FixedColumnMap = ColumnMap; /// A Map indexed by polynomial ID, for a specific polynomial type (e.g. fixed or witness). /// For performance reasons, it uses a Vec internally and assumes that the polynomial IDs -/// are contiguous. It also assumes that all polynomials have the same degree. +/// are contiguous. #[derive(Clone)] pub struct ColumnMap { - pub degree: Option, values: Vec, _ptype: PhantomData, } impl ColumnMap { /// Create a new ColumnMap with the given initial value and size. - pub fn new(initial_value: V, size: usize, degree: Option) -> Self { + pub fn new(initial_value: V, size: usize, _degree: Option) -> Self { ColumnMap { - degree, values: vec![initial_value; size], _ptype: PhantomData, } @@ -50,9 +48,8 @@ impl ColumnMap { } impl ColumnMap { - pub fn from(values: impl Iterator, degree: Option) -> Self { + pub fn from(values: impl Iterator) -> Self { ColumnMap { - degree, values: values.collect(), _ptype: PhantomData, } @@ -64,30 +61,21 @@ impl ColumnMap { V: Default, { let mut values: Vec = (0..len).map(|_| V::default()).collect(); - let mut degree: Option = None; for (poly, value) in items { - values[poly.id as usize] = value; - assert_eq!(poly.ptype, T::P_TYPE); - if degree.is_none() { - degree = Some(poly.degree.unwrap()); - } - assert_eq!(poly.degree, degree); + values[poly.id() as usize] = value; + assert_eq!(poly.ptype(), T::P_TYPE); } ColumnMap { - degree, values, _ptype: PhantomData, } } pub fn keys(&self) -> impl Iterator { - let degree = self.degree; - (0..self.values.len()).map(move |i| PolyID { id: i as u64, ptype: T::P_TYPE, - degree, }) } @@ -113,7 +101,7 @@ impl Index<&PolyID> for ColumnMap { #[inline] fn index(&self, poly_id: &PolyID) -> &Self::Output { - assert!(poly_id.ptype == T::P_TYPE); + assert!(poly_id.ptype() == T::P_TYPE); &self.values[poly_id.id as usize] } } @@ -121,7 +109,7 @@ impl Index<&PolyID> for ColumnMap { impl IndexMut<&PolyID> for ColumnMap { #[inline] fn index_mut(&mut self, poly_id: &PolyID) -> &mut Self::Output { - assert!(poly_id.ptype == T::P_TYPE); + assert!(poly_id.ptype() == T::P_TYPE); &mut self.values[poly_id.id as usize] } } diff --git a/executor/src/witgen/data_structures/finalizable_data.rs b/executor/src/witgen/data_structures/finalizable_data.rs index b689ceec8..2652300a3 100644 --- a/executor/src/witgen/data_structures/finalizable_data.rs +++ b/executor/src/witgen/data_structures/finalizable_data.rs @@ -4,7 +4,7 @@ use std::{ }; use bit_vec::BitVec; -use powdr_ast::analyzed::PolyID; +use powdr_ast::analyzed::RawPolyID as PolyID; use powdr_number::FieldElement; use crate::witgen::rows::Row; diff --git a/executor/src/witgen/fixed_evaluator.rs b/executor/src/witgen/fixed_evaluator.rs index 77a57f9e6..5c91ef9af 100644 --- a/executor/src/witgen/fixed_evaluator.rs +++ b/executor/src/witgen/fixed_evaluator.rs @@ -23,7 +23,7 @@ impl<'a, T: FieldElement> SymbolicVariables for FixedEvaluator<'a, T> { poly.is_fixed(), "Can only access fixed columns in the fixed evaluator." ); - let col_data = self.fixed_data.fixed_cols[&poly.poly_id].values; + let col_data = self.fixed_data.fixed_cols[&poly.poly_id.raw].values; let degree = col_data.len(); let row = if poly.next { (self.row + 1) % degree diff --git a/executor/src/witgen/generator.rs b/executor/src/witgen/generator.rs index bbdce0ba8..a40419ddf 100644 --- a/executor/src/witgen/generator.rs +++ b/executor/src/witgen/generator.rs @@ -1,5 +1,5 @@ use powdr_ast::analyzed::{ - AlgebraicExpression as Expression, AlgebraicReference, Identity, PolyID, + AlgebraicExpression as Expression, AlgebraicReference, Identity, RawPolyID as PolyID, }; use powdr_number::{DegreeType, FieldElement}; use std::collections::{BTreeMap, HashMap, HashSet}; @@ -24,7 +24,6 @@ struct ProcessResult<'a, T: FieldElement> { } pub struct Generator<'a, T: FieldElement> { - degree: DegreeType, connecting_identities: BTreeMap>>, fixed_data: &'a FixedData<'a, T>, identities: Vec<&'a Identity>>, @@ -40,7 +39,7 @@ impl<'a, T: FieldElement> Machine<'a, T> for Generator<'a, T> { } fn degree(&self) -> DegreeType { - self.degree + self.fixed_data.analyzed.max_degree() } fn name(&self) -> &str { @@ -119,20 +118,11 @@ impl<'a, T: FieldElement> Generator<'a, T> { witnesses: HashSet, latch: Option>, ) -> Self { - let data = FinalizableData::new(&witnesses); + let witnesses = witnesses.into_iter().map(Into::into).collect(); - // get the degree of all witnesses, which must match - let degree = witnesses - .iter() - .map(|p| p.degree.unwrap()) - .reduce(|acc, degree| { - assert_eq!(acc, degree); - acc - }) - .unwrap(); + let data = FinalizableData::new(&witnesses); Self { - degree, connecting_identities: connecting_identities.clone(), name, fixed_data, @@ -235,6 +225,9 @@ impl<'a, T: FieldElement> Generator<'a, T> { &self.witnesses, [first_row].into_iter(), ); + + let degree = self.degree(); + let mut processor = VmProcessor::new( RowIndex::from_degree(row_offset, self.degree()), self.fixed_data, @@ -242,6 +235,7 @@ impl<'a, T: FieldElement> Generator<'a, T> { &self.witnesses, data, mutable_state, + degree, ); if let Some(outer_query) = outer_query { processor = processor.with_outer_query(outer_query); @@ -257,19 +251,15 @@ impl<'a, T: FieldElement> Generator<'a, T> { assert_eq!(self.data.len() as DegreeType, self.degree() + 1); let last_row = self.data.pop().unwrap(); - self.data[0] = WitnessColumnMap::from( - self.data[0] - .values() - .zip(last_row.values()) - .map(|(cell1, cell2)| match (&cell1.value, &cell2.value) { - (CellValue::Known(v1), CellValue::Known(v2)) => { - assert_eq!(v1, v2); - cell1.clone() - } - (CellValue::Known(_), _) => cell1.clone(), - _ => cell2.clone(), - }), - Some(self.degree()), - ); + self.data[0] = WitnessColumnMap::from(self.data[0].values().zip(last_row.values()).map( + |(cell1, cell2)| match (&cell1.value, &cell2.value) { + (CellValue::Known(v1), CellValue::Known(v2)) => { + assert_eq!(v1, v2); + cell1.clone() + } + (CellValue::Known(_), _) => cell1.clone(), + _ => cell2.clone(), + }, + )); } } diff --git a/executor/src/witgen/global_constraints.rs b/executor/src/witgen/global_constraints.rs index 27b2002c4..09541b29f 100644 --- a/executor/src/witgen/global_constraints.rs +++ b/executor/src/witgen/global_constraints.rs @@ -5,7 +5,7 @@ use num_traits::Zero; use powdr_ast::analyzed::{ AlgebraicBinaryOperation, AlgebraicBinaryOperator, AlgebraicExpression as Expression, - AlgebraicReference, Identity, IdentityKind, PolyID, PolynomialType, + AlgebraicReference, Identity, IdentityKind, PolynomialType, RawPolyID as PolyID, }; use powdr_number::FieldElement; @@ -32,7 +32,7 @@ impl<'a, T: FieldElement> RangeConstraintSet<&AlgebraicReference, T> { fn range_constraint(&self, id: &AlgebraicReference) -> Option> { assert!(!id.next); - self.range_constraints.get(&id.poly_id).cloned() + self.range_constraints.get(&id.poly_id.raw).cloned() } } @@ -93,8 +93,8 @@ pub struct GlobalConstraints { impl RangeConstraintSet<&AlgebraicReference, T> for GlobalConstraints { fn range_constraint(&self, id: &AlgebraicReference) -> Option> { assert!(!id.next); - let poly_id = id.poly_id; - match poly_id.ptype { + let poly_id = id.poly_id.raw; + match poly_id.ptype() { PolynomialType::Constant => self.fixed_constraints[&poly_id].clone(), PolynomialType::Committed => self.witness_constraints[&poly_id].clone(), PolynomialType::Intermediate => None, @@ -145,7 +145,7 @@ pub fn set_global_constraints<'a, T: FieldElement>( log::debug!("Determined the following global range constraints:"); for (poly_id, con) in &known_constraints { - if poly_id.ptype == PolynomialType::Committed { + if poly_id.ptype() == PolynomialType::Committed { log::debug!(" {}: {con}", fixed_data.column_name(poly_id)); } } @@ -158,7 +158,7 @@ pub fn set_global_constraints<'a, T: FieldElement>( let mut witness_constraints: WitnessColumnMap>> = fixed_data.witness_map_with(None); for (poly_id, con) in known_constraints { - if poly_id.ptype == PolynomialType::Committed { + if poly_id.ptype() == PolynomialType::Committed { // It's theoretically possible to have a constraint for both X and X'. // In that case, we take the conjunction. let con = witness_constraints[&poly_id] @@ -245,7 +245,7 @@ fn propagate_constraints( { if let Some(constraint) = known_constraints.get(&right.poly_id).cloned() { known_constraints - .entry(left.poly_id) + .entry(left.poly_id.raw) .and_modify(|existing| *existing = existing.conjunction(&constraint)) .or_insert(constraint); } @@ -304,7 +304,7 @@ fn is_binary_constraint(expr: &Expression) -> Option return None; } if (value1.is_zero() && value2.is_one()) || (value1.is_one() && value2.is_zero()) { - return Some(id1.poly_id); + return Some(id1.poly_id.raw); } } } @@ -340,7 +340,7 @@ fn try_transfer_constraints( .flat_map(|(poly, cons)| { if let Constraint::RangeConstraint(cons) = cons { assert!(!poly.next); - Some((poly.poly_id, cons)) + Some((poly.poly_id.raw, cons)) } else { None } @@ -359,8 +359,8 @@ fn smallest_period_candidate(fixed: &[T]) -> Option { mod test { use std::collections::BTreeMap; - use powdr_ast::analyzed::{PolyID, PolynomialType}; - use powdr_number::{DegreeType, GoldilocksField}; + use powdr_ast::analyzed::PolynomialType; + use powdr_number::{GoldilocksField}; use pretty_assertions::assert_eq; use test_log::test; @@ -402,19 +402,17 @@ mod test { ); } - fn constant_poly_id(i: u64, degree: DegreeType) -> PolyID { + fn constant_poly_id(i: u64) -> PolyID { PolyID { ptype: PolynomialType::Constant, id: i, - degree: Some(degree), } } - fn witness_poly_id(i: u64, degree: DegreeType) -> PolyID { + fn witness_poly_id(i: u64) -> PolyID { PolyID { ptype: PolynomialType::Committed, id: i, - degree: Some(degree), } } @@ -437,10 +435,9 @@ namespace Global(2**20); { D } in { SHIFTED }; "; let analyzed = powdr_pil_analyzer::analyze_string::(pil_source); - let degree = analyzed.max_degree(); let constants = crate::constant_evaluator::generate(&analyzed); let fixed_polys = (0..constants.len()) - .map(|i| constant_poly_id(i as u64, degree)) + .map(|i| constant_poly_id(i as u64)) .collect::>(); let mut known_constraints = fixed_polys .iter() @@ -453,20 +450,11 @@ namespace Global(2**20); known_constraints, vec![ // Global.BYTE - ( - constant_poly_id(0, degree), - RangeConstraint::from_max_bit(7) - ), + (constant_poly_id(0), RangeConstraint::from_max_bit(7)), // Global.BYTE2 - ( - constant_poly_id(1, degree), - RangeConstraint::from_max_bit(15) - ), + (constant_poly_id(1), RangeConstraint::from_max_bit(15)), // Global.SHIFTED - ( - constant_poly_id(2, degree), - RangeConstraint::from_mask(0xff0_u32) - ), + (constant_poly_id(2), RangeConstraint::from_mask(0xff0_u32)), ] .into_iter() .collect() @@ -479,34 +467,19 @@ namespace Global(2**20); known_constraints, vec![ // Global.A - (witness_poly_id(0, degree), RangeConstraint::from_max_bit(0)), + (witness_poly_id(0), RangeConstraint::from_max_bit(0)), // Global.B - (witness_poly_id(1, degree), RangeConstraint::from_max_bit(7)), + (witness_poly_id(1), RangeConstraint::from_max_bit(7)), // Global.C - ( - witness_poly_id(2, degree), - RangeConstraint::from_mask(0x2ff_u32) - ), + (witness_poly_id(2), RangeConstraint::from_mask(0x2ff_u32)), // Global.D - ( - witness_poly_id(3, degree), - RangeConstraint::from_mask(0xf0_u32) - ), + (witness_poly_id(3), RangeConstraint::from_mask(0xf0_u32)), // Global.BYTE - ( - constant_poly_id(0, degree), - RangeConstraint::from_max_bit(7) - ), + (constant_poly_id(0), RangeConstraint::from_max_bit(7)), // Global.BYTE2 - ( - constant_poly_id(1, degree), - RangeConstraint::from_max_bit(15) - ), + (constant_poly_id(1), RangeConstraint::from_max_bit(15)), // Global.SHIFTED - ( - constant_poly_id(2, degree), - RangeConstraint::from_mask(0xff0_u32) - ), + (constant_poly_id(2), RangeConstraint::from_mask(0xff0_u32)), ] .into_iter() .collect::>() @@ -525,12 +498,9 @@ namespace Global(1024); { X * 4 } in { bytes }; "; let analyzed = powdr_pil_analyzer::analyze_string::(pil_source); - let known_constraints = vec![( - constant_poly_id(0, analyzed.max_degree()), - RangeConstraint::from_max_bit(7), - )] - .into_iter() - .collect(); + let known_constraints = vec![(constant_poly_id(0), RangeConstraint::from_max_bit(7))] + .into_iter() + .collect(); assert_eq!(analyzed.identities.len(), 1); let (_, removed) = propagate_constraints( known_constraints, diff --git a/executor/src/witgen/machines/block_machine.rs b/executor/src/witgen/machines/block_machine.rs index ff07b8ddc..6030d7df1 100644 --- a/executor/src/witgen/machines/block_machine.rs +++ b/executor/src/witgen/machines/block_machine.rs @@ -17,8 +17,8 @@ use crate::witgen::{machines::Machine, EvalError, EvalValue, IncompleteCause}; use crate::witgen::{MutableState, QueryCallback}; use itertools::Itertools; use powdr_ast::analyzed::{ - AlgebraicExpression as Expression, AlgebraicReference, Identity, IdentityKind, PolyID, - PolynomialType, + AlgebraicExpression as Expression, AlgebraicReference, Identity, IdentityKind, PolynomialType, + RawPolyID as PolyID, }; use powdr_ast::parsed::visitor::ExpressionVisitable; use powdr_number::{DegreeType, FieldElement}; @@ -128,24 +128,15 @@ impl<'a, T: FieldElement> BlockMachine<'a, T> { connecting_identities: &BTreeMap>>, identities: &[&'a Identity>], witness_cols: &HashSet, + degree: u64, ) -> Option { - // get the degree of all witnesses, which must match - let degree = witness_cols - .iter() - .map(|p| p.degree.unwrap()) - .reduce(|acc, degree| { - assert_eq!(acc, degree); - acc - }) - .unwrap(); - let (is_permutation, block_size, latch_row) = detect_connection_type_and_block_size(fixed_data, connecting_identities, degree)?; for id in connecting_identities.values() { for r in id.right.expressions.iter() { if let Some(poly) = try_to_simple_poly(r) { - if poly.poly_id.ptype == PolynomialType::Constant { + if poly.poly_id.ptype() == PolynomialType::Constant { // It does not really make sense to have constant polynomials on the RHS // of a block machine lookup, as all constant polynomials are periodic, so // it would always return the same value. diff --git a/executor/src/witgen/machines/double_sorted_witness_machine.rs b/executor/src/witgen/machines/double_sorted_witness_machine.rs index 9a36c4370..3beafa820 100644 --- a/executor/src/witgen/machines/double_sorted_witness_machine.rs +++ b/executor/src/witgen/machines/double_sorted_witness_machine.rs @@ -10,7 +10,9 @@ use crate::witgen::{EvalResult, FixedData, MutableState, QueryCallback}; use crate::witgen::{EvalValue, IncompleteCause}; use powdr_number::{DegreeType, FieldElement}; -use powdr_ast::analyzed::{AlgebraicExpression as Expression, Identity, IdentityKind, PolyID}; +use powdr_ast::analyzed::{ + AlgebraicExpression as Expression, Identity, IdentityKind, RawPolyID as PolyID, +}; /// If all witnesses of a machine have a name in this list (disregarding the namespace), /// we'll consider it to be a double-sorted machine. @@ -81,17 +83,8 @@ impl<'a, T: FieldElement> DoubleSortedWitnesses<'a, T> { fixed_data: &'a FixedData, connecting_identities: &BTreeMap>>, witness_cols: &HashSet, + degree: u64, ) -> Option { - // get the degree of all witnesses, which must match - let degree = witness_cols - .iter() - .map(|p| p.degree.unwrap()) - .reduce(|acc, degree| { - assert_eq!(acc, degree); - acc - }) - .unwrap(); - // get the namespaces and column names let (mut namespaces, columns): (HashSet<_>, HashSet<_>) = witness_cols .iter() @@ -117,7 +110,7 @@ impl<'a, T: FieldElement> DoubleSortedWitnesses<'a, T> { .selector .as_ref() .and_then(|r| try_to_simple_poly(r)) - .map(|p| (i.id, p.poly_id)) + .map(|p| (i.id, p.poly_id.raw)) }) .collect::>>()?; diff --git a/executor/src/witgen/machines/fixed_lookup_machine.rs b/executor/src/witgen/machines/fixed_lookup_machine.rs index 1b174901b..5c2f7b63b 100644 --- a/executor/src/witgen/machines/fixed_lookup_machine.rs +++ b/executor/src/witgen/machines/fixed_lookup_machine.rs @@ -241,7 +241,7 @@ impl FixedLookup { ) -> EvalResult<'b, T> { if left.len() == 1 && !left.first().unwrap().is_constant() - && right.peek().unwrap().poly_id.ptype == PolynomialType::Constant + && right.peek().unwrap().poly_id.ptype() == PolynomialType::Constant { // Lookup of the form "c { X } in { B }". Might be a conditional range check. return self.process_range_check(rows, left.first().unwrap(), right.peek().unwrap()); @@ -337,7 +337,7 @@ impl FixedLookup { updates .constraints .into_iter() - .filter(|(poly, _)| poly.poly_id.ptype == PolynomialType::Committed) + .filter(|(poly, _)| poly.poly_id.ptype() == PolynomialType::Committed) .collect(), IncompleteCause::NotConcrete, )) @@ -357,7 +357,7 @@ impl RangeConstraintSet<&AlgebraicReference, T> for UnifiedRangeConstraints<'_, T> { fn range_constraint(&self, poly: &AlgebraicReference) -> Option> { - match poly.poly_id.ptype { + match poly.poly_id.ptype() { PolynomialType::Committed => self.witness_constraints.range_constraint(poly), PolynomialType::Constant => self.global_constraints.range_constraint(poly), PolynomialType::Intermediate => unimplemented!(), diff --git a/executor/src/witgen/machines/machine_extractor.rs b/executor/src/witgen/machines/machine_extractor.rs index ef3b5f581..59592e069 100644 --- a/executor/src/witgen/machines/machine_extractor.rs +++ b/executor/src/witgen/machines/machine_extractor.rs @@ -10,7 +10,9 @@ use super::KnownMachine; use crate::witgen::generator::Generator; use crate::witgen::machines::write_once_memory::WriteOnceMemory; use itertools::Itertools; -use powdr_ast::analyzed::{AlgebraicExpression as Expression, Identity, IdentityKind, PolyID}; +use powdr_ast::analyzed::{ + AlgebraicExpression as Expression, Identity, IdentityKind, RawPolyID as PolyID, +}; use powdr_ast::parsed::visitor::ExpressionVisitable; use powdr_ast::parsed::SelectedExpressions; use powdr_number::FieldElement; @@ -22,6 +24,25 @@ pub struct ExtractionOutput<'a, T: FieldElement> { pub base_witnesses: HashSet, } +fn to_raw_and_degree( + machine_witnesses: HashSet, +) -> (HashSet, u64) { + let mut res = HashSet::default(); + let mut degree = None; + for id in machine_witnesses { + res.insert(id.raw); + match degree { + None => { + degree = id.degree; + } + degree => { + assert_eq!(id.degree, degree); + } + } + } + (res, degree.unwrap()) +} + /// Finds machines in the witness columns and identities /// and returns a list of machines and the identities /// that are not "internal" to the machines. @@ -33,7 +54,14 @@ pub fn split_out_machines<'a, T: FieldElement>( let mut machines: Vec> = vec![]; - let all_witnesses = fixed.witness_cols.keys().collect::>(); + // we use `PolyID` because we need to extract the degree of each machine + let all_witnesses: HashSet = fixed + .analyzed + .committed_polys_in_source_order() + .iter() + .flat_map(|(symbol, _)| symbol.array_elements().map(|(_, id)| id)) + .collect(); + let mut remaining_witnesses = all_witnesses.clone(); let mut base_identities = identities.clone(); let mut id_counter = 0; @@ -112,12 +140,16 @@ pub fn split_out_machines<'a, T: FieldElement>( id_counter += 1; let name_with_type = |t: &str| format!("Secondary machine {id}: {name} ({t})"); + // Internal processing happens over `RawPolyID` + let (machine_witnesses, degree) = to_raw_and_degree(machine_witnesses); + if let Some(machine) = SortedWitnesses::try_new( name_with_type("SortedWitness"), fixed, &connecting_identities, &machine_identities, &machine_witnesses, + degree, ) { log::debug!("Detected machine: sorted witnesses / write-once memory"); machines.push(KnownMachine::SortedWitnesses(machine)); @@ -126,6 +158,7 @@ pub fn split_out_machines<'a, T: FieldElement>( fixed, &connecting_identities, &machine_witnesses, + degree, ) { log::debug!("Detected machine: memory"); machines.push(KnownMachine::DoubleSortedWitnesses(machine)); @@ -134,6 +167,7 @@ pub fn split_out_machines<'a, T: FieldElement>( fixed, &connecting_identities, &machine_identities, + degree, ) { log::debug!("Detected machine: write-once memory"); machines.push(KnownMachine::WriteOnceMemory(machine)); @@ -143,6 +177,7 @@ pub fn split_out_machines<'a, T: FieldElement>( &connecting_identities, &machine_identities, &machine_witnesses, + degree, ) { log::debug!("Detected machine: {machine}"); machines.push(KnownMachine::BlockMachine(machine)); @@ -177,6 +212,9 @@ pub fn split_out_machines<'a, T: FieldElement>( ))); } } + + let (remaining_witnesses, _) = to_raw_and_degree(remaining_witnesses); + ExtractionOutput { fixed_lookup, machines, @@ -189,10 +227,10 @@ pub fn split_out_machines<'a, T: FieldElement>( /// Two witnesses are row-connected if they are part of a polynomial identity /// or part of the same side of a lookup. fn all_row_connected_witnesses( - mut witnesses: HashSet, - all_witnesses: &HashSet, + mut witnesses: HashSet, + all_witnesses: &HashSet, identities: &[&Identity>], -) -> HashSet { +) -> HashSet { loop { let count = witnesses.len(); for i in identities { @@ -224,8 +262,10 @@ fn all_row_connected_witnesses( } /// Extracts all references to names from an identity. -pub fn refs_in_identity(identity: &Identity>) -> HashSet { - let mut refs: HashSet = Default::default(); +pub fn refs_in_identity( + identity: &Identity>, +) -> HashSet { + let mut refs: HashSet<_> = Default::default(); identity.pre_visit_expressions(&mut |expr| { ref_of_expression(expr).map(|id| refs.insert(id)); }); @@ -235,8 +275,8 @@ pub fn refs_in_identity(identity: &Identity>) -> HashSet( sel_expr: &SelectedExpressions>, -) -> HashSet { - let mut refs: HashSet = Default::default(); +) -> HashSet { + let mut refs: HashSet<_> = Default::default(); sel_expr.pre_visit_expressions(&mut |expr| { ref_of_expression(expr).map(|id| refs.insert(id)); }); @@ -245,7 +285,7 @@ pub fn refs_in_selected_expressions( /// Extracts all references to names from an expression, /// NON-recursively. -pub fn ref_of_expression(expr: &Expression) -> Option { +pub fn ref_of_expression(expr: &Expression) -> Option { match expr { Expression::Reference(p) => Some(p.poly_id), _ => None, diff --git a/executor/src/witgen/machines/sorted_witness_machine.rs b/executor/src/witgen/machines/sorted_witness_machine.rs index 7fde73702..737dacc52 100644 --- a/executor/src/witgen/machines/sorted_witness_machine.rs +++ b/executor/src/witgen/machines/sorted_witness_machine.rs @@ -12,7 +12,8 @@ use crate::witgen::{ }; use crate::witgen::{EvalValue, IncompleteCause, MutableState, QueryCallback}; use powdr_ast::analyzed::{ - AlgebraicExpression as Expression, AlgebraicReference, Identity, IdentityKind, PolyID, + AlgebraicExpression as Expression, AlgebraicReference, Identity, IdentityKind, + RawPolyID as PolyID, }; use powdr_number::{DegreeType, FieldElement}; @@ -42,21 +43,12 @@ impl<'a, T: FieldElement> SortedWitnesses<'a, T> { connecting_identities: &BTreeMap>>, identities: &[&Identity>], witnesses: &HashSet, + degree: u64, ) -> Option { if identities.len() != 1 { return None; } - // get the degree of all witnesses, which must match - let degree = witnesses - .iter() - .map(|p| p.degree.unwrap()) - .reduce(|acc, degree| { - assert_eq!(acc, degree); - acc - }) - .unwrap(); - check_identity(fixed_data, identities.first().unwrap(), degree).and_then(|key_col| { let witness_positions = witnesses .iter() @@ -170,7 +162,7 @@ fn check_constraint(constraint: &Expression) -> Option Machine<'a, T> for SortedWitnesses<'a, T> { @@ -238,7 +230,10 @@ impl<'a, T: FieldElement> SortedWitnesses<'a, T> { .map(|e| caller_rows.evaluate(e).unwrap()) .collect::>(); let rhs = self.rhs_references.get(&identity_id).unwrap(); - let key_index = rhs.iter().position(|&x| x.poly_id == self.key_col).unwrap(); + let key_index = rhs + .iter() + .position(|&x| x.poly_id.raw == self.key_col) + .unwrap(); let key_value = left[key_index].constant_value().ok_or_else(|| { format!( diff --git a/executor/src/witgen/machines/write_once_memory.rs b/executor/src/witgen/machines/write_once_memory.rs index 7a345b2c7..dae3b1136 100644 --- a/executor/src/witgen/machines/write_once_memory.rs +++ b/executor/src/witgen/machines/write_once_memory.rs @@ -3,7 +3,7 @@ use std::collections::{BTreeMap, HashMap}; use itertools::{Either, Itertools}; use powdr_ast::analyzed::{ - AlgebraicExpression as Expression, Identity, IdentityKind, PolyID, PolynomialType, + AlgebraicExpression as Expression, Identity, IdentityKind, PolynomialType, RawPolyID as PolyID, }; use powdr_number::{DegreeType, FieldElement}; @@ -47,6 +47,7 @@ impl<'a, T: FieldElement> WriteOnceMemory<'a, T> { fixed_data: &'a FixedData<'a, T>, connecting_identities: &BTreeMap>>, identities: &[&Identity>], + degree: u64, ) -> Option { if !identities.is_empty() { return None; @@ -92,23 +93,13 @@ impl<'a, T: FieldElement> WriteOnceMemory<'a, T> { // Build a Vec for the key and value polynomials let (key_polys, value_polys): (Vec<_>, Vec<_>) = rhs_polys.into_iter().partition_map(|p| { assert!(!p.next); - if p.poly_id.ptype == PolynomialType::Constant { - Either::Left(p.poly_id) + if p.poly_id.ptype() == PolynomialType::Constant { + Either::Left(p.poly_id.raw) } else { - Either::Right(p.poly_id) + Either::Right(p.poly_id.raw) } }); - // get the degree of all witnesses, which must match - let degree = value_polys - .iter() - .map(|p| p.degree.unwrap()) - .reduce(|acc, degree| { - assert_eq!(acc, degree); - acc - }) - .unwrap(); - let mut key_to_index = BTreeMap::new(); for row in 0..degree { let key = key_polys @@ -148,7 +139,7 @@ impl<'a, T: FieldElement> WriteOnceMemory<'a, T> { .iter() .zip(identity.right.expressions.iter()) .partition(|(_, r)| { - try_to_simple_poly(r).unwrap().poly_id.ptype == PolynomialType::Constant + try_to_simple_poly(r).unwrap().poly_id.ptype() == PolynomialType::Constant }); let key = key_expressions .into_iter() diff --git a/executor/src/witgen/mod.rs b/executor/src/witgen/mod.rs index d395bf058..3d3aeb332 100644 --- a/executor/src/witgen/mod.rs +++ b/executor/src/witgen/mod.rs @@ -3,8 +3,8 @@ use std::rc::Rc; use std::sync::Arc; use powdr_ast::analyzed::{ - AlgebraicExpression, AlgebraicReference, Analyzed, Expression, FunctionValueDefinition, PolyID, - PolynomialType, SymbolKind, TypedExpression, + AlgebraicExpression, AlgebraicReference, Analyzed, Expression, FunctionValueDefinition, + PolynomialType, RawPolyID as PolyID, SymbolKind, TypedExpression, }; use powdr_ast::parsed::visitor::ExpressionVisitable; use powdr_ast::parsed::{FunctionKind, LambdaExpression}; @@ -315,7 +315,7 @@ impl<'a, T: FieldElement> FixedData<'a, T> { }) .collect::>() }, - ), Some(analyzed.max_degree())); + )); if !external_witness_values.is_empty() { let available_columns = witness_cols @@ -329,10 +329,8 @@ impl<'a, T: FieldElement> FixedData<'a, T> { ); } - let fixed_cols = FixedColumnMap::from( - fixed_col_values.iter().map(|(n, v)| FixedColumn::new(n, v)), - Some(analyzed.max_degree()), - ); + let fixed_cols = + FixedColumnMap::from(fixed_col_values.iter().map(|(n, v)| FixedColumn::new(n, v))); // The global range constraints are not set yet. let global_range_constraints = GlobalConstraints { @@ -391,7 +389,7 @@ impl<'a, T: FieldElement> FixedData<'a, T> { } fn column_name(&self, poly_id: &PolyID) -> &str { - match poly_id.ptype { + match poly_id.ptype() { PolynomialType::Committed => &self.witness_cols[poly_id].poly.name, PolynomialType::Constant => &self.fixed_cols[poly_id].name, PolynomialType::Intermediate => unimplemented!(), @@ -443,12 +441,12 @@ pub struct WitnessColumn<'a, T> { impl<'a, T> WitnessColumn<'a, T> { pub fn new( - poly_id: PolyID, + poly_id: powdr_ast::analyzed::PolyID, name: &str, value: Option<&'a FunctionValueDefinition>, external_values: Option<&'a Vec>, ) -> WitnessColumn<'a, T> { - assert_eq!(poly_id.ptype, PolynomialType::Committed); + assert_eq!(poly_id.ptype(), PolynomialType::Committed); let query = if let Some(FunctionValueDefinition::Expression(TypedExpression { e: diff --git a/executor/src/witgen/processor.rs b/executor/src/witgen/processor.rs index 09faab656..1fd42324f 100644 --- a/executor/src/witgen/processor.rs +++ b/executor/src/witgen/processor.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, HashSet}; use powdr_ast::analyzed::PolynomialType; use powdr_ast::analyzed::{ - AlgebraicExpression as Expression, AlgebraicReference, Identity, PolyID, + AlgebraicExpression as Expression, AlgebraicReference, Identity, RawPolyID as PolyID, }; use powdr_number::{DegreeType, FieldElement}; @@ -99,22 +99,11 @@ impl<'a, 'b, 'c, T: FieldElement, Q: QueryCallback> Processor<'a, 'b, 'c, T, fixed_data: &'a FixedData<'a, T>, witness_cols: &'c HashSet, ) -> Self { - // get the degree of all witnesses, which must match - let degree = witness_cols - .iter() - .map(|p| p.degree.unwrap()) - .reduce(|acc, degree| { - assert_eq!(acc, degree); - acc - }) - .unwrap(); - let is_relevant_witness = WitnessColumnMap::from( fixed_data .witness_cols .keys() .map(|poly_id| witness_cols.contains(&poly_id)), - Some(degree), ); let prover_query_witnesses = fixed_data .witness_cols @@ -153,7 +142,7 @@ impl<'a, 'b, 'c, T: FieldElement, Q: QueryCallback> Processor<'a, 'b, 'c, T, if let Some(right_poly) = try_to_simple_poly(r).map(|p| p.poly_id) { if let Some(l) = l.constant_value() { log::trace!(" {} = {}", r, l); - inputs.push((right_poly, l)); + inputs.push((right_poly.raw, l)); } } } @@ -356,7 +345,8 @@ Known values in current row (local: {row_index}, global {global_row_index}): } } for (poly, _) in &input_updates.constraints { - self.previously_set_inputs.insert(poly.poly_id, row_index); + self.previously_set_inputs + .insert(poly.poly_id.raw, row_index); } self.apply_updates(row_index, &input_updates, || "inputs".to_string()) } @@ -437,11 +427,11 @@ Known values in current row (local: {row_index}, global {global_row_index}): // Have to materialize the other cells to please the borrow checker... let others = self .copy_constraints - .iter_equivalence_class((poly.poly_id, row)) + .iter_equivalence_class((poly.poly_id.raw, row)) .skip(1) .collect::>(); for (other_poly, other_row) in others { - if other_poly.ptype != PolynomialType::Committed { + if other_poly.ptype() != PolynomialType::Committed { unimplemented!( "Copy constraints to fixed columns are not yet supported (#1335)!" ); diff --git a/executor/src/witgen/query_processor.rs b/executor/src/witgen/query_processor.rs index fd97d1662..d073d3028 100644 --- a/executor/src/witgen/query_processor.rs +++ b/executor/src/witgen/query_processor.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use powdr_ast::analyzed::Challenge; -use powdr_ast::analyzed::{AlgebraicReference, Expression, PolyID, PolynomialType}; +use powdr_ast::analyzed::{AlgebraicReference, Expression, PolynomialType, RawPolyID as PolyID}; use powdr_ast::parsed::types::Type; use powdr_number::{BigInt, FieldElement}; use powdr_pil_analyzer::evaluator::{self, Definitions, EvalError, SymbolLookup, Value}; @@ -138,7 +138,7 @@ impl<'a, T: FieldElement> SymbolLookup<'a, T> for Symbols<'a, T> { &self, poly_ref: &AlgebraicReference, ) -> Result>, EvalError> { - Ok(Value::FieldElement(match poly_ref.poly_id.ptype { + Ok(Value::FieldElement(match poly_ref.poly_id.ptype() { PolynomialType::Committed | PolynomialType::Intermediate => self .rows .get_value(poly_ref) diff --git a/executor/src/witgen/rows.rs b/executor/src/witgen/rows.rs index 380f8ed70..d4c0d3b15 100644 --- a/executor/src/witgen/rows.rs +++ b/executor/src/witgen/rows.rs @@ -5,7 +5,9 @@ use std::{ }; use itertools::Itertools; -use powdr_ast::analyzed::{AlgebraicExpression as Expression, AlgebraicReference, PolyID}; +use powdr_ast::analyzed::{ + AlgebraicExpression as Expression, AlgebraicReference, RawPolyID as PolyID, +}; use powdr_number::{DegreeType, FieldElement}; use crate::witgen::Constraint; @@ -226,7 +228,6 @@ impl<'a, T: FieldElement> Row<'a, T> { }; Cell { name, value } }), - Some(row.num_rows), ) } @@ -270,12 +271,9 @@ impl<'a, T: FieldElement> Row<'a, T> { impl From> for WitnessColumnMap { /// Builds a map from polynomial ID to value. Unknown values are set to zero. fn from(val: Row) -> Self { - let degree = val.degree; - WitnessColumnMap::from( val.into_iter() .map(|(_, cell)| cell.value.unwrap_or_default()), - degree, ) } } diff --git a/executor/src/witgen/vm_processor.rs b/executor/src/witgen/vm_processor.rs index 20c8d9287..59833c135 100644 --- a/executor/src/witgen/vm_processor.rs +++ b/executor/src/witgen/vm_processor.rs @@ -1,7 +1,8 @@ use indicatif::{ProgressBar, ProgressStyle}; use itertools::Itertools; use powdr_ast::analyzed::{ - AlgebraicExpression as Expression, AlgebraicReference, Identity, IdentityKind, PolyID, + AlgebraicExpression as Expression, AlgebraicReference, Identity, IdentityKind, + RawPolyID as PolyID, }; use powdr_ast::indent; use powdr_number::{DegreeType, FieldElement}; @@ -71,17 +72,8 @@ impl<'a, 'b, 'c, T: FieldElement, Q: QueryCallback> VmProcessor<'a, 'b, 'c, T witnesses: &'c HashSet, data: FinalizableData<'a, T>, mutable_state: &'c mut MutableState<'a, 'b, T, Q>, + degree: u64, ) -> Self { - // get the degree of all witnesses, which must match - let degree = witnesses - .iter() - .map(|p| p.degree.unwrap()) - .reduce(|acc, degree| { - assert_eq!(acc, degree); - acc - }) - .unwrap(); - let (identities_with_next, identities_without_next): (Vec<_>, Vec<_>) = identities .iter() .partition(|identity| identity.contains_next_ref()); From c9e0ebd8725a1902de1aad92fb70d8e3fcf7f282 Mon Sep 17 00:00:00 2001 From: schaeff Date: Tue, 25 Jun 2024 16:27:36 +0200 Subject: [PATCH 06/39] clean --- ast/src/analyzed/mod.rs | 8 -------- .../src/estark/json_exporter/expression_counter.rs | 4 ++-- backend/src/estark/json_exporter/mod.rs | 4 ++-- executor/src/witgen/data_structures/column_map.rs | 3 +-- executor/src/witgen/global_constraints.rs | 12 ++++++------ executor/src/witgen/machines/block_machine.rs | 2 +- .../src/witgen/machines/fixed_lookup_machine.rs | 10 +++++----- executor/src/witgen/machines/machine_extractor.rs | 4 ++-- .../src/witgen/machines/sorted_witness_machine.rs | 2 +- executor/src/witgen/mod.rs | 14 +++----------- executor/src/witgen/processor.rs | 12 ++++++------ executor/src/witgen/query_processor.rs | 2 +- executor/src/witgen/rows.rs | 8 ++++---- executor/src/witgen/symbolic_witness_evaluator.rs | 2 +- 14 files changed, 35 insertions(+), 52 deletions(-) diff --git a/ast/src/analyzed/mod.rs b/ast/src/analyzed/mod.rs index 0fe3d9cb9..3b74248cc 100644 --- a/ast/src/analyzed/mod.rs +++ b/ast/src/analyzed/mod.rs @@ -1156,14 +1156,6 @@ pub struct PolyID { pub degree: Option, } -impl std::ops::Deref for PolyID { - type Target = RawPolyID; - - fn deref(&self) -> &Self::Target { - &self.raw - } -} - impl PolyID { pub fn with_id(mut self, id: u64) -> Self { self.raw = self.raw.with_id(id); diff --git a/backend/src/estark/json_exporter/expression_counter.rs b/backend/src/estark/json_exporter/expression_counter.rs index d3cf84628..66a0b3645 100644 --- a/backend/src/estark/json_exporter/expression_counter.rs +++ b/backend/src/estark/json_exporter/expression_counter.rs @@ -20,8 +20,8 @@ pub fn compute_intermediate_expression_ids(analyzed: &Analyzed) -> HashMap poly.expression_count() } else if let Some((poly, _)) = analyzed.intermediate_columns.get(name) { assert!(poly.kind == SymbolKind::Poly(PolynomialType::Intermediate)); - for (index, (_, id)) in poly.array_elements().enumerate() { - ids.insert(id.id, (expression_counter + index) as u64); + for (index, (_, poly_id)) in poly.array_elements().enumerate() { + ids.insert(poly_id.id(), (expression_counter + index) as u64); } poly.expression_count() } else { diff --git a/backend/src/estark/json_exporter/mod.rs b/backend/src/estark/json_exporter/mod.rs index be07c5661..20f44440b 100644 --- a/backend/src/estark/json_exporter/mod.rs +++ b/backend/src/estark/json_exporter/mod.rs @@ -54,11 +54,11 @@ pub fn export(analyzed: &Analyzed) -> PIL { StatementIdentifier::Definition(name) => { if let Some((poly, value)) = analyzed.intermediate_columns.get(name) { assert_eq!(poly.kind, SymbolKind::Poly(PolynomialType::Intermediate)); - for ((_, id), value) in poly.array_elements().zip(value) { + for ((_, poly_id), value) in poly.array_elements().zip(value) { let expression_id = exporter.extract_expression(value, 1); assert_eq!( expression_id, - exporter.intermediate_poly_expression_ids[&id.id] as usize + exporter.intermediate_poly_expression_ids[&poly_id.id()] as usize ); } } diff --git a/executor/src/witgen/data_structures/column_map.rs b/executor/src/witgen/data_structures/column_map.rs index 3e293b5f8..22b06d725 100644 --- a/executor/src/witgen/data_structures/column_map.rs +++ b/executor/src/witgen/data_structures/column_map.rs @@ -4,7 +4,6 @@ use std::{ }; use powdr_ast::analyzed::{PolynomialType, RawPolyID as PolyID}; -use powdr_number::DegreeType; // Marker types for each PolynomialType #[derive(Clone, Copy)] @@ -39,7 +38,7 @@ pub struct ColumnMap { impl ColumnMap { /// Create a new ColumnMap with the given initial value and size. - pub fn new(initial_value: V, size: usize, _degree: Option) -> Self { + pub fn new(initial_value: V, size: usize) -> Self { ColumnMap { values: vec![initial_value; size], _ptype: PhantomData, diff --git a/executor/src/witgen/global_constraints.rs b/executor/src/witgen/global_constraints.rs index 09541b29f..de707b749 100644 --- a/executor/src/witgen/global_constraints.rs +++ b/executor/src/witgen/global_constraints.rs @@ -93,10 +93,10 @@ pub struct GlobalConstraints { impl RangeConstraintSet<&AlgebraicReference, T> for GlobalConstraints { fn range_constraint(&self, id: &AlgebraicReference) -> Option> { assert!(!id.next); - let poly_id = id.poly_id.raw; + let poly_id = id.poly_id; match poly_id.ptype() { - PolynomialType::Constant => self.fixed_constraints[&poly_id].clone(), - PolynomialType::Committed => self.witness_constraints[&poly_id].clone(), + PolynomialType::Constant => self.fixed_constraints[&poly_id.raw].clone(), + PolynomialType::Committed => self.witness_constraints[&poly_id.raw].clone(), PolynomialType::Intermediate => None, } } @@ -243,7 +243,7 @@ fn propagate_constraints( if let (Some(left), Some(right)) = (try_to_simple_poly(left), try_to_simple_poly(right)) { - if let Some(constraint) = known_constraints.get(&right.poly_id).cloned() { + if let Some(constraint) = known_constraints.get(&right.poly_id.raw).cloned() { known_constraints .entry(left.poly_id.raw) .and_modify(|existing| *existing = existing.conjunction(&constraint)) @@ -256,7 +256,7 @@ fn propagate_constraints( // provides all values in the span. if let Some(name) = try_to_simple_poly(&identity.right.expressions[0]) { if try_to_simple_poly(&identity.left.expressions[0]).is_some() - && full_span.contains(&name.poly_id) + && full_span.contains(&name.poly_id.raw) { remove = true; } @@ -360,7 +360,7 @@ mod test { use std::collections::BTreeMap; use powdr_ast::analyzed::PolynomialType; - use powdr_number::{GoldilocksField}; + use powdr_number::GoldilocksField; use pretty_assertions::assert_eq; use test_log::test; diff --git a/executor/src/witgen/machines/block_machine.rs b/executor/src/witgen/machines/block_machine.rs index 6030d7df1..0458bd50b 100644 --- a/executor/src/witgen/machines/block_machine.rs +++ b/executor/src/witgen/machines/block_machine.rs @@ -252,7 +252,7 @@ fn try_to_period( return None; } - let values = fixed_data.fixed_cols[&poly.poly_id].values; + let values = fixed_data.fixed_cols[&poly.poly_id.raw].values; let offset = values.iter().position(|v| v.is_one())?; let period = 1 + values.iter().skip(offset + 1).position(|v| v.is_one())?; diff --git a/executor/src/witgen/machines/fixed_lookup_machine.rs b/executor/src/witgen/machines/fixed_lookup_machine.rs index 5c2f7b63b..89e2dafef 100644 --- a/executor/src/witgen/machines/fixed_lookup_machine.rs +++ b/executor/src/witgen/machines/fixed_lookup_machine.rs @@ -90,23 +90,23 @@ impl IndexedColumns { "Generating index for lookup in columns (in: {}, out: {})", sorted_input_fixed_columns .iter() - .map(|c| fixed_data.column_name(c).to_string()) + .map(|c| fixed_data.column_name(&c.raw).to_string()) .join(", "), sorted_output_fixed_columns .iter() - .map(|c| fixed_data.column_name(c).to_string()) + .map(|c| fixed_data.column_name(&c.raw).to_string()) .join(", ") ); // get all values for the columns to be indexed let input_column_values = sorted_input_fixed_columns .iter() - .map(|id| fixed_data.fixed_cols[id].values) + .map(|id| fixed_data.fixed_cols[&id.raw].values) .collect::>(); let output_column_values = sorted_output_fixed_columns .iter() - .map(|id| fixed_data.fixed_cols[id].values) + .map(|id| fixed_data.fixed_cols[&id.raw].values) .collect::>(); let degree = input_column_values @@ -295,7 +295,7 @@ impl FixedLookup { let output = output_columns .iter() - .map(|column| fixed_data.fixed_cols[column].values[row]); + .map(|column| fixed_data.fixed_cols[&column.raw].values[row]); let mut result = EvalValue::complete(vec![]); for (l, r) in output_expressions.into_iter().zip(output) { diff --git a/executor/src/witgen/machines/machine_extractor.rs b/executor/src/witgen/machines/machine_extractor.rs index 59592e069..7c6f16ee1 100644 --- a/executor/src/witgen/machines/machine_extractor.rs +++ b/executor/src/witgen/machines/machine_extractor.rs @@ -111,7 +111,7 @@ pub fn split_out_machines<'a, T: FieldElement>( "\nExtracted a machine with the following witnesses:\n{} \n and identities:\n{} \n and connecting identities:\n{}", machine_witnesses .iter() - .map(|s| fixed.column_name(s)) + .map(|s| fixed.column_name(&s.raw)) .sorted() .collect::>() .join(", "), @@ -128,7 +128,7 @@ pub fn split_out_machines<'a, T: FieldElement>( ); let first_witness = machine_witnesses.iter().next().unwrap(); - let first_witness_name = fixed.column_name(first_witness); + let first_witness_name = fixed.column_name(&first_witness.raw); let namespace = first_witness_name .rfind('.') .map(|idx| &first_witness_name[..idx]); diff --git a/executor/src/witgen/machines/sorted_witness_machine.rs b/executor/src/witgen/machines/sorted_witness_machine.rs index 737dacc52..f120a432d 100644 --- a/executor/src/witgen/machines/sorted_witness_machine.rs +++ b/executor/src/witgen/machines/sorted_witness_machine.rs @@ -248,7 +248,7 @@ impl<'a, T: FieldElement> SortedWitnesses<'a, T> { .entry(key_value) .or_insert_with(|| vec![None; self.witness_positions.len()]); for (l, &r) in left.iter().zip(rhs.iter()).skip(1) { - let stored_value = &mut stored_values[self.witness_positions[&r.poly_id]]; + let stored_value = &mut stored_values[self.witness_positions[&r.poly_id.raw]]; match stored_value { // There is a stored value Some(v) => { diff --git a/executor/src/witgen/mod.rs b/executor/src/witgen/mod.rs index 3d3aeb332..cee505f69 100644 --- a/executor/src/witgen/mod.rs +++ b/executor/src/witgen/mod.rs @@ -334,16 +334,8 @@ impl<'a, T: FieldElement> FixedData<'a, T> { // The global range constraints are not set yet. let global_range_constraints = GlobalConstraints { - witness_constraints: WitnessColumnMap::new( - None, - witness_cols.len(), - Some(analyzed.max_degree()), - ), - fixed_constraints: FixedColumnMap::new( - None, - fixed_cols.len(), - Some(analyzed.max_degree()), - ), + witness_constraints: WitnessColumnMap::new(None, witness_cols.len()), + fixed_constraints: FixedColumnMap::new(None, fixed_cols.len()), }; FixedData { @@ -385,7 +377,7 @@ impl<'a, T: FieldElement> FixedData<'a, T> { } fn witness_map_with(&self, initial_value: V) -> WitnessColumnMap { - WitnessColumnMap::new(initial_value, self.witness_cols.len(), None) + WitnessColumnMap::new(initial_value, self.witness_cols.len()) } fn column_name(&self, poly_id: &PolyID) -> &str { diff --git a/executor/src/witgen/processor.rs b/executor/src/witgen/processor.rs index 1fd42324f..d6bb7ca48 100644 --- a/executor/src/witgen/processor.rs +++ b/executor/src/witgen/processor.rs @@ -304,7 +304,7 @@ Known values in current row (local: {row_index}, global {global_row_index}): .constraints .into_iter() .filter(|(poly, update)| match update { - Constraint::Assignment(_) => !self.is_relevant_witness[&poly.poly_id], + Constraint::Assignment(_) => !self.is_relevant_witness[&poly.poly_id.raw], // Range constraints are currently not communicated between callee and caller. Constraint::RangeConstraint(_) => false, }) @@ -334,13 +334,13 @@ Known values in current row (local: {row_index}, global {global_row_index}): for (poly, _) in &input_updates.constraints { let poly_id = poly.poly_id; - if let Some(start_row) = self.previously_set_inputs.remove(&poly_id) { + if let Some(start_row) = self.previously_set_inputs.remove(&poly_id.raw) { log::trace!( " Resetting previously set inputs for column: {}", - self.fixed_data.column_name(&poly_id) + self.fixed_data.column_name(&poly_id.raw) ); for row_index in start_row..row_index { - self.data[row_index][&poly_id].value = CellValue::Unknown; + self.data[row_index][&poly_id.raw].value = CellValue::Unknown; } } } @@ -387,7 +387,7 @@ Known values in current row (local: {row_index}, global {global_row_index}): let mut progress = false; for (poly, c) in &updates.constraints { - if self.witness_cols.contains(&poly.poly_id) { + if self.witness_cols.contains(&poly.poly_id.raw) { // Build RowUpdater // (a bit complicated, because we need two mutable // references to elements of the same vector) @@ -441,7 +441,7 @@ Known values in current row (local: {row_index}, global {global_row_index}): self.set_value(local_index, expression, *v, || { format!( "Copy constraint: {} (Row {}) -> {} (Row {})", - self.fixed_data.column_name(&poly.poly_id), + self.fixed_data.column_name(&poly.poly_id.raw), row, self.fixed_data.column_name(&other_poly), other_row diff --git a/executor/src/witgen/query_processor.rs b/executor/src/witgen/query_processor.rs index d073d3028..8d63f37d3 100644 --- a/executor/src/witgen/query_processor.rs +++ b/executor/src/witgen/query_processor.rs @@ -144,7 +144,7 @@ impl<'a, T: FieldElement> SymbolLookup<'a, T> for Symbols<'a, T> { .get_value(poly_ref) .ok_or(EvalError::DataNotAvailable)?, PolynomialType::Constant => { - let values = self.fixed_data.fixed_cols[&poly_ref.poly_id].values; + let values = self.fixed_data.fixed_cols[&poly_ref.poly_id.raw].values; let row = self.rows.current_row_index + if poly_ref.next { 1 } else { 0 }; values[usize::from(row)] } diff --git a/executor/src/witgen/rows.rs b/executor/src/witgen/rows.rs index d4c0d3b15..44048b4ca 100644 --- a/executor/src/witgen/rows.rs +++ b/executor/src/witgen/rows.rs @@ -322,8 +322,8 @@ impl<'row, 'a, T: FieldElement> RowUpdater<'row, 'a, T> { fn get_cell_mut<'b>(&'b mut self, poly: &AlgebraicReference) -> &'b mut Cell<'a, T> { match poly.next { - false => &mut self.current[&poly.poly_id], - true => &mut self.next[&poly.poly_id], + false => &mut self.current[&poly.poly_id.raw], + true => &mut self.next[&poly.poly_id.raw], } } @@ -393,8 +393,8 @@ impl<'row, 'a, T: FieldElement> RowPair<'row, 'a, T> { /// [RowPair::from_single_row]. fn get_cell(&self, poly: &AlgebraicReference) -> &Cell { match (poly.next, self.next.as_ref()) { - (false, _) => &self.current[&poly.poly_id], - (true, Some(next)) => &next[&poly.poly_id], + (false, _) => &self.current[&poly.poly_id.raw], + (true, Some(next)) => &next[&poly.poly_id.raw], (true, None) => panic!("Tried to access next row, but it is not available."), } } diff --git a/executor/src/witgen/symbolic_witness_evaluator.rs b/executor/src/witgen/symbolic_witness_evaluator.rs index 6d9831d8b..4b2dc1048 100644 --- a/executor/src/witgen/symbolic_witness_evaluator.rs +++ b/executor/src/witgen/symbolic_witness_evaluator.rs @@ -45,7 +45,7 @@ where self.witness_access.value(poly) } else { // Constant polynomial (or something else) - let values = self.fixed_data.fixed_cols[&poly.poly_id].values; + let values = self.fixed_data.fixed_cols[&poly.poly_id.raw].values; let row = if poly.next { self.row + 1 } else { self.row } % (values.len() as DegreeType); Ok(values[row as usize].into()) From 925611ceee98f0531a8d3df529f7b5844a4a88ce Mon Sep 17 00:00:00 2001 From: schaeff Date: Tue, 25 Jun 2024 23:30:02 +0200 Subject: [PATCH 07/39] fix tests --- ast/src/analyzed/display.rs | 4 ++-- pil-analyzer/tests/condenser.rs | 4 ++-- pil-analyzer/tests/parse_display.rs | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ast/src/analyzed/display.rs b/ast/src/analyzed/display.rs index 754dcc1db..6f4c72aa3 100644 --- a/ast/src/analyzed/display.rs +++ b/ast/src/analyzed/display.rs @@ -31,9 +31,9 @@ impl Display for Analyzed { current_namespace = namespace; writeln!( f, - "namespace {}({});", + "namespace {}{};", current_namespace.relative_to(&Default::default()), - degree.map(|d| d.to_string()).unwrap_or_default() + degree.map(|d| format!("({d})")).unwrap_or_default() )?; }; Ok((name, !current_namespace.is_empty())) diff --git a/pil-analyzer/tests/condenser.rs b/pil-analyzer/tests/condenser.rs index fae7751da..0e74dcd0b 100644 --- a/pil-analyzer/tests/condenser.rs +++ b/pil-analyzer/tests/condenser.rs @@ -122,9 +122,9 @@ pub fn degree() { w = std::convert::expr(d); "#; let formatted = analyze_string::(input).to_string(); - let expected = r#"namespace std::convert(); + let expected = r#"namespace std::convert; let expr = []; -namespace std::prover(); +namespace std::prover; let degree = []; namespace Main(8); let d: int = std::prover::degree(); diff --git a/pil-analyzer/tests/parse_display.rs b/pil-analyzer/tests/parse_display.rs index 4435a3a7f..6d40054ed 100644 --- a/pil-analyzer/tests/parse_display.rs +++ b/pil-analyzer/tests/parse_display.rs @@ -695,7 +695,7 @@ fn namespace_no_degree() { namespace T(8); let k = X::y; "; - let expected = "namespace X(); + let expected = "namespace X; let y: int = 7; namespace T(8); let k: int = X.y; @@ -711,7 +711,7 @@ fn find_in_prelude() { namespace T(8); let k = y; "; - let expected = "namespace std::prelude(8); + let expected = "namespace std::prelude; let y: int = 7; namespace T(8); let k: int = std::prelude::y; From 17da5cec9fb26204a31547f42e9a1cc26a4c3f68 Mon Sep 17 00:00:00 2001 From: schaeff Date: Wed, 26 Jun 2024 12:18:45 +0200 Subject: [PATCH 08/39] fix estark artifacts --- ast/src/analyzed/mod.rs | 13 ++++ backend/src/estark/mod.rs | 87 +++++++++++++++++---------- backend/src/estark/polygon_wrapper.rs | 5 +- backend/src/estark/starky_wrapper.rs | 7 ++- backend/src/halo2/mod.rs | 3 + backend/src/lib.rs | 2 + pipeline/src/verify.rs | 2 +- pipeline/tests/pil.rs | 4 +- 8 files changed, 83 insertions(+), 40 deletions(-) diff --git a/ast/src/analyzed/mod.rs b/ast/src/analyzed/mod.rs index 3b74248cc..87e95c45e 100644 --- a/ast/src/analyzed/mod.rs +++ b/ast/src/analyzed/mod.rs @@ -43,6 +43,11 @@ pub struct Analyzed { } impl Analyzed { + /// Returns the max of all degrees in this [`Analyzed`]. + /// + /// # Panics + /// + /// Panics if there are no symbols pub fn max_degree(&self) -> DegreeType { self.definitions .values() @@ -51,6 +56,14 @@ impl Analyzed { .unwrap() } + /// Returns the set of all degrees in this [`Analyzed`]. + pub fn degrees(&self) -> HashSet { + self.definitions + .values() + .filter_map(|(symbol, _)| symbol.degree) + .collect::>() + } + /// @returns the number of committed polynomials (with multiplicities for arrays) pub fn commitment_count(&self) -> usize { self.declaration_type_count(PolynomialType::Committed) diff --git a/backend/src/estark/mod.rs b/backend/src/estark/mod.rs index 21110804b..594962df1 100644 --- a/backend/src/estark/mod.rs +++ b/backend/src/estark/mod.rs @@ -1,10 +1,13 @@ +mod bin_exporter; mod json_exporter; #[cfg(feature = "estark-polygon")] pub mod polygon_wrapper; pub mod starky_wrapper; use std::{ - fs::{hard_link, remove_file}, + borrow::Cow, + fs::File, + io::{self, BufWriter, Write}, iter::{once, repeat}, path::{Path, PathBuf}, }; @@ -13,7 +16,7 @@ use crate::{Backend, BackendFactory, BackendOptions, Error, Proof}; use powdr_ast::analyzed::Analyzed; use powdr_executor::witgen::WitgenCallback; -use powdr_number::{buffered_write_file, write_polys_file, DegreeType, FieldElement}; +use powdr_number::{DegreeType, FieldElement}; use serde::Serialize; use starky::types::{StarkStruct, Step, PIL}; @@ -64,7 +67,7 @@ fn create_stark_struct(degree: DegreeType, hash_type: &str) -> StarkStruct { } } -type PatchedConstants = Vec<(String, Vec)>; +type Constants<'a, F> = Cow<'a, [(String, Vec)]>; /// eStark provers require a fixed column with the equivalent semantics to /// Polygon zkEVM's `L1` column. Powdr generated PIL will always have @@ -75,7 +78,7 @@ type PatchedConstants = Vec<(String, Vec)>; fn first_step_fixup<'a, F: FieldElement>( pil: &'a Analyzed, fixed: &'a [(String, Vec)], -) -> (PIL, Option>) { +) -> (PIL, Constants<'a, F>) { let degree = pil.max_degree(); let mut pil: PIL = json_exporter::export(pil); @@ -96,21 +99,19 @@ fn first_step_fixup<'a, F: FieldElement>( }, ); - Some( - fixed - .iter() - .cloned() - .chain(once(( - "main.first_step".to_string(), - once(F::one()) - .chain(repeat(F::zero())) - .take(degree as usize) - .collect(), - ))) - .collect(), - ) + fixed + .iter() + .cloned() + .chain(once(( + "main.first_step".to_string(), + once(F::one()) + .chain(repeat(F::zero())) + .take(degree as usize) + .collect(), + ))) + .collect() } else { - None + fixed.into() }; (pil, patched_constants) @@ -121,7 +122,7 @@ struct EStarkFilesCommon<'a, F: FieldElement> { pil: PIL, /// If this field is present, it means the constants were patched with /// "main.first_step" column and must be written again to a file. - patched_constants: Option)>>, + constants: Cow<'a, [(String, Vec)]>, output_dir: Option<&'a Path>, proof_type: ProofType, } @@ -154,16 +155,19 @@ impl<'a, F: FieldElement> EStarkFilesCommon<'a, F> { if verification_app_key.is_some() { return Err(Error::NoAggregationAvailable); } + if analyzed.degrees().len() > 1 { + return Err(Error::NoVariableDegreeAvailable); + } // Pre-process the PIL and fixed columns. - let (pil, patched_constants) = first_step_fixup(analyzed, fixed); + let (pil, constants) = first_step_fixup(analyzed, fixed); let proof_type: ProofType = ProofType::from(options); Ok(EStarkFilesCommon { degree: analyzed.max_degree(), pil, - patched_constants, + constants, output_dir, proof_type, }) @@ -171,6 +175,7 @@ impl<'a, F: FieldElement> EStarkFilesCommon<'a, F> { } struct ProverInputFilePaths { + commits: PathBuf, constants: PathBuf, stark_struct: PathBuf, contraints: PathBuf, @@ -178,22 +183,23 @@ struct ProverInputFilePaths { impl<'a, F: FieldElement> EStarkFilesCommon<'a, F> { /// Write the files in the EStark Polygon format. - fn write_files(&self, output_dir: &Path) -> Result { + fn write_files( + &self, + witness: &[(String, Vec)], + output_dir: &Path, + ) -> Result { let paths = ProverInputFilePaths { + commits: output_dir.join("commits_estark.bin"), constants: output_dir.join("constants_estark.bin"), stark_struct: output_dir.join("starkstruct.json"), contraints: output_dir.join("constraints.json"), }; - // If they were patched, write them. Otherwise, just hardlink. - if let Some(patched_constants) = &self.patched_constants { - log::info!("Writing {}.", paths.constants.to_string_lossy()); - write_polys_file(&paths.constants, patched_constants)?; - } else { - log::info!("Hardlinking constants.bin to constants_estark.bin."); - let _ = remove_file(&paths.constants); - hard_link(output_dir.join("constants.bin"), &paths.constants)?; - } + log::info!("Writing {}.", paths.constants.to_string_lossy()); + bin_exporter::write_polys_file(&paths.constants, &self.constants)?; + + log::info!("Writing {}.", paths.commits.to_string_lossy()); + bin_exporter::write_polys_file(&paths.commits, witness)?; // Write the stark struct JSON. write_json_file( @@ -222,6 +228,10 @@ impl BackendFactory for DumpFactory { verification_app_key: Option<&mut dyn std::io::Read>, options: BackendOptions, ) -> Result + 'a>, Error> { + if analyzed.degrees().len() > 1 { + return Err(Error::NoVariableDegreeAvailable); + } + Ok(Box::new(DumpBackend(EStarkFilesCommon::create( analyzed, fixed, @@ -240,7 +250,7 @@ struct DumpBackend<'a, F: FieldElement>(EStarkFilesCommon<'a, F>); impl<'a, F: FieldElement> Backend<'a, F> for DumpBackend<'a, F> { fn prove( &self, - _witness: &[(String, Vec)], + witness: &[(String, Vec)], prev_proof: Option, // TODO: Implement challenges _witgen_callback: WitgenCallback, @@ -254,8 +264,19 @@ impl<'a, F: FieldElement> Backend<'a, F> for DumpBackend<'a, F> { .output_dir .ok_or(Error::BackendError("output_dir is None".to_owned()))?; - self.0.write_files(output_dir)?; + self.0.write_files(witness, output_dir)?; Ok(Vec::new()) } } + +fn buffered_write_file( + path: &Path, + do_write: impl FnOnce(&mut BufWriter) -> R, +) -> Result { + let mut writer = BufWriter::new(File::create(path)?); + let result = do_write(&mut writer); + writer.flush()?; + + Ok(result) +} diff --git a/backend/src/estark/polygon_wrapper.rs b/backend/src/estark/polygon_wrapper.rs index 2068dfe16..078fe9ebf 100644 --- a/backend/src/estark/polygon_wrapper.rs +++ b/backend/src/estark/polygon_wrapper.rs @@ -21,6 +21,9 @@ impl BackendFactory for Factory { verification_app_key: Option<&mut dyn std::io::Read>, options: BackendOptions, ) -> Result + 'a>, Error> { + if analyzed.degrees().len() > 1 { + return Err(Error::NoVariableDegreeAvailable); + } Ok(Box::new(PolygonBackend(EStarkFilesCommon::create( analyzed, fixed, @@ -60,7 +63,7 @@ impl<'a, F: FieldElement> Backend<'a, F> for PolygonBackend<'a, F> { let input_paths = self.0.write_files(output_dir)?; - let commits_path = output_dir.join("commits.bin"); + let commits_path = output_dir.join("commits_estark.bin"); // Generate the proof. let proof_paths = pil_stark_prover::generate_proof( diff --git a/backend/src/estark/starky_wrapper.rs b/backend/src/estark/starky_wrapper.rs index 4a9d85826..39cb36e2b 100644 --- a/backend/src/estark/starky_wrapper.rs +++ b/backend/src/estark/starky_wrapper.rs @@ -44,14 +44,15 @@ impl BackendFactory for Factory { if verification_app_key.is_some() { return Err(Error::NoAggregationAvailable); } + if pil.degrees().len() > 1 { + return Err(Error::NoVariableDegreeAvailable); + } let proof_type: ProofType = ProofType::from(options); let params = create_stark_struct(pil.max_degree(), proof_type.hash_type()); - let (pil_json, patched_fixed) = first_step_fixup(pil, fixed); - - let fixed = patched_fixed.map_or_else(|| Cow::Borrowed(fixed), Cow::Owned); + let (pil_json, fixed) = first_step_fixup(pil, fixed); let const_pols = to_starky_pols_array(&fixed, &pil_json, PolKind::Constant); diff --git a/backend/src/halo2/mod.rs b/backend/src/halo2/mod.rs index ac7dcc955..0f6e93c7d 100644 --- a/backend/src/halo2/mod.rs +++ b/backend/src/halo2/mod.rs @@ -81,6 +81,9 @@ impl BackendFactory for Halo2ProverFactory { verification_app_key: Option<&mut dyn io::Read>, options: BackendOptions, ) -> Result + 'a>, Error> { + if pil.degrees().len() > 1 { + return Err(Error::NoVariableDegreeAvailable); + } let proof_type = ProofType::from(options); let mut halo2 = Box::new(Halo2Prover::new(pil, fixed, setup, proof_type)?); if let Some(vk) = verification_key { diff --git a/backend/src/lib.rs b/backend/src/lib.rs index 25083db98..ad5f245a2 100644 --- a/backend/src/lib.rs +++ b/backend/src/lib.rs @@ -72,6 +72,8 @@ pub enum Error { NoEthereumVerifierAvailable, #[error("the backend does not support proof aggregation")] NoAggregationAvailable, + #[error("the backend does not support variable degrees")] + NoVariableDegreeAvailable, #[error("internal backend error")] BackendError(String), } diff --git a/pipeline/src/verify.rs b/pipeline/src/verify.rs index 7fe9698cf..31721a25a 100644 --- a/pipeline/src/verify.rs +++ b/pipeline/src/verify.rs @@ -5,7 +5,7 @@ pub fn verify(temp_dir: &Path) -> Result<(), String> { .expect("Please set the PILCOM environment variable to the path to the pilcom repository."); let constants_file = format!("{}/constants_estark.bin", temp_dir.to_str().unwrap()); - let commits_file = format!("{}/commits.bin", temp_dir.to_str().unwrap()); + let commits_file = format!("{}/commits_estark.bin", temp_dir.to_str().unwrap()); let constraints_file = format!("{}/constraints.json", temp_dir.to_str().unwrap()); let verifier_output = Command::new("node") diff --git a/pipeline/tests/pil.rs b/pipeline/tests/pil.rs index b3b048a57..4b552e213 100644 --- a/pipeline/tests/pil.rs +++ b/pipeline/tests/pil.rs @@ -88,8 +88,8 @@ fn permutation_with_selector() { fn fibonacci() { let f = "pil/fibonacci.pil"; verify_pil(f, Default::default()); - test_halo2(f, Default::default()); - gen_estark_proof(f, Default::default()); + // test_halo2(f, Default::default()); + // gen_estark_proof(f, Default::default()); } #[test] From adadec7d35c84d3588ea8eaa27af5d0b65ee6eb3 Mon Sep 17 00:00:00 2001 From: schaeff Date: Wed, 26 Jun 2024 12:31:32 +0200 Subject: [PATCH 09/39] use cbor --- number/Cargo.toml | 2 +- number/src/serialize.rs | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/number/Cargo.toml b/number/Cargo.toml index 7c6bc97b0..7466a194a 100644 --- a/number/Cargo.toml +++ b/number/Cargo.toml @@ -20,7 +20,7 @@ serde = { version = "1.0", default-features = false, features = ["alloc", "deriv serde_with = "3.6.1" schemars = { version = "0.8.16", features = ["preserve_order"]} ibig = { version = "0.3.6", features = ["serde"]} -serde_json = "1.0.117" +serde_cbor = "0.11.2" [dev-dependencies] test-log = "0.2.12" diff --git a/number/src/serialize.rs b/number/src/serialize.rs index 5a36f3d19..109575917 100644 --- a/number/src/serialize.rs +++ b/number/src/serialize.rs @@ -105,7 +105,7 @@ pub fn buffered_write_file( pub fn write_polys_file( path: &Path, polys: &[(String, Vec)], -) -> Result<(), io::Error> { +) -> Result<(), serde_cbor::Error> { buffered_write_file(path, |writer| write_polys_stream(writer, polys))??; Ok(()) @@ -114,15 +114,14 @@ pub fn write_polys_file( fn write_polys_stream( file: &mut impl Write, polys: &[(String, Vec)], -) -> Result<(), io::Error> { - Ok(serde_json::to_writer(file, polys)?) +) -> Result<(), serde_cbor::Error> { + Ok(serde_cbor::to_writer(file, &polys)?) } pub fn read_polys_file( file: &mut impl Read, - // columns: &[String], ) -> Vec<(String, Vec)> { - serde_json::from_reader(file).unwrap() + serde_cbor::from_reader(file).unwrap() } // Serde wrappers for serialize/deserialize From 896a1dd97490c39c6f97f33da6985272c0e2a6f2 Mon Sep 17 00:00:00 2001 From: schaeff Date: Wed, 26 Jun 2024 12:42:54 +0200 Subject: [PATCH 10/39] clippy --- number/src/serialize.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/number/src/serialize.rs b/number/src/serialize.rs index 109575917..d19a61fbc 100644 --- a/number/src/serialize.rs +++ b/number/src/serialize.rs @@ -115,7 +115,7 @@ fn write_polys_stream( file: &mut impl Write, polys: &[(String, Vec)], ) -> Result<(), serde_cbor::Error> { - Ok(serde_cbor::to_writer(file, &polys)?) + serde_cbor::to_writer(file, &polys) } pub fn read_polys_file( From 9a23e3102cdcc6dfc020d757ec6b00870d69482f Mon Sep 17 00:00:00 2001 From: schaeff Date: Wed, 26 Jun 2024 12:59:47 +0200 Subject: [PATCH 11/39] commit file.. fmt lint --- backend/src/estark/bin_exporter.rs | 52 ++++++++++++++++++++++++++++++ number/src/serialize.rs | 4 +-- 2 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 backend/src/estark/bin_exporter.rs diff --git a/backend/src/estark/bin_exporter.rs b/backend/src/estark/bin_exporter.rs new file mode 100644 index 000000000..6b9029ebe --- /dev/null +++ b/backend/src/estark/bin_exporter.rs @@ -0,0 +1,52 @@ +/// The custom serialization format expected by estark +use std::{ + io::Write, + io::{self}, + path::Path, +}; + +use powdr_number::FieldElement; + +use super::buffered_write_file; + +pub fn write_polys_file( + path: &Path, + polys: &[(String, Vec)], +) -> Result<(), io::Error> { + buffered_write_file(path, |writer| write_polys_stream(writer, polys))??; + + Ok(()) +} + +fn ceil_div(num: usize, div: usize) -> usize { + (num + div - 1) / div +} + +fn write_polys_stream( + file: &mut impl Write, + polys: &[(String, Vec)], +) -> Result<(), io::Error> { + let ceil_div = ceil_div(T::BITS as usize, 64); + let width = ceil_div * 8; + + if polys.is_empty() { + return Ok(()); + } + + // TODO maybe the witness should have a proper type that + // explicitly has a degree or length? + let degree = polys[0].1.len(); + for (_, values) in polys { + assert_eq!(values.len(), degree); + } + + for i in 0..degree { + for (_name, constant) in polys { + let bytes = constant[i].to_bytes_le(); + assert_eq!(bytes.len(), width); + file.write_all(&bytes)?; + } + } + + Ok(()) +} diff --git a/number/src/serialize.rs b/number/src/serialize.rs index d19a61fbc..631298c91 100644 --- a/number/src/serialize.rs +++ b/number/src/serialize.rs @@ -118,9 +118,7 @@ fn write_polys_stream( serde_cbor::to_writer(file, &polys) } -pub fn read_polys_file( - file: &mut impl Read, -) -> Vec<(String, Vec)> { +pub fn read_polys_file(file: &mut impl Read) -> Vec<(String, Vec)> { serde_cbor::from_reader(file).unwrap() } From 9420d0521e3a9fdcb572f640007324b7ea239a80 Mon Sep 17 00:00:00 2001 From: schaeff Date: Wed, 26 Jun 2024 13:20:30 +0200 Subject: [PATCH 12/39] clean --- backend/src/estark/mod.rs | 4 ---- backend/src/estark/polygon_wrapper.rs | 3 --- executor/src/witgen/generator.rs | 4 +++- pipeline/tests/pil.rs | 4 ++-- 4 files changed, 5 insertions(+), 10 deletions(-) diff --git a/backend/src/estark/mod.rs b/backend/src/estark/mod.rs index 594962df1..b965517b6 100644 --- a/backend/src/estark/mod.rs +++ b/backend/src/estark/mod.rs @@ -228,10 +228,6 @@ impl BackendFactory for DumpFactory { verification_app_key: Option<&mut dyn std::io::Read>, options: BackendOptions, ) -> Result + 'a>, Error> { - if analyzed.degrees().len() > 1 { - return Err(Error::NoVariableDegreeAvailable); - } - Ok(Box::new(DumpBackend(EStarkFilesCommon::create( analyzed, fixed, diff --git a/backend/src/estark/polygon_wrapper.rs b/backend/src/estark/polygon_wrapper.rs index 078fe9ebf..e7ff039e3 100644 --- a/backend/src/estark/polygon_wrapper.rs +++ b/backend/src/estark/polygon_wrapper.rs @@ -21,9 +21,6 @@ impl BackendFactory for Factory { verification_app_key: Option<&mut dyn std::io::Read>, options: BackendOptions, ) -> Result + 'a>, Error> { - if analyzed.degrees().len() > 1 { - return Err(Error::NoVariableDegreeAvailable); - } Ok(Box::new(PolygonBackend(EStarkFilesCommon::create( analyzed, fixed, diff --git a/executor/src/witgen/generator.rs b/executor/src/witgen/generator.rs index a40419ddf..0f9500569 100644 --- a/executor/src/witgen/generator.rs +++ b/executor/src/witgen/generator.rs @@ -31,6 +31,7 @@ pub struct Generator<'a, T: FieldElement> { data: FinalizableData<'a, T>, latch: Option>, name: String, + degree: DegreeType, } impl<'a, T: FieldElement> Machine<'a, T> for Generator<'a, T> { @@ -39,7 +40,7 @@ impl<'a, T: FieldElement> Machine<'a, T> for Generator<'a, T> { } fn degree(&self) -> DegreeType { - self.fixed_data.analyzed.max_degree() + self.degree } fn name(&self) -> &str { @@ -123,6 +124,7 @@ impl<'a, T: FieldElement> Generator<'a, T> { let data = FinalizableData::new(&witnesses); Self { + degree: fixed_data.analyzed.max_degree(), connecting_identities: connecting_identities.clone(), name, fixed_data, diff --git a/pipeline/tests/pil.rs b/pipeline/tests/pil.rs index 4b552e213..b3b048a57 100644 --- a/pipeline/tests/pil.rs +++ b/pipeline/tests/pil.rs @@ -88,8 +88,8 @@ fn permutation_with_selector() { fn fibonacci() { let f = "pil/fibonacci.pil"; verify_pil(f, Default::default()); - // test_halo2(f, Default::default()); - // gen_estark_proof(f, Default::default()); + test_halo2(f, Default::default()); + gen_estark_proof(f, Default::default()); } #[test] From 0f6ca053d3751db976f814b682f07e15259287b1 Mon Sep 17 00:00:00 2001 From: schaeff Date: Wed, 26 Jun 2024 13:40:44 +0200 Subject: [PATCH 13/39] fix polygon --- backend/src/estark/polygon_wrapper.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/backend/src/estark/polygon_wrapper.rs b/backend/src/estark/polygon_wrapper.rs index e7ff039e3..4ed7476e4 100644 --- a/backend/src/estark/polygon_wrapper.rs +++ b/backend/src/estark/polygon_wrapper.rs @@ -41,7 +41,7 @@ impl<'a, F: FieldElement> Backend<'a, F> for PolygonBackend<'a, F> { fn prove( &self, // Witness is taken from file written by the pipeline. - _witness: &[(String, Vec)], + witness: &[(String, Vec)], prev_proof: Option, // TODO: Implement challenges _witgen_callback: WitgenCallback, @@ -58,16 +58,14 @@ impl<'a, F: FieldElement> Backend<'a, F> for PolygonBackend<'a, F> { tmp_dir.as_path() }; - let input_paths = self.0.write_files(output_dir)?; - - let commits_path = output_dir.join("commits_estark.bin"); + let input_paths = self.0.write_files(witness, output_dir)?; // Generate the proof. let proof_paths = pil_stark_prover::generate_proof( &input_paths.contraints, &input_paths.stark_struct, &input_paths.constants, - &commits_path, + &input_paths.commits, output_dir, ) .map_err(|e| Error::BackendError(e.to_string()))?; From c3c17008e02498dd295125907c102e1c1a62dce3 Mon Sep 17 00:00:00 2001 From: schaeff Date: Wed, 26 Jun 2024 13:58:29 +0200 Subject: [PATCH 14/39] expect proving to fail --- pipeline/tests/pil.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/pipeline/tests/pil.rs b/pipeline/tests/pil.rs index b3b048a57..f812f64eb 100644 --- a/pipeline/tests/pil.rs +++ b/pipeline/tests/pil.rs @@ -300,6 +300,7 @@ fn naive_byte_decomposition_gl() { } #[test] +#[should_panic = "NoVariableDegreeAvailable"] fn different_degrees() { let f = "pil/two_proofs.pil"; verify_pil(f, Default::default()); From 22fe520bd64d7a96e0bb6054078e3370bbecec7e Mon Sep 17 00:00:00 2001 From: schaeff Date: Wed, 26 Jun 2024 14:31:40 +0200 Subject: [PATCH 15/39] fix test --- pipeline/src/pipeline.rs | 14 +++++--------- pipeline/src/util.rs | 9 ++++----- pipeline/tests/asm.rs | 7 ++++--- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/pipeline/src/pipeline.rs b/pipeline/src/pipeline.rs index 0b5385764..5026fb878 100644 --- a/pipeline/src/pipeline.rs +++ b/pipeline/src/pipeline.rs @@ -30,7 +30,7 @@ use powdr_schemas::SerializedAnalyzed; use crate::{ handle_simple_queries_callback, inputs_to_query_callback, serde_data_to_query_callback, - util::{try_read_poly_set, FixedPolySet, WitnessPolySet}, + util::{read_poly_set, FixedPolySet, WitnessPolySet}, }; type Columns = Vec<(String, Vec)>; @@ -382,10 +382,8 @@ impl Pipeline { } /// Reads previously generated fixed columns from the provided directory. - pub fn read_constants(mut self, directory: &Path) -> Self { - let pil = self.compute_optimized_pil().unwrap(); - - let fixed = try_read_poly_set::(&pil, directory).unwrap_or_default(); + pub fn read_constants(self, directory: &Path) -> Self { + let fixed = read_poly_set::(directory); Pipeline { artifact: Artifacts { @@ -397,10 +395,8 @@ impl Pipeline { } /// Reads a previously generated witness from the provided directory. - pub fn read_witness(mut self, directory: &Path) -> Self { - let pil = self.compute_optimized_pil().unwrap(); - - let witness = try_read_poly_set::(&pil, directory).unwrap_or_default(); + pub fn read_witness(self, directory: &Path) -> Self { + let witness = read_poly_set::(directory); Pipeline { artifact: Artifacts { diff --git a/pipeline/src/util.rs b/pipeline/src/util.rs index 6c56295af..e5d9e8972 100644 --- a/pipeline/src/util.rs +++ b/pipeline/src/util.rs @@ -32,12 +32,11 @@ impl PolySet for WitnessPolySet { } #[allow(clippy::type_complexity)] -pub fn try_read_poly_set( - _pil: &Analyzed, +pub fn read_poly_set( dir: &Path, -) -> Option)>> { +) -> Vec<(String, Vec)> { let path = dir.join(P::FILE_NAME); - Some(read_polys_file(&mut BufReader::new( + read_polys_file(&mut BufReader::new( File::open(path).unwrap(), - ))) + )) } diff --git a/pipeline/tests/asm.rs b/pipeline/tests/asm.rs index 23de14c27..213853bfa 100644 --- a/pipeline/tests/asm.rs +++ b/pipeline/tests/asm.rs @@ -2,7 +2,7 @@ use powdr_backend::BackendType; use powdr_number::{Bn254Field, FieldElement, GoldilocksField}; use powdr_pipeline::{ test_util::{gen_estark_proof, resolve_test_file, test_halo2, verify_test_file}, - util::{try_read_poly_set, FixedPolySet, WitnessPolySet}, + util::{read_poly_set, FixedPolySet, WitnessPolySet}, Pipeline, }; use test_log::test; @@ -377,12 +377,13 @@ fn read_poly_files() { pipeline.compute_proof().unwrap(); // check fixed cols (may have no fixed cols) - if let Some(fixed) = try_read_poly_set::(&pil, tmp_dir.as_path()) { + let fixed = read_poly_set::(tmp_dir.as_path()); + if !fixed.is_empty() { assert_eq!(pil.max_degree(), fixed[0].1.len() as u64); } // check witness cols (examples assumed to have at least one witness col) - let witness = try_read_poly_set::(&pil, tmp_dir.as_path()).unwrap(); + let witness = read_poly_set::(tmp_dir.as_path()); assert_eq!(pil.max_degree(), witness[0].1.len() as u64); } } From 625bf41942208d51324d10bb536fcb7356426071 Mon Sep 17 00:00:00 2001 From: schaeff Date: Wed, 26 Jun 2024 17:34:27 +0200 Subject: [PATCH 16/39] fmt --- pipeline/src/util.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pipeline/src/util.rs b/pipeline/src/util.rs index e5d9e8972..8c1ddf620 100644 --- a/pipeline/src/util.rs +++ b/pipeline/src/util.rs @@ -32,11 +32,7 @@ impl PolySet for WitnessPolySet { } #[allow(clippy::type_complexity)] -pub fn read_poly_set( - dir: &Path, -) -> Vec<(String, Vec)> { +pub fn read_poly_set(dir: &Path) -> Vec<(String, Vec)> { let path = dir.join(P::FILE_NAME); - read_polys_file(&mut BufReader::new( - File::open(path).unwrap(), - )) + read_polys_file(&mut BufReader::new(File::open(path).unwrap())) } From e7091f8f6de831af6e55466acfee6ebd2d0762dd Mon Sep 17 00:00:00 2001 From: schaeff Date: Wed, 26 Jun 2024 18:17:32 +0200 Subject: [PATCH 17/39] commit two_proof.pil --- test_data/pil/two_proofs.pil | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 test_data/pil/two_proofs.pil diff --git a/test_data/pil/two_proofs.pil b/test_data/pil/two_proofs.pil new file mode 100644 index 000000000..d2462c6e5 --- /dev/null +++ b/test_data/pil/two_proofs.pil @@ -0,0 +1,24 @@ +constant %N = 4; + +// a block machine which performs addition +namespace Add(%N); + let af = |i| (2 * i + 13) & 0xffffffff; + let bf = |i| ((2 * i + 19) * 17) & 0xffffffff; + col fixed a(i) { af(i) }; + col fixed b(i) { bf(i) }; + col fixed c(i) { af(i) + bf(i) }; + +// a machine which calls `Add` every other row on made up inputs +namespace Main(2 * %N); + col fixed a(i) { (i + 13) & 0xffffffff }; + col fixed b(i) { ((i + 19) * 17) & 0xffffffff }; + col witness c; + + // only make a call every other row, otherwise set `c` to 0 + // we do this to prevent running out of blocks in `Add` + col fixed CALL = [1, 0]*; + (1 - CALL) * c = 0; + + CALL {a, b, c} in {Add.a, Add.b, Add.c}; + + From d3b8414e6d76576269223a9cf9f115c54a0573a6 Mon Sep 17 00:00:00 2001 From: schaeff Date: Thu, 27 Jun 2024 00:49:24 +0200 Subject: [PATCH 18/39] fix vec_median test --- pil-analyzer/src/condenser.rs | 3 +-- pipeline/tests/pil.rs | 2 +- test_data/pil/{two_proofs.pil => different_degrees.pil} | 0 3 files changed, 2 insertions(+), 3 deletions(-) rename test_data/pil/{two_proofs.pil => different_degrees.pil} (100%) diff --git a/pil-analyzer/src/condenser.rs b/pil-analyzer/src/condenser.rs index f5db9bb8b..c7189034e 100644 --- a/pil-analyzer/src/condenser.rs +++ b/pil-analyzer/src/condenser.rs @@ -368,8 +368,7 @@ impl<'a, T: FieldElement> SymbolLookup<'a, T> for Condenser<'a, T> { stage: None, kind: SymbolKind::Poly(PolynomialType::Committed), length: None, - // TODO: what is the actual degree? - degree: None, + degree: Some(self.degree.unwrap()), }; self.next_witness_id += 1; self.all_new_witness_names.insert(name.clone()); diff --git a/pipeline/tests/pil.rs b/pipeline/tests/pil.rs index f812f64eb..63b4c46ae 100644 --- a/pipeline/tests/pil.rs +++ b/pipeline/tests/pil.rs @@ -302,7 +302,7 @@ fn naive_byte_decomposition_gl() { #[test] #[should_panic = "NoVariableDegreeAvailable"] fn different_degrees() { - let f = "pil/two_proofs.pil"; + let f = "pil/different_degrees.pil"; verify_pil(f, Default::default()); } diff --git a/test_data/pil/two_proofs.pil b/test_data/pil/different_degrees.pil similarity index 100% rename from test_data/pil/two_proofs.pil rename to test_data/pil/different_degrees.pil From 7ba9587799269c5bdd0c374866bcfac28af249be Mon Sep 17 00:00:00 2001 From: schaeff Date: Thu, 27 Jun 2024 01:28:05 +0200 Subject: [PATCH 19/39] revert pil file --- test_data/pil/single_line_blocks.pil | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_data/pil/single_line_blocks.pil b/test_data/pil/single_line_blocks.pil index 9f1ca0833..279b0b4b8 100644 --- a/test_data/pil/single_line_blocks.pil +++ b/test_data/pil/single_line_blocks.pil @@ -8,7 +8,7 @@ namespace Add(%N); A + B = C; // a machine which calls `Add` every other row on made up inputs -namespace Main(2 * %N); +namespace Main(%N); col fixed a(i) { (i + 13) & 0xffffffff }; col fixed b(i) { ((i + 19) * 17) & 0xffffffff }; col witness c; From 62a0af7c10f66692a9ac40911cb13a01e787a31c Mon Sep 17 00:00:00 2001 From: schaeff Date: Thu, 27 Jun 2024 13:58:44 +0200 Subject: [PATCH 20/39] fix btree test, pass correct degree to main machine generator --- executor/src/witgen/generator.rs | 3 +- .../src/witgen/machines/machine_extractor.rs | 30 ++++++++++++---- executor/src/witgen/mod.rs | 36 ++++++++++--------- pipeline/src/test_util.rs | 12 +++++++ pipeline/tests/powdr_std.rs | 5 ++- 5 files changed, 59 insertions(+), 27 deletions(-) diff --git a/executor/src/witgen/generator.rs b/executor/src/witgen/generator.rs index 0f9500569..4892836ee 100644 --- a/executor/src/witgen/generator.rs +++ b/executor/src/witgen/generator.rs @@ -118,13 +118,14 @@ impl<'a, T: FieldElement> Generator<'a, T> { identities: Vec<&'a Identity>>, witnesses: HashSet, latch: Option>, + degree: DegreeType, ) -> Self { let witnesses = witnesses.into_iter().map(Into::into).collect(); let data = FinalizableData::new(&witnesses); Self { - degree: fixed_data.analyzed.max_degree(), + degree, connecting_identities: connecting_identities.clone(), name, fixed_data, diff --git a/executor/src/witgen/machines/machine_extractor.rs b/executor/src/witgen/machines/machine_extractor.rs index 7c6f16ee1..0d35fc804 100644 --- a/executor/src/witgen/machines/machine_extractor.rs +++ b/executor/src/witgen/machines/machine_extractor.rs @@ -15,18 +15,25 @@ use powdr_ast::analyzed::{ }; use powdr_ast::parsed::visitor::ExpressionVisitable; use powdr_ast::parsed::SelectedExpressions; +use powdr_number::DegreeType; use powdr_number::FieldElement; pub struct ExtractionOutput<'a, T: FieldElement> { pub fixed_lookup: FixedLookup, pub machines: Vec>, - pub base_identities: Vec<&'a Identity>>, - pub base_witnesses: HashSet, + /// The leftover witnesses and identities, checked to having the same degree + pub base: Option> +} + +pub struct Base<'a, T> { + pub identities: Vec<&'a Identity>>, + pub witnesses: HashSet, + pub degree: DegreeType, } fn to_raw_and_degree( machine_witnesses: HashSet, -) -> (HashSet, u64) { +) -> (HashSet, Option) { let mut res = HashSet::default(); let mut degree = None; for id in machine_witnesses { @@ -40,7 +47,7 @@ fn to_raw_and_degree( } } } - (res, degree.unwrap()) + (res, degree) } /// Finds machines in the witness columns and identities @@ -143,6 +150,9 @@ pub fn split_out_machines<'a, T: FieldElement>( // Internal processing happens over `RawPolyID` let (machine_witnesses, degree) = to_raw_and_degree(machine_witnesses); + // We must have found a degree + let degree = degree.unwrap(); + if let Some(machine) = SortedWitnesses::try_new( name_with_type("SortedWitness"), fixed, @@ -209,17 +219,23 @@ pub fn split_out_machines<'a, T: FieldElement>( machine_identities, machine_witnesses, Some(latch), + degree, ))); } } - let (remaining_witnesses, _) = to_raw_and_degree(remaining_witnesses); + let (remaining_witnesses, base_degree) = to_raw_and_degree(remaining_witnesses); + + let base = base_degree.map(|degree| Base { + identities: base_identities, + witnesses: remaining_witnesses, + degree + }); ExtractionOutput { fixed_lookup, machines, - base_identities, - base_witnesses: remaining_witnesses, + base, } } diff --git a/executor/src/witgen/mod.rs b/executor/src/witgen/mod.rs index 5fe77f30e..c8b8cfb3c 100644 --- a/executor/src/witgen/mod.rs +++ b/executor/src/witgen/mod.rs @@ -187,32 +187,36 @@ impl<'a, 'b, T: FieldElement> WitnessGenerator<'a, 'b, T> { let ExtractionOutput { mut fixed_lookup, mut machines, - base_identities, - base_witnesses, + base, } = machines::machine_extractor::split_out_machines(&fixed, retained_identities); + let mut query_callback = self.query_callback; let mut mutable_state = MutableState { fixed_lookup: &mut fixed_lookup, machines: Machines::from(machines.iter_mut()), query_callback: &mut query_callback, }; - let mut generator = Generator::new( - "Main Machine".to_string(), - &fixed, - &BTreeMap::new(), // No connecting identities - base_identities, - base_witnesses, - // We could set the latch of the main VM here, but then we would have to detect it. - // Instead, the main VM will be computed in one block, directly continuing into the - // infinite loop after the first return. - None, - ); - generator.run(&mut mutable_state); + let main_columns = base.map(|base| { + let mut generator = Generator::new( + "Main Machine".to_string(), + &fixed, + &BTreeMap::new(), // No connecting identities + base.identities, + base.witnesses, + // We could set the latch of the main VM here, but then we would have to detect it. + // Instead, the main VM will be computed in one block, directly continuing into the + // infinite loop after the first return. + None, + base.degree, + ); + + generator.run(&mut mutable_state); + generator + .take_witness_col_values(mutable_state.fixed_lookup, mutable_state.query_callback) + }).unwrap_or_default(); // Get columns from machines - let main_columns = generator - .take_witness_col_values(mutable_state.fixed_lookup, mutable_state.query_callback); let mut columns = mutable_state .machines .iter_mut() diff --git a/pipeline/src/test_util.rs b/pipeline/src/test_util.rs index 1f7b9f502..a90082e2d 100644 --- a/pipeline/src/test_util.rs +++ b/pipeline/src/test_util.rs @@ -16,6 +16,18 @@ pub fn resolve_test_file(file_name: &str) -> PathBuf { )) } +pub fn execute_test_file( + file_name: &str, + inputs: Vec, + external_witness_values: Vec<(String, Vec)>, +) -> Result)>>, Vec> { + Pipeline::default() + .from_file(resolve_test_file(file_name)) + .with_prover_inputs(inputs) + .add_external_witness_values(external_witness_values) + .compute_witness() +} + pub fn verify_test_file( file_name: &str, inputs: Vec, diff --git a/pipeline/tests/powdr_std.rs b/pipeline/tests/powdr_std.rs index 0c165d1a9..26f5bb37f 100644 --- a/pipeline/tests/powdr_std.rs +++ b/pipeline/tests/powdr_std.rs @@ -5,8 +5,7 @@ use powdr_number::{BigInt, Bn254Field, GoldilocksField}; use powdr_pil_analyzer::evaluator::Value; use powdr_pipeline::{ test_util::{ - evaluate_function, evaluate_integer_function, gen_estark_proof, gen_halo2_proof, - resolve_test_file, std_analyzed, test_halo2, verify_test_file, + evaluate_function, evaluate_integer_function, execute_test_file, gen_estark_proof, gen_halo2_proof, resolve_test_file, std_analyzed, test_halo2, verify_test_file }, Pipeline, }; @@ -313,5 +312,5 @@ fn sort() { #[test] fn btree() { let f = "std/btree_test.asm"; - verify_test_file(f, Default::default(), vec![]).unwrap(); + execute_test_file(f, Default::default(), vec![]).unwrap(); } From 5bc0ec0b9c8b655503897349625fe9e44e12424f Mon Sep 17 00:00:00 2001 From: schaeff Date: Thu, 27 Jun 2024 16:09:42 +0200 Subject: [PATCH 21/39] simplify, thanks chris --- ast/src/analyzed/display.rs | 15 ++- ast/src/analyzed/mod.rs | 115 +++++------------- .../json_exporter/expression_counter.rs | 4 +- backend/src/estark/json_exporter/mod.rs | 20 +-- backend/src/estark/mod.rs | 4 +- backend/src/estark/starky_wrapper.rs | 2 +- backend/src/halo2/circuit_builder.rs | 4 +- backend/src/halo2/mock_prover.rs | 2 +- backend/src/halo2/prover.rs | 8 +- executor/src/constant_evaluator/mod.rs | 36 +++--- executor/src/witgen/block_processor.rs | 6 +- .../src/witgen/data_structures/column_map.rs | 10 +- .../data_structures/finalizable_data.rs | 2 +- executor/src/witgen/fixed_evaluator.rs | 2 +- executor/src/witgen/generator.rs | 10 +- executor/src/witgen/global_constraints.rs | 26 ++-- executor/src/witgen/machines/block_machine.rs | 11 +- .../machines/double_sorted_witness_machine.rs | 9 +- .../witgen/machines/fixed_lookup_machine.rs | 16 +-- .../src/witgen/machines/machine_extractor.rs | 88 +++----------- .../witgen/machines/sorted_witness_machine.rs | 15 +-- .../src/witgen/machines/write_once_memory.rs | 13 +- executor/src/witgen/mod.rs | 90 +++++++++----- executor/src/witgen/processor.rs | 23 ++-- executor/src/witgen/query_processor.rs | 6 +- executor/src/witgen/rows.rs | 12 +- .../src/witgen/symbolic_witness_evaluator.rs | 2 +- executor/src/witgen/vm_processor.rs | 6 +- pipeline/src/test_util.rs | 6 +- pipeline/tests/asm.rs | 4 +- pipeline/tests/powdr_std.rs | 3 +- plonky3/src/circuit_builder.rs | 8 +- plonky3/src/prover/mod.rs | 4 +- riscv/src/continuations.rs | 2 +- 34 files changed, 252 insertions(+), 332 deletions(-) diff --git a/ast/src/analyzed/display.rs b/ast/src/analyzed/display.rs index 6f4c72aa3..324c55caf 100644 --- a/ast/src/analyzed/display.rs +++ b/ast/src/analyzed/display.rs @@ -483,17 +483,26 @@ mod test { let x = AlgebraicExpression::Reference(super::AlgebraicReference { name: "x".into(), - poly_id: super::PolyID::new(0, super::PolynomialType::Committed), + poly_id: super::PolyID { + id: 0, + ptype: super::PolynomialType::Committed, + }, next: false, }); let y = AlgebraicExpression::Reference(super::AlgebraicReference { name: "y".into(), - poly_id: super::PolyID::new(1, super::PolynomialType::Committed), + poly_id: super::PolyID { + id: 1, + ptype: super::PolynomialType::Committed, + }, next: false, }); let z = AlgebraicExpression::Reference(super::AlgebraicReference { name: "z".into(), - poly_id: super::PolyID::new(2, super::PolynomialType::Committed), + poly_id: super::PolyID { + id: 2, + ptype: super::PolynomialType::Committed, + }, next: false, }); diff --git a/ast/src/analyzed/mod.rs b/ast/src/analyzed/mod.rs index 87e95c45e..f77afe8c9 100644 --- a/ast/src/analyzed/mod.rs +++ b/ast/src/analyzed/mod.rs @@ -43,16 +43,19 @@ pub struct Analyzed { } impl Analyzed { - /// Returns the max of all degrees in this [`Analyzed`]. + /// Returns the common degree in this [`Analyzed`]. /// /// # Panics /// - /// Panics if there are no symbols - pub fn max_degree(&self) -> DegreeType { + /// Panics if there is no common degree or if there are no symbols + pub fn degree(&self) -> DegreeType { self.definitions .values() .filter_map(|(symbol, _)| symbol.degree) - .max() + .reduce(|acc, degree| { + assert_eq!(acc, degree); + acc + }) .unwrap() } @@ -229,8 +232,14 @@ impl Analyzed { let length = symbol.length.unwrap_or(1); // Empty arrays still need ID replacement for i in 0..max(length, 1) { - let old_poly_id = PolyID::from(symbol).with_id(symbol.id + i); - let new_poly_id = PolyID::from(symbol).with_id(new_id + i); + let old_poly_id = PolyID { + id: symbol.id + i, + ..PolyID::from(symbol) + }; + let new_poly_id = PolyID { + id: new_id + i, + ..PolyID::from(symbol) + }; replacements.insert(old_poly_id, new_poly_id); } new_id + length @@ -250,14 +259,14 @@ impl Analyzed { self.definitions.values_mut().for_each(|(poly, _def)| { if matches!(poly.kind, SymbolKind::Poly(_)) { let poly_id = PolyID::from(poly as &Symbol); - poly.id = replacements[&poly_id].id(); + poly.id = replacements[&poly_id].id; } }); self.intermediate_columns .values_mut() .for_each(|(poly, _def)| { let poly_id = PolyID::from(poly as &Symbol); - poly.id = replacements[&poly_id].id(); + poly.id = replacements[&poly_id].id; }); let visitor = &mut |expr: &mut Expression| { if let Expression::Reference(_, Reference::Poly(poly)) = expr { @@ -350,7 +359,7 @@ fn substitute_intermediate( .scan(HashMap::default(), |cache, mut identity| { identity.post_visit_expressions_mut(&mut |e| { if let AlgebraicExpression::Reference(poly) = e { - match poly.poly_id.ptype() { + match poly.poly_id.ptype { PolynomialType::Committed => {} PolynomialType::Constant => {} PolynomialType::Intermediate => { @@ -378,10 +387,7 @@ fn inlined_expression_from_intermediate_poly_id( intermediate_polynomials: &HashMap>, cache: &mut HashMap>, ) -> AlgebraicExpression { - assert_eq!( - poly_to_replace.poly_id.ptype(), - PolynomialType::Intermediate - ); + assert_eq!(poly_to_replace.poly_id.ptype, PolynomialType::Intermediate); if let Some(e) = cache.get(&poly_to_replace) { return e.clone(); } @@ -398,7 +404,7 @@ fn inlined_expression_from_intermediate_poly_id( ); } r.next = r.next || poly_to_replace.next; - match r.poly_id.ptype() { + match r.poly_id.ptype { PolynomialType::Committed | PolynomialType::Constant => {} PolynomialType::Intermediate => { *e = inlined_expression_from_intermediate_poly_id( @@ -476,11 +482,8 @@ impl Symbol { ( self.array_element_name(i), PolyID { - raw: RawPolyID { - id: self.id + i, - ptype, - }, - degree: self.degree, + id: self.id + i, + ptype, }, ) }) @@ -780,11 +783,11 @@ pub struct AlgebraicReference { impl AlgebraicReference { #[inline] pub fn is_witness(&self) -> bool { - self.poly_id.ptype() == PolynomialType::Committed + self.poly_id.ptype == PolynomialType::Committed } #[inline] pub fn is_fixed(&self) -> bool { - self.poly_id.ptype() == PolynomialType::Constant + self.poly_id.ptype == PolynomialType::Constant } } @@ -1161,90 +1164,30 @@ pub struct PolynomialReference { pub type_args: Option>, } -#[derive( - Debug, Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Serialize, Deserialize, JsonSchema, Hash, -)] -pub struct PolyID { - pub raw: RawPolyID, - pub degree: Option, -} - -impl PolyID { - pub fn with_id(mut self, id: u64) -> Self { - self.raw = self.raw.with_id(id); - self - } - - pub fn id(&self) -> u64 { - self.raw.id() - } - - pub fn ptype(&self) -> PolynomialType { - self.raw.ptype() - } - - pub fn new(id: u64, ptype: PolynomialType) -> Self { - Self { - raw: RawPolyID { id, ptype }, - degree: None, - } - } -} - #[derive( Debug, Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Serialize, Deserialize, JsonSchema, )] -pub struct RawPolyID { +pub struct PolyID { pub id: u64, pub ptype: PolynomialType, } -impl From<&Symbol> for RawPolyID { +impl From<&Symbol> for PolyID { fn from(symbol: &Symbol) -> Self { let SymbolKind::Poly(ptype) = symbol.kind else { panic!() }; - RawPolyID { + PolyID { id: symbol.id, ptype, } } } -impl Hash for RawPolyID { +impl Hash for PolyID { fn hash(&self, state: &mut H) { // single call to hash is faster - ((self.id() << 2) + self.ptype() as u64).hash(state); - } -} - -impl From for RawPolyID { - fn from(value: PolyID) -> Self { - value.raw - } -} - -impl RawPolyID { - fn with_id(mut self, id: u64) -> Self { - self.id = id; - self - } - - pub fn id(&self) -> u64 { - self.id - } - - pub fn ptype(&self) -> PolynomialType { - self.ptype - } -} - -impl From<&Symbol> for PolyID { - fn from(symbol: &Symbol) -> Self { - PolyID { - raw: symbol.into(), - degree: symbol.degree, - } + ((self.id << 2) + self.ptype as u64).hash(state); } } diff --git a/backend/src/estark/json_exporter/expression_counter.rs b/backend/src/estark/json_exporter/expression_counter.rs index 66a0b3645..d3cf84628 100644 --- a/backend/src/estark/json_exporter/expression_counter.rs +++ b/backend/src/estark/json_exporter/expression_counter.rs @@ -20,8 +20,8 @@ pub fn compute_intermediate_expression_ids(analyzed: &Analyzed) -> HashMap poly.expression_count() } else if let Some((poly, _)) = analyzed.intermediate_columns.get(name) { assert!(poly.kind == SymbolKind::Poly(PolynomialType::Intermediate)); - for (index, (_, poly_id)) in poly.array_elements().enumerate() { - ids.insert(poly_id.id(), (expression_counter + index) as u64); + for (index, (_, id)) in poly.array_elements().enumerate() { + ids.insert(id.id, (expression_counter + index) as u64); } poly.expression_count() } else { diff --git a/backend/src/estark/json_exporter/mod.rs b/backend/src/estark/json_exporter/mod.rs index 20f44440b..e70324f3f 100644 --- a/backend/src/estark/json_exporter/mod.rs +++ b/backend/src/estark/json_exporter/mod.rs @@ -5,7 +5,7 @@ use std::{cmp, path::PathBuf}; use powdr_ast::analyzed::{ AlgebraicBinaryOperation, AlgebraicBinaryOperator, AlgebraicExpression as Expression, AlgebraicUnaryOperation, AlgebraicUnaryOperator, Analyzed, IdentityKind, PolyID, - PolynomialType, RawPolyID, StatementIdentifier, SymbolKind, + PolynomialType, StatementIdentifier, SymbolKind, }; use powdr_parser_util::SourceRef; use starky::types::{ @@ -54,11 +54,11 @@ pub fn export(analyzed: &Analyzed) -> PIL { StatementIdentifier::Definition(name) => { if let Some((poly, value)) = analyzed.intermediate_columns.get(name) { assert_eq!(poly.kind, SymbolKind::Poly(PolynomialType::Intermediate)); - for ((_, poly_id), value) in poly.array_elements().zip(value) { + for ((_, id), value) in poly.array_elements().zip(value) { let expression_id = exporter.extract_expression(value, 1); assert_eq!( expression_id, - exporter.intermediate_poly_expression_ids[&poly_id.id()] as usize + exporter.intermediate_poly_expression_ids[&id.id] as usize ); } } @@ -68,7 +68,10 @@ pub fn export(analyzed: &Analyzed) -> PIL { let pub_ref = &pub_def.polynomial; let poly_id = pub_ref.poly_id.unwrap(); let (_, expr) = exporter.polynomial_reference_to_json( - poly_id.with_id(poly_id.id() + pub_def.array_index.unwrap_or_default() as u64), + PolyID { + id: poly_id.id + pub_def.array_index.unwrap_or_default() as u64, + ..poly_id + }, false, ); let id = publics.len(); @@ -229,7 +232,7 @@ impl<'a, T: FieldElement> Exporter<'a, T> { polType: None, type_: symbol_kind_to_json_string(symbol.kind).to_string(), id: id as usize, - polDeg: self.analyzed.max_degree() as usize, + polDeg: self.analyzed.degree() as usize, isArray: symbol.is_array(), elementType: None, len: symbol.length.map(|l| l as usize), @@ -248,7 +251,7 @@ impl<'a, T: FieldElement> Exporter<'a, T> { polType: None, type_: symbol_kind_to_json_string(symbol.kind).to_string(), id: id as usize, - polDeg: self.analyzed.max_degree() as usize, + polDeg: self.analyzed.degree() as usize, isArray: symbol.is_array(), elementType: None, len: symbol.length.map(|l| l as usize), @@ -366,10 +369,7 @@ impl<'a, T: FieldElement> Exporter<'a, T> { fn polynomial_reference_to_json( &self, - PolyID { - raw: RawPolyID { id, ptype }, - .. - }: PolyID, + PolyID { id, ptype }: PolyID, next: bool, ) -> (u32, StarkyExpr) { let id = if ptype == PolynomialType::Intermediate { diff --git a/backend/src/estark/mod.rs b/backend/src/estark/mod.rs index b965517b6..110e2e6bd 100644 --- a/backend/src/estark/mod.rs +++ b/backend/src/estark/mod.rs @@ -79,7 +79,7 @@ fn first_step_fixup<'a, F: FieldElement>( pil: &'a Analyzed, fixed: &'a [(String, Vec)], ) -> (PIL, Constants<'a, F>) { - let degree = pil.max_degree(); + let degree = pil.degree(); let mut pil: PIL = json_exporter::export(pil); @@ -165,7 +165,7 @@ impl<'a, F: FieldElement> EStarkFilesCommon<'a, F> { let proof_type: ProofType = ProofType::from(options); Ok(EStarkFilesCommon { - degree: analyzed.max_degree(), + degree: analyzed.degree(), pil, constants, output_dir, diff --git a/backend/src/estark/starky_wrapper.rs b/backend/src/estark/starky_wrapper.rs index 39cb36e2b..c1e1682e0 100644 --- a/backend/src/estark/starky_wrapper.rs +++ b/backend/src/estark/starky_wrapper.rs @@ -50,7 +50,7 @@ impl BackendFactory for Factory { let proof_type: ProofType = ProofType::from(options); - let params = create_stark_struct(pil.max_degree(), proof_type.hash_type()); + let params = create_stark_struct(pil.degree(), proof_type.hash_type()); let (pil_json, fixed) = first_step_fixup(pil, fixed); diff --git a/backend/src/halo2/circuit_builder.rs b/backend/src/halo2/circuit_builder.rs index 91730e43f..f5023feed 100644 --- a/backend/src/halo2/circuit_builder.rs +++ b/backend/src/halo2/circuit_builder.rs @@ -272,7 +272,7 @@ impl<'a, T: FieldElement, F: PrimeField> Circuit for PowdrCi let first_row = meta.query_advice(config.advice[name], Rotation::cur()); let last_row = meta.query_advice( config.advice[name], - Rotation(analyzed.max_degree().try_into().unwrap()), + Rotation(analyzed.degree().try_into().unwrap()), ); let expr = first_step.clone() * (first_row - last_row); (format!("enforce wrapping ({name})"), expr) @@ -411,7 +411,7 @@ impl<'a, T: FieldElement, F: PrimeField> Circuit for PowdrCi )?; } } - let degree = self.analyzed.max_degree() as usize; + let degree = self.analyzed.degree() as usize; for i in 0..(2 * degree) { let value = F::from((i < degree) as u64); region.assign_fixed( diff --git a/backend/src/halo2/mock_prover.rs b/backend/src/halo2/mock_prover.rs index ed1afe2d2..65e2e206d 100644 --- a/backend/src/halo2/mock_prover.rs +++ b/backend/src/halo2/mock_prover.rs @@ -20,7 +20,7 @@ pub fn mock_prove( // double the row count in order to make space for the cells introduced by the backend // TODO: use a precise count of the extra rows needed to avoid using so many rows - let circuit_row_count_log = usize::BITS - pil.max_degree().leading_zeros(); + let circuit_row_count_log = usize::BITS - pil.degree().leading_zeros(); let expanded_row_count_log = circuit_row_count_log + 1; let circuit = PowdrCircuit::new(pil, constants) diff --git a/backend/src/halo2/prover.rs b/backend/src/halo2/prover.rs index 4c5971fb0..2830c3f62 100644 --- a/backend/src/halo2/prover.rs +++ b/backend/src/halo2/prover.rs @@ -94,10 +94,10 @@ impl<'a, F: FieldElement> Halo2Prover<'a, F> { let mut params = setup .map(|mut setup| ParamsKZG::::read(&mut setup)) .transpose()? - .unwrap_or_else(|| generate_setup(analyzed.max_degree())); + .unwrap_or_else(|| generate_setup(analyzed.degree())); if matches!(proof_type, ProofType::Poseidon | ProofType::SnarkSingle) { - params.downsize(degree_bits(analyzed.max_degree())); + params.downsize(degree_bits(analyzed.degree())); } Ok(Self { @@ -250,7 +250,7 @@ impl<'a, F: FieldElement> Halo2Prover<'a, F> { log::info!("Generating VK for app snark..."); let mut params_app = self.params.clone(); - params_app.downsize(degree_bits(self.analyzed.max_degree())); + params_app.downsize(degree_bits(self.analyzed.degree())); log::info!("Generating circuit for compression snark..."); let protocol_app = compile( @@ -375,7 +375,7 @@ impl<'a, F: FieldElement> Halo2Prover<'a, F> { }; let mut params_app = self.params.clone(); - params_app.downsize(degree_bits(self.analyzed.max_degree())); + params_app.downsize(degree_bits(self.analyzed.degree())); let protocol_app = compile( ¶ms_app, diff --git a/executor/src/constant_evaluator/mod.rs b/executor/src/constant_evaluator/mod.rs index 4a8b16e34..0ca3ded9c 100644 --- a/executor/src/constant_evaluator/mod.rs +++ b/executor/src/constant_evaluator/mod.rs @@ -186,7 +186,7 @@ mod test { col fixed LAST(i) { if i == N - 1 { 1 } else { 0 } }; "#; let analyzed = analyze_string(src); - assert_eq!(analyzed.max_degree(), 8); + assert_eq!(analyzed.degree(), 8); let constants = generate(&analyzed); assert_eq!( constants, @@ -202,7 +202,7 @@ mod test { pol constant EVEN(i) { 2 * (i - 1) + 4 }; "#; let analyzed = analyze_string(src); - assert_eq!(analyzed.max_degree(), 8); + assert_eq!(analyzed.degree(), 8); let constants = generate(&analyzed); assert_eq!( constants, @@ -221,7 +221,7 @@ mod test { pol constant X(i) { i ^ (i + 17) | 3 }; "#; let analyzed = analyze_string(src); - assert_eq!(analyzed.max_degree(), 8); + assert_eq!(analyzed.degree(), 8); let constants = generate(&analyzed); assert_eq!( constants, @@ -245,7 +245,7 @@ mod test { } + 1 }; "#; let analyzed = analyze_string(src); - assert_eq!(analyzed.max_degree(), 8); + assert_eq!(analyzed.degree(), 8); let constants = generate(&analyzed); assert_eq!( constants, @@ -261,7 +261,7 @@ mod test { let X: col = |i| if i < 3 { 7 } else { 9 }; "#; let analyzed = analyze_string(src); - assert_eq!(analyzed.max_degree(), 8); + assert_eq!(analyzed.degree(), 8); let constants = generate(&analyzed); assert_eq!( constants, @@ -278,7 +278,7 @@ mod test { pol constant EVEN(i) { 2 * minus_one(i) + 2 }; "#; let analyzed = analyze_string(src); - assert_eq!(analyzed.max_degree(), 8); + assert_eq!(analyzed.degree(), 8); let constants = generate(&analyzed); assert_eq!( constants, @@ -304,7 +304,7 @@ mod test { col fixed doubled_half_nibble(i) { half_nibble_f(i / 2) }; "#; let analyzed = analyze_string(src); - assert_eq!(analyzed.max_degree(), 10); + assert_eq!(analyzed.degree(), 10); let constants = generate(&analyzed); assert_eq!(constants.len(), 4); assert_eq!( @@ -346,7 +346,7 @@ mod test { col fixed ref_other = [n-1, f(1), 8] + [0]*; "#; let analyzed = analyze_string(src); - assert_eq!(analyzed.max_degree(), 10); + assert_eq!(analyzed.degree(), 10); let constants = generate(&analyzed); assert_eq!(constants.len(), 3); assert_eq!( @@ -377,7 +377,7 @@ mod test { col fixed arr = [0, 1, 2]* + [7]; "#; let analyzed = analyze_string(src); - assert_eq!(analyzed.max_degree(), 10); + assert_eq!(analyzed.degree(), 10); let constants = generate(&analyzed); assert_eq!(constants.len(), 1); assert_eq!( @@ -412,7 +412,7 @@ mod test { col fixed greater_eq(i) { if id(i) >= inv(i) { 1 } else { 0 } }; "#; let analyzed = analyze_string(src); - assert_eq!(analyzed.max_degree(), 6); + assert_eq!(analyzed.degree(), 6); let constants = generate(&analyzed); assert_eq!( constants[0], @@ -471,7 +471,7 @@ mod test { let x: col = |i| w(i) + 1; "#; let analyzed = analyze_string::(src); - assert_eq!(analyzed.max_degree(), 10); + assert_eq!(analyzed.degree(), 10); generate(&analyzed); } @@ -484,7 +484,7 @@ mod test { let x = |i| w(i) + 1; "#; let analyzed = analyze_string::(src); - assert_eq!(analyzed.max_degree(), 10); + assert_eq!(analyzed.degree(), 10); generate(&analyzed); } @@ -498,7 +498,7 @@ mod test { col fixed y = [1, 2, 3]*; "#; let analyzed = analyze_string::(src); - assert_eq!(analyzed.max_degree(), 10); + assert_eq!(analyzed.degree(), 10); generate(&analyzed); } @@ -513,7 +513,7 @@ mod test { let Y: col = y; "#; let analyzed = analyze_string::(src); - assert_eq!(analyzed.max_degree(), 4); + assert_eq!(analyzed.degree(), 4); let constants = generate(&analyzed); assert_eq!( constants[0], @@ -536,7 +536,7 @@ mod test { let x: col = |i| (1 << (2000 + i)) >> 2000; "#; let analyzed = analyze_string::(src); - assert_eq!(analyzed.max_degree(), 4); + assert_eq!(analyzed.degree(), 4); let constants = generate(&analyzed); assert_eq!( constants[0], @@ -556,7 +556,7 @@ mod test { let x: col = |i| 100 + x_arr[i]; "#; let analyzed = analyze_string::(src); - assert_eq!(analyzed.max_degree(), 4); + assert_eq!(analyzed.degree(), 4); let constants = generate(&analyzed); // Semantics of p % q involving negative numbers: // sgn(p) * |p| % |q| @@ -576,7 +576,7 @@ mod test { let fe = || fe(); "#; let analyzed = analyze_string::(src); - assert_eq!(analyzed.max_degree(), 4); + assert_eq!(analyzed.degree(), 4); let constants = generate(&analyzed); assert_eq!( constants[0], @@ -600,7 +600,7 @@ mod test { let a: col = |i| std::convert::fe(i + seven) + seven; "#; let analyzed = analyze_string::(src); - assert_eq!(analyzed.max_degree(), 4); + assert_eq!(analyzed.degree(), 4); let constants = generate(&analyzed); assert_eq!( constants[0], diff --git a/executor/src/witgen/block_processor.rs b/executor/src/witgen/block_processor.rs index c7931ddf0..b8668e71f 100644 --- a/executor/src/witgen/block_processor.rs +++ b/executor/src/witgen/block_processor.rs @@ -1,7 +1,7 @@ use std::collections::HashSet; use powdr_ast::analyzed::{ - AlgebraicExpression as Expression, AlgebraicReference, Identity, RawPolyID as PolyID, + AlgebraicExpression as Expression, AlgebraicReference, Identity, PolyID, }; use powdr_number::FieldElement; @@ -108,7 +108,7 @@ impl<'a, 'b, 'c, T: FieldElement, Q: QueryCallback> BlockProcessor<'a, 'b, 'c mod tests { use std::collections::BTreeMap; - use powdr_ast::analyzed::{PolynomialType, RawPolyID as PolyID}; + use powdr_ast::analyzed::{PolyID, PolynomialType}; use powdr_number::{FieldElement, GoldilocksField}; use powdr_pil_analyzer::analyze_string; @@ -155,7 +155,7 @@ mod tests { let mut machines = []; // The degree is the max degree as we have a single machine - let degree = fixed_data.analyzed.max_degree(); + let degree = fixed_data.analyzed.degree(); let columns = (0..fixed_data.witness_cols.len()) .map(move |i| PolyID { diff --git a/executor/src/witgen/data_structures/column_map.rs b/executor/src/witgen/data_structures/column_map.rs index 22b06d725..e90aedb2b 100644 --- a/executor/src/witgen/data_structures/column_map.rs +++ b/executor/src/witgen/data_structures/column_map.rs @@ -3,7 +3,7 @@ use std::{ ops::{Index, IndexMut}, }; -use powdr_ast::analyzed::{PolynomialType, RawPolyID as PolyID}; +use powdr_ast::analyzed::{PolyID, PolynomialType}; // Marker types for each PolynomialType #[derive(Clone, Copy)] @@ -61,8 +61,8 @@ impl ColumnMap { { let mut values: Vec = (0..len).map(|_| V::default()).collect(); for (poly, value) in items { - values[poly.id() as usize] = value; - assert_eq!(poly.ptype(), T::P_TYPE); + values[poly.id as usize] = value; + assert_eq!(poly.ptype, T::P_TYPE); } ColumnMap { @@ -100,7 +100,7 @@ impl Index<&PolyID> for ColumnMap { #[inline] fn index(&self, poly_id: &PolyID) -> &Self::Output { - assert!(poly_id.ptype() == T::P_TYPE); + assert!(poly_id.ptype == T::P_TYPE); &self.values[poly_id.id as usize] } } @@ -108,7 +108,7 @@ impl Index<&PolyID> for ColumnMap { impl IndexMut<&PolyID> for ColumnMap { #[inline] fn index_mut(&mut self, poly_id: &PolyID) -> &mut Self::Output { - assert!(poly_id.ptype() == T::P_TYPE); + assert!(poly_id.ptype == T::P_TYPE); &mut self.values[poly_id.id as usize] } } diff --git a/executor/src/witgen/data_structures/finalizable_data.rs b/executor/src/witgen/data_structures/finalizable_data.rs index 2652300a3..b689ceec8 100644 --- a/executor/src/witgen/data_structures/finalizable_data.rs +++ b/executor/src/witgen/data_structures/finalizable_data.rs @@ -4,7 +4,7 @@ use std::{ }; use bit_vec::BitVec; -use powdr_ast::analyzed::RawPolyID as PolyID; +use powdr_ast::analyzed::PolyID; use powdr_number::FieldElement; use crate::witgen::rows::Row; diff --git a/executor/src/witgen/fixed_evaluator.rs b/executor/src/witgen/fixed_evaluator.rs index 5c91ef9af..77a57f9e6 100644 --- a/executor/src/witgen/fixed_evaluator.rs +++ b/executor/src/witgen/fixed_evaluator.rs @@ -23,7 +23,7 @@ impl<'a, T: FieldElement> SymbolicVariables for FixedEvaluator<'a, T> { poly.is_fixed(), "Can only access fixed columns in the fixed evaluator." ); - let col_data = self.fixed_data.fixed_cols[&poly.poly_id.raw].values; + let col_data = self.fixed_data.fixed_cols[&poly.poly_id].values; let degree = col_data.len(); let row = if poly.next { (self.row + 1) % degree diff --git a/executor/src/witgen/generator.rs b/executor/src/witgen/generator.rs index 4892836ee..a54b6fc0f 100644 --- a/executor/src/witgen/generator.rs +++ b/executor/src/witgen/generator.rs @@ -1,5 +1,5 @@ use powdr_ast::analyzed::{ - AlgebraicExpression as Expression, AlgebraicReference, Identity, RawPolyID as PolyID, + AlgebraicExpression as Expression, AlgebraicReference, Identity, PolyID, }; use powdr_number::{DegreeType, FieldElement}; use std::collections::{BTreeMap, HashMap, HashSet}; @@ -118,14 +118,11 @@ impl<'a, T: FieldElement> Generator<'a, T> { identities: Vec<&'a Identity>>, witnesses: HashSet, latch: Option>, - degree: DegreeType, ) -> Self { - let witnesses = witnesses.into_iter().map(Into::into).collect(); - let data = FinalizableData::new(&witnesses); Self { - degree, + degree: fixed_data.common_degree(witnesses.iter().cloned()), connecting_identities: connecting_identities.clone(), name, fixed_data, @@ -232,13 +229,12 @@ impl<'a, T: FieldElement> Generator<'a, T> { let degree = self.degree(); let mut processor = VmProcessor::new( - RowIndex::from_degree(row_offset, self.degree()), + RowIndex::from_degree(row_offset, degree), self.fixed_data, &self.identities, &self.witnesses, data, mutable_state, - degree, ); if let Some(outer_query) = outer_query { processor = processor.with_outer_query(outer_query); diff --git a/executor/src/witgen/global_constraints.rs b/executor/src/witgen/global_constraints.rs index de707b749..1b540fcf6 100644 --- a/executor/src/witgen/global_constraints.rs +++ b/executor/src/witgen/global_constraints.rs @@ -5,7 +5,7 @@ use num_traits::Zero; use powdr_ast::analyzed::{ AlgebraicBinaryOperation, AlgebraicBinaryOperator, AlgebraicExpression as Expression, - AlgebraicReference, Identity, IdentityKind, PolynomialType, RawPolyID as PolyID, + AlgebraicReference, Identity, IdentityKind, PolyID, PolynomialType, }; use powdr_number::FieldElement; @@ -32,7 +32,7 @@ impl<'a, T: FieldElement> RangeConstraintSet<&AlgebraicReference, T> { fn range_constraint(&self, id: &AlgebraicReference) -> Option> { assert!(!id.next); - self.range_constraints.get(&id.poly_id.raw).cloned() + self.range_constraints.get(&id.poly_id).cloned() } } @@ -94,9 +94,9 @@ impl RangeConstraintSet<&AlgebraicReference, T> for GlobalConst fn range_constraint(&self, id: &AlgebraicReference) -> Option> { assert!(!id.next); let poly_id = id.poly_id; - match poly_id.ptype() { - PolynomialType::Constant => self.fixed_constraints[&poly_id.raw].clone(), - PolynomialType::Committed => self.witness_constraints[&poly_id.raw].clone(), + match poly_id.ptype { + PolynomialType::Constant => self.fixed_constraints[&poly_id].clone(), + PolynomialType::Committed => self.witness_constraints[&poly_id].clone(), PolynomialType::Intermediate => None, } } @@ -145,7 +145,7 @@ pub fn set_global_constraints<'a, T: FieldElement>( log::debug!("Determined the following global range constraints:"); for (poly_id, con) in &known_constraints { - if poly_id.ptype() == PolynomialType::Committed { + if poly_id.ptype == PolynomialType::Committed { log::debug!(" {}: {con}", fixed_data.column_name(poly_id)); } } @@ -158,7 +158,7 @@ pub fn set_global_constraints<'a, T: FieldElement>( let mut witness_constraints: WitnessColumnMap>> = fixed_data.witness_map_with(None); for (poly_id, con) in known_constraints { - if poly_id.ptype() == PolynomialType::Committed { + if poly_id.ptype == PolynomialType::Committed { // It's theoretically possible to have a constraint for both X and X'. // In that case, we take the conjunction. let con = witness_constraints[&poly_id] @@ -243,9 +243,9 @@ fn propagate_constraints( if let (Some(left), Some(right)) = (try_to_simple_poly(left), try_to_simple_poly(right)) { - if let Some(constraint) = known_constraints.get(&right.poly_id.raw).cloned() { + if let Some(constraint) = known_constraints.get(&right.poly_id).cloned() { known_constraints - .entry(left.poly_id.raw) + .entry(left.poly_id) .and_modify(|existing| *existing = existing.conjunction(&constraint)) .or_insert(constraint); } @@ -256,7 +256,7 @@ fn propagate_constraints( // provides all values in the span. if let Some(name) = try_to_simple_poly(&identity.right.expressions[0]) { if try_to_simple_poly(&identity.left.expressions[0]).is_some() - && full_span.contains(&name.poly_id.raw) + && full_span.contains(&name.poly_id) { remove = true; } @@ -304,7 +304,7 @@ fn is_binary_constraint(expr: &Expression) -> Option return None; } if (value1.is_zero() && value2.is_one()) || (value1.is_one() && value2.is_zero()) { - return Some(id1.poly_id.raw); + return Some(id1.poly_id); } } } @@ -340,7 +340,7 @@ fn try_transfer_constraints( .flat_map(|(poly, cons)| { if let Constraint::RangeConstraint(cons) = cons { assert!(!poly.next); - Some((poly.poly_id.raw, cons)) + Some((poly.poly_id, cons)) } else { None } @@ -359,7 +359,7 @@ fn smallest_period_candidate(fixed: &[T]) -> Option { mod test { use std::collections::BTreeMap; - use powdr_ast::analyzed::PolynomialType; + use powdr_ast::analyzed::{PolyID, PolynomialType}; use powdr_number::GoldilocksField; use pretty_assertions::assert_eq; use test_log::test; diff --git a/executor/src/witgen/machines/block_machine.rs b/executor/src/witgen/machines/block_machine.rs index 0458bd50b..4f74cf939 100644 --- a/executor/src/witgen/machines/block_machine.rs +++ b/executor/src/witgen/machines/block_machine.rs @@ -17,8 +17,8 @@ use crate::witgen::{machines::Machine, EvalError, EvalValue, IncompleteCause}; use crate::witgen::{MutableState, QueryCallback}; use itertools::Itertools; use powdr_ast::analyzed::{ - AlgebraicExpression as Expression, AlgebraicReference, Identity, IdentityKind, PolynomialType, - RawPolyID as PolyID, + AlgebraicExpression as Expression, AlgebraicReference, Identity, IdentityKind, PolyID, + PolynomialType, }; use powdr_ast::parsed::visitor::ExpressionVisitable; use powdr_number::{DegreeType, FieldElement}; @@ -128,15 +128,16 @@ impl<'a, T: FieldElement> BlockMachine<'a, T> { connecting_identities: &BTreeMap>>, identities: &[&'a Identity>], witness_cols: &HashSet, - degree: u64, ) -> Option { + let degree = fixed_data.common_degree(witness_cols.iter().cloned()); + let (is_permutation, block_size, latch_row) = detect_connection_type_and_block_size(fixed_data, connecting_identities, degree)?; for id in connecting_identities.values() { for r in id.right.expressions.iter() { if let Some(poly) = try_to_simple_poly(r) { - if poly.poly_id.ptype() == PolynomialType::Constant { + if poly.poly_id.ptype == PolynomialType::Constant { // It does not really make sense to have constant polynomials on the RHS // of a block machine lookup, as all constant polynomials are periodic, so // it would always return the same value. @@ -252,7 +253,7 @@ fn try_to_period( return None; } - let values = fixed_data.fixed_cols[&poly.poly_id.raw].values; + let values = fixed_data.fixed_cols[&poly.poly_id].values; let offset = values.iter().position(|v| v.is_one())?; let period = 1 + values.iter().skip(offset + 1).position(|v| v.is_one())?; diff --git a/executor/src/witgen/machines/double_sorted_witness_machine.rs b/executor/src/witgen/machines/double_sorted_witness_machine.rs index 3beafa820..52f2565e6 100644 --- a/executor/src/witgen/machines/double_sorted_witness_machine.rs +++ b/executor/src/witgen/machines/double_sorted_witness_machine.rs @@ -10,9 +10,7 @@ use crate::witgen::{EvalResult, FixedData, MutableState, QueryCallback}; use crate::witgen::{EvalValue, IncompleteCause}; use powdr_number::{DegreeType, FieldElement}; -use powdr_ast::analyzed::{ - AlgebraicExpression as Expression, Identity, IdentityKind, RawPolyID as PolyID, -}; +use powdr_ast::analyzed::{AlgebraicExpression as Expression, Identity, IdentityKind, PolyID}; /// If all witnesses of a machine have a name in this list (disregarding the namespace), /// we'll consider it to be a double-sorted machine. @@ -83,8 +81,9 @@ impl<'a, T: FieldElement> DoubleSortedWitnesses<'a, T> { fixed_data: &'a FixedData, connecting_identities: &BTreeMap>>, witness_cols: &HashSet, - degree: u64, ) -> Option { + let degree = fixed_data.common_degree(witness_cols.iter().cloned()); + // get the namespaces and column names let (mut namespaces, columns): (HashSet<_>, HashSet<_>) = witness_cols .iter() @@ -110,7 +109,7 @@ impl<'a, T: FieldElement> DoubleSortedWitnesses<'a, T> { .selector .as_ref() .and_then(|r| try_to_simple_poly(r)) - .map(|p| (i.id, p.poly_id.raw)) + .map(|p| (i.id, p.poly_id)) }) .collect::>>()?; diff --git a/executor/src/witgen/machines/fixed_lookup_machine.rs b/executor/src/witgen/machines/fixed_lookup_machine.rs index 89e2dafef..1b174901b 100644 --- a/executor/src/witgen/machines/fixed_lookup_machine.rs +++ b/executor/src/witgen/machines/fixed_lookup_machine.rs @@ -90,23 +90,23 @@ impl IndexedColumns { "Generating index for lookup in columns (in: {}, out: {})", sorted_input_fixed_columns .iter() - .map(|c| fixed_data.column_name(&c.raw).to_string()) + .map(|c| fixed_data.column_name(c).to_string()) .join(", "), sorted_output_fixed_columns .iter() - .map(|c| fixed_data.column_name(&c.raw).to_string()) + .map(|c| fixed_data.column_name(c).to_string()) .join(", ") ); // get all values for the columns to be indexed let input_column_values = sorted_input_fixed_columns .iter() - .map(|id| fixed_data.fixed_cols[&id.raw].values) + .map(|id| fixed_data.fixed_cols[id].values) .collect::>(); let output_column_values = sorted_output_fixed_columns .iter() - .map(|id| fixed_data.fixed_cols[&id.raw].values) + .map(|id| fixed_data.fixed_cols[id].values) .collect::>(); let degree = input_column_values @@ -241,7 +241,7 @@ impl FixedLookup { ) -> EvalResult<'b, T> { if left.len() == 1 && !left.first().unwrap().is_constant() - && right.peek().unwrap().poly_id.ptype() == PolynomialType::Constant + && right.peek().unwrap().poly_id.ptype == PolynomialType::Constant { // Lookup of the form "c { X } in { B }". Might be a conditional range check. return self.process_range_check(rows, left.first().unwrap(), right.peek().unwrap()); @@ -295,7 +295,7 @@ impl FixedLookup { let output = output_columns .iter() - .map(|column| fixed_data.fixed_cols[&column.raw].values[row]); + .map(|column| fixed_data.fixed_cols[column].values[row]); let mut result = EvalValue::complete(vec![]); for (l, r) in output_expressions.into_iter().zip(output) { @@ -337,7 +337,7 @@ impl FixedLookup { updates .constraints .into_iter() - .filter(|(poly, _)| poly.poly_id.ptype() == PolynomialType::Committed) + .filter(|(poly, _)| poly.poly_id.ptype == PolynomialType::Committed) .collect(), IncompleteCause::NotConcrete, )) @@ -357,7 +357,7 @@ impl RangeConstraintSet<&AlgebraicReference, T> for UnifiedRangeConstraints<'_, T> { fn range_constraint(&self, poly: &AlgebraicReference) -> Option> { - match poly.poly_id.ptype() { + match poly.poly_id.ptype { PolynomialType::Committed => self.witness_constraints.range_constraint(poly), PolynomialType::Constant => self.global_constraints.range_constraint(poly), PolynomialType::Intermediate => unimplemented!(), diff --git a/executor/src/witgen/machines/machine_extractor.rs b/executor/src/witgen/machines/machine_extractor.rs index 0d35fc804..ef3b5f581 100644 --- a/executor/src/witgen/machines/machine_extractor.rs +++ b/executor/src/witgen/machines/machine_extractor.rs @@ -10,44 +10,16 @@ use super::KnownMachine; use crate::witgen::generator::Generator; use crate::witgen::machines::write_once_memory::WriteOnceMemory; use itertools::Itertools; -use powdr_ast::analyzed::{ - AlgebraicExpression as Expression, Identity, IdentityKind, RawPolyID as PolyID, -}; +use powdr_ast::analyzed::{AlgebraicExpression as Expression, Identity, IdentityKind, PolyID}; use powdr_ast::parsed::visitor::ExpressionVisitable; use powdr_ast::parsed::SelectedExpressions; -use powdr_number::DegreeType; use powdr_number::FieldElement; pub struct ExtractionOutput<'a, T: FieldElement> { pub fixed_lookup: FixedLookup, pub machines: Vec>, - /// The leftover witnesses and identities, checked to having the same degree - pub base: Option> -} - -pub struct Base<'a, T> { - pub identities: Vec<&'a Identity>>, - pub witnesses: HashSet, - pub degree: DegreeType, -} - -fn to_raw_and_degree( - machine_witnesses: HashSet, -) -> (HashSet, Option) { - let mut res = HashSet::default(); - let mut degree = None; - for id in machine_witnesses { - res.insert(id.raw); - match degree { - None => { - degree = id.degree; - } - degree => { - assert_eq!(id.degree, degree); - } - } - } - (res, degree) + pub base_identities: Vec<&'a Identity>>, + pub base_witnesses: HashSet, } /// Finds machines in the witness columns and identities @@ -61,14 +33,7 @@ pub fn split_out_machines<'a, T: FieldElement>( let mut machines: Vec> = vec![]; - // we use `PolyID` because we need to extract the degree of each machine - let all_witnesses: HashSet = fixed - .analyzed - .committed_polys_in_source_order() - .iter() - .flat_map(|(symbol, _)| symbol.array_elements().map(|(_, id)| id)) - .collect(); - + let all_witnesses = fixed.witness_cols.keys().collect::>(); let mut remaining_witnesses = all_witnesses.clone(); let mut base_identities = identities.clone(); let mut id_counter = 0; @@ -118,7 +83,7 @@ pub fn split_out_machines<'a, T: FieldElement>( "\nExtracted a machine with the following witnesses:\n{} \n and identities:\n{} \n and connecting identities:\n{}", machine_witnesses .iter() - .map(|s| fixed.column_name(&s.raw)) + .map(|s| fixed.column_name(s)) .sorted() .collect::>() .join(", "), @@ -135,7 +100,7 @@ pub fn split_out_machines<'a, T: FieldElement>( ); let first_witness = machine_witnesses.iter().next().unwrap(); - let first_witness_name = fixed.column_name(&first_witness.raw); + let first_witness_name = fixed.column_name(first_witness); let namespace = first_witness_name .rfind('.') .map(|idx| &first_witness_name[..idx]); @@ -147,19 +112,12 @@ pub fn split_out_machines<'a, T: FieldElement>( id_counter += 1; let name_with_type = |t: &str| format!("Secondary machine {id}: {name} ({t})"); - // Internal processing happens over `RawPolyID` - let (machine_witnesses, degree) = to_raw_and_degree(machine_witnesses); - - // We must have found a degree - let degree = degree.unwrap(); - if let Some(machine) = SortedWitnesses::try_new( name_with_type("SortedWitness"), fixed, &connecting_identities, &machine_identities, &machine_witnesses, - degree, ) { log::debug!("Detected machine: sorted witnesses / write-once memory"); machines.push(KnownMachine::SortedWitnesses(machine)); @@ -168,7 +126,6 @@ pub fn split_out_machines<'a, T: FieldElement>( fixed, &connecting_identities, &machine_witnesses, - degree, ) { log::debug!("Detected machine: memory"); machines.push(KnownMachine::DoubleSortedWitnesses(machine)); @@ -177,7 +134,6 @@ pub fn split_out_machines<'a, T: FieldElement>( fixed, &connecting_identities, &machine_identities, - degree, ) { log::debug!("Detected machine: write-once memory"); machines.push(KnownMachine::WriteOnceMemory(machine)); @@ -187,7 +143,6 @@ pub fn split_out_machines<'a, T: FieldElement>( &connecting_identities, &machine_identities, &machine_witnesses, - degree, ) { log::debug!("Detected machine: {machine}"); machines.push(KnownMachine::BlockMachine(machine)); @@ -219,23 +174,14 @@ pub fn split_out_machines<'a, T: FieldElement>( machine_identities, machine_witnesses, Some(latch), - degree, ))); } } - - let (remaining_witnesses, base_degree) = to_raw_and_degree(remaining_witnesses); - - let base = base_degree.map(|degree| Base { - identities: base_identities, - witnesses: remaining_witnesses, - degree - }); - ExtractionOutput { fixed_lookup, machines, - base, + base_identities, + base_witnesses: remaining_witnesses, } } @@ -243,10 +189,10 @@ pub fn split_out_machines<'a, T: FieldElement>( /// Two witnesses are row-connected if they are part of a polynomial identity /// or part of the same side of a lookup. fn all_row_connected_witnesses( - mut witnesses: HashSet, - all_witnesses: &HashSet, + mut witnesses: HashSet, + all_witnesses: &HashSet, identities: &[&Identity>], -) -> HashSet { +) -> HashSet { loop { let count = witnesses.len(); for i in identities { @@ -278,10 +224,8 @@ fn all_row_connected_witnesses( } /// Extracts all references to names from an identity. -pub fn refs_in_identity( - identity: &Identity>, -) -> HashSet { - let mut refs: HashSet<_> = Default::default(); +pub fn refs_in_identity(identity: &Identity>) -> HashSet { + let mut refs: HashSet = Default::default(); identity.pre_visit_expressions(&mut |expr| { ref_of_expression(expr).map(|id| refs.insert(id)); }); @@ -291,8 +235,8 @@ pub fn refs_in_identity( /// Extracts all references to names from selected expressions. pub fn refs_in_selected_expressions( sel_expr: &SelectedExpressions>, -) -> HashSet { - let mut refs: HashSet<_> = Default::default(); +) -> HashSet { + let mut refs: HashSet = Default::default(); sel_expr.pre_visit_expressions(&mut |expr| { ref_of_expression(expr).map(|id| refs.insert(id)); }); @@ -301,7 +245,7 @@ pub fn refs_in_selected_expressions( /// Extracts all references to names from an expression, /// NON-recursively. -pub fn ref_of_expression(expr: &Expression) -> Option { +pub fn ref_of_expression(expr: &Expression) -> Option { match expr { Expression::Reference(p) => Some(p.poly_id), _ => None, diff --git a/executor/src/witgen/machines/sorted_witness_machine.rs b/executor/src/witgen/machines/sorted_witness_machine.rs index f120a432d..fc7665fa9 100644 --- a/executor/src/witgen/machines/sorted_witness_machine.rs +++ b/executor/src/witgen/machines/sorted_witness_machine.rs @@ -12,8 +12,7 @@ use crate::witgen::{ }; use crate::witgen::{EvalValue, IncompleteCause, MutableState, QueryCallback}; use powdr_ast::analyzed::{ - AlgebraicExpression as Expression, AlgebraicReference, Identity, IdentityKind, - RawPolyID as PolyID, + AlgebraicExpression as Expression, AlgebraicReference, Identity, IdentityKind, PolyID, }; use powdr_number::{DegreeType, FieldElement}; @@ -43,8 +42,9 @@ impl<'a, T: FieldElement> SortedWitnesses<'a, T> { connecting_identities: &BTreeMap>>, identities: &[&Identity>], witnesses: &HashSet, - degree: u64, ) -> Option { + let degree = fixed_data.common_degree(witnesses.iter().cloned()); + if identities.len() != 1 { return None; } @@ -162,7 +162,7 @@ fn check_constraint(constraint: &Expression) -> Option Machine<'a, T> for SortedWitnesses<'a, T> { @@ -230,10 +230,7 @@ impl<'a, T: FieldElement> SortedWitnesses<'a, T> { .map(|e| caller_rows.evaluate(e).unwrap()) .collect::>(); let rhs = self.rhs_references.get(&identity_id).unwrap(); - let key_index = rhs - .iter() - .position(|&x| x.poly_id.raw == self.key_col) - .unwrap(); + let key_index = rhs.iter().position(|&x| x.poly_id == self.key_col).unwrap(); let key_value = left[key_index].constant_value().ok_or_else(|| { format!( @@ -248,7 +245,7 @@ impl<'a, T: FieldElement> SortedWitnesses<'a, T> { .entry(key_value) .or_insert_with(|| vec![None; self.witness_positions.len()]); for (l, &r) in left.iter().zip(rhs.iter()).skip(1) { - let stored_value = &mut stored_values[self.witness_positions[&r.poly_id.raw]]; + let stored_value = &mut stored_values[self.witness_positions[&r.poly_id]]; match stored_value { // There is a stored value Some(v) => { diff --git a/executor/src/witgen/machines/write_once_memory.rs b/executor/src/witgen/machines/write_once_memory.rs index dae3b1136..6fe6967f1 100644 --- a/executor/src/witgen/machines/write_once_memory.rs +++ b/executor/src/witgen/machines/write_once_memory.rs @@ -3,7 +3,7 @@ use std::collections::{BTreeMap, HashMap}; use itertools::{Either, Itertools}; use powdr_ast::analyzed::{ - AlgebraicExpression as Expression, Identity, IdentityKind, PolynomialType, RawPolyID as PolyID, + AlgebraicExpression as Expression, Identity, IdentityKind, PolyID, PolynomialType, }; use powdr_number::{DegreeType, FieldElement}; @@ -47,7 +47,6 @@ impl<'a, T: FieldElement> WriteOnceMemory<'a, T> { fixed_data: &'a FixedData<'a, T>, connecting_identities: &BTreeMap>>, identities: &[&Identity>], - degree: u64, ) -> Option { if !identities.is_empty() { return None; @@ -93,13 +92,15 @@ impl<'a, T: FieldElement> WriteOnceMemory<'a, T> { // Build a Vec for the key and value polynomials let (key_polys, value_polys): (Vec<_>, Vec<_>) = rhs_polys.into_iter().partition_map(|p| { assert!(!p.next); - if p.poly_id.ptype() == PolynomialType::Constant { - Either::Left(p.poly_id.raw) + if p.poly_id.ptype == PolynomialType::Constant { + Either::Left(p.poly_id) } else { - Either::Right(p.poly_id.raw) + Either::Right(p.poly_id) } }); + let degree = fixed_data.common_degree(key_polys.iter().chain(value_polys.iter()).cloned()); + let mut key_to_index = BTreeMap::new(); for row in 0..degree { let key = key_polys @@ -139,7 +140,7 @@ impl<'a, T: FieldElement> WriteOnceMemory<'a, T> { .iter() .zip(identity.right.expressions.iter()) .partition(|(_, r)| { - try_to_simple_poly(r).unwrap().poly_id.ptype() == PolynomialType::Constant + try_to_simple_poly(r).unwrap().poly_id.ptype == PolynomialType::Constant }); let key = key_expressions .into_iter() diff --git a/executor/src/witgen/mod.rs b/executor/src/witgen/mod.rs index c8b8cfb3c..a6a1a0377 100644 --- a/executor/src/witgen/mod.rs +++ b/executor/src/witgen/mod.rs @@ -1,9 +1,9 @@ -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::Arc; use powdr_ast::analyzed::{ - AlgebraicExpression, AlgebraicReference, Analyzed, Expression, FunctionValueDefinition, - PolynomialType, RawPolyID as PolyID, SymbolKind, TypedExpression, + AlgebraicExpression, AlgebraicReference, Analyzed, Expression, FunctionValueDefinition, PolyID, + PolynomialType, SymbolKind, TypedExpression, }; use powdr_ast::parsed::visitor::ExpressionVisitable; use powdr_ast::parsed::{FunctionKind, LambdaExpression}; @@ -187,9 +187,9 @@ impl<'a, 'b, T: FieldElement> WitnessGenerator<'a, 'b, T> { let ExtractionOutput { mut fixed_lookup, mut machines, - base, + base_identities, + base_witnesses, } = machines::machine_extractor::split_out_machines(&fixed, retained_identities); - let mut query_callback = self.query_callback; let mut mutable_state = MutableState { fixed_lookup: &mut fixed_lookup, @@ -197,24 +197,27 @@ impl<'a, 'b, T: FieldElement> WitnessGenerator<'a, 'b, T> { query_callback: &mut query_callback, }; - let main_columns = base.map(|base| { - let mut generator = Generator::new( - "Main Machine".to_string(), - &fixed, - &BTreeMap::new(), // No connecting identities - base.identities, - base.witnesses, - // We could set the latch of the main VM here, but then we would have to detect it. - // Instead, the main VM will be computed in one block, directly continuing into the - // infinite loop after the first return. - None, - base.degree, - ); - - generator.run(&mut mutable_state); - generator - .take_witness_col_values(mutable_state.fixed_lookup, mutable_state.query_callback) - }).unwrap_or_default(); + let main_columns = (!base_witnesses.is_empty()) + .then(|| { + let mut generator = Generator::new( + "Main Machine".to_string(), + &fixed, + &BTreeMap::new(), // No connecting identities + base_identities, + base_witnesses, + // We could set the latch of the main VM here, but then we would have to detect it. + // Instead, the main VM will be computed in one block, directly continuing into the + // infinite loop after the first return. + None, + ); + + generator.run(&mut mutable_state); + generator.take_witness_col_values( + mutable_state.fixed_lookup, + mutable_state.query_callback, + ) + }) + .unwrap_or_default(); // Get columns from machines let mut columns = mutable_state @@ -282,6 +285,33 @@ pub struct FixedData<'a, T: FieldElement> { } impl<'a, T: FieldElement> FixedData<'a, T> { + pub fn common_degree(&self, ids: impl IntoIterator) -> DegreeType { + let ids: HashSet<_> = ids.into_iter().collect(); + + self.analyzed + .definitions + .iter() + .flat_map(|(_, (symbol, _))| { + // get all polynomials and their degrees + symbol.array_elements().map(|(_, id)| { + ( + id, + symbol.degree.expect("all polynomials should have a degree"), + ) + }) + }) + .filter_map(|(id, degree)| ids.contains(&id).then_some(degree)) + .reduce(|acc, degree| { + assert_eq!(acc, degree); + acc + }) + .unwrap() + } + + pub fn degree(&self, id: PolyID) -> DegreeType { + self.common_degree(std::iter::once(id)) + } + pub fn new( analyzed: &'a Analyzed, fixed_col_values: &'a [(String, Vec)], @@ -314,7 +344,7 @@ impl<'a, T: FieldElement> FixedData<'a, T> { // Remove any hint for witness columns of a later stage // (because it might reference a challenge that is not available yet) let value = if poly.stage.unwrap_or_default() <= stage.into() { value.as_ref() } else { None }; - WitnessColumn::new(poly_id, &name, value, external_values) + WitnessColumn::new(poly_id.id as usize, &name, value, external_values) }) .collect::>() }, @@ -384,7 +414,7 @@ impl<'a, T: FieldElement> FixedData<'a, T> { } fn column_name(&self, poly_id: &PolyID) -> &str { - match poly_id.ptype() { + match poly_id.ptype { PolynomialType::Committed => &self.witness_cols[poly_id].poly.name, PolynomialType::Constant => &self.fixed_cols[poly_id].name, PolynomialType::Intermediate => unimplemented!(), @@ -396,6 +426,7 @@ impl<'a, T: FieldElement> FixedData<'a, T> { } fn external_witness(&self, row: DegreeType, column: &PolyID) -> Option { + let row = row % self.degree(*column); self.witness_cols[column] .external_values .as_ref() @@ -436,13 +467,11 @@ pub struct WitnessColumn<'a, T> { impl<'a, T> WitnessColumn<'a, T> { pub fn new( - poly_id: powdr_ast::analyzed::PolyID, + id: usize, name: &str, value: Option<&'a FunctionValueDefinition>, external_values: Option<&'a Vec>, ) -> WitnessColumn<'a, T> { - assert_eq!(poly_id.ptype(), PolynomialType::Committed); - let query = if let Some(FunctionValueDefinition::Expression(TypedExpression { e: query @ Expression::LambdaExpression( @@ -460,7 +489,10 @@ impl<'a, T> WitnessColumn<'a, T> { None }; let poly = AlgebraicReference { - poly_id, + poly_id: PolyID { + id: id as u64, + ptype: PolynomialType::Committed, + }, name: name.to_string(), next: false, }; diff --git a/executor/src/witgen/processor.rs b/executor/src/witgen/processor.rs index d6bb7ca48..3bd34a3a9 100644 --- a/executor/src/witgen/processor.rs +++ b/executor/src/witgen/processor.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, HashSet}; use powdr_ast::analyzed::PolynomialType; use powdr_ast::analyzed::{ - AlgebraicExpression as Expression, AlgebraicReference, Identity, RawPolyID as PolyID, + AlgebraicExpression as Expression, AlgebraicReference, Identity, PolyID, }; use powdr_number::{DegreeType, FieldElement}; @@ -142,7 +142,7 @@ impl<'a, 'b, 'c, T: FieldElement, Q: QueryCallback> Processor<'a, 'b, 'c, T, if let Some(right_poly) = try_to_simple_poly(r).map(|p| p.poly_id) { if let Some(l) = l.constant_value() { log::trace!(" {} = {}", r, l); - inputs.push((right_poly.raw, l)); + inputs.push((right_poly, l)); } } } @@ -304,7 +304,7 @@ Known values in current row (local: {row_index}, global {global_row_index}): .constraints .into_iter() .filter(|(poly, update)| match update { - Constraint::Assignment(_) => !self.is_relevant_witness[&poly.poly_id.raw], + Constraint::Assignment(_) => !self.is_relevant_witness[&poly.poly_id], // Range constraints are currently not communicated between callee and caller. Constraint::RangeConstraint(_) => false, }) @@ -334,19 +334,18 @@ Known values in current row (local: {row_index}, global {global_row_index}): for (poly, _) in &input_updates.constraints { let poly_id = poly.poly_id; - if let Some(start_row) = self.previously_set_inputs.remove(&poly_id.raw) { + if let Some(start_row) = self.previously_set_inputs.remove(&poly_id) { log::trace!( " Resetting previously set inputs for column: {}", - self.fixed_data.column_name(&poly_id.raw) + self.fixed_data.column_name(&poly_id) ); for row_index in start_row..row_index { - self.data[row_index][&poly_id.raw].value = CellValue::Unknown; + self.data[row_index][&poly_id].value = CellValue::Unknown; } } } for (poly, _) in &input_updates.constraints { - self.previously_set_inputs - .insert(poly.poly_id.raw, row_index); + self.previously_set_inputs.insert(poly.poly_id, row_index); } self.apply_updates(row_index, &input_updates, || "inputs".to_string()) } @@ -387,7 +386,7 @@ Known values in current row (local: {row_index}, global {global_row_index}): let mut progress = false; for (poly, c) in &updates.constraints { - if self.witness_cols.contains(&poly.poly_id.raw) { + if self.witness_cols.contains(&poly.poly_id) { // Build RowUpdater // (a bit complicated, because we need two mutable // references to elements of the same vector) @@ -427,11 +426,11 @@ Known values in current row (local: {row_index}, global {global_row_index}): // Have to materialize the other cells to please the borrow checker... let others = self .copy_constraints - .iter_equivalence_class((poly.poly_id.raw, row)) + .iter_equivalence_class((poly.poly_id, row)) .skip(1) .collect::>(); for (other_poly, other_row) in others { - if other_poly.ptype() != PolynomialType::Committed { + if other_poly.ptype != PolynomialType::Committed { unimplemented!( "Copy constraints to fixed columns are not yet supported (#1335)!" ); @@ -441,7 +440,7 @@ Known values in current row (local: {row_index}, global {global_row_index}): self.set_value(local_index, expression, *v, || { format!( "Copy constraint: {} (Row {}) -> {} (Row {})", - self.fixed_data.column_name(&poly.poly_id.raw), + self.fixed_data.column_name(&poly.poly_id), row, self.fixed_data.column_name(&other_poly), other_row diff --git a/executor/src/witgen/query_processor.rs b/executor/src/witgen/query_processor.rs index 8d63f37d3..fd97d1662 100644 --- a/executor/src/witgen/query_processor.rs +++ b/executor/src/witgen/query_processor.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use powdr_ast::analyzed::Challenge; -use powdr_ast::analyzed::{AlgebraicReference, Expression, PolynomialType, RawPolyID as PolyID}; +use powdr_ast::analyzed::{AlgebraicReference, Expression, PolyID, PolynomialType}; use powdr_ast::parsed::types::Type; use powdr_number::{BigInt, FieldElement}; use powdr_pil_analyzer::evaluator::{self, Definitions, EvalError, SymbolLookup, Value}; @@ -138,13 +138,13 @@ impl<'a, T: FieldElement> SymbolLookup<'a, T> for Symbols<'a, T> { &self, poly_ref: &AlgebraicReference, ) -> Result>, EvalError> { - Ok(Value::FieldElement(match poly_ref.poly_id.ptype() { + Ok(Value::FieldElement(match poly_ref.poly_id.ptype { PolynomialType::Committed | PolynomialType::Intermediate => self .rows .get_value(poly_ref) .ok_or(EvalError::DataNotAvailable)?, PolynomialType::Constant => { - let values = self.fixed_data.fixed_cols[&poly_ref.poly_id.raw].values; + let values = self.fixed_data.fixed_cols[&poly_ref.poly_id].values; let row = self.rows.current_row_index + if poly_ref.next { 1 } else { 0 }; values[usize::from(row)] } diff --git a/executor/src/witgen/rows.rs b/executor/src/witgen/rows.rs index 44048b4ca..98e03edd6 100644 --- a/executor/src/witgen/rows.rs +++ b/executor/src/witgen/rows.rs @@ -5,9 +5,7 @@ use std::{ }; use itertools::Itertools; -use powdr_ast::analyzed::{ - AlgebraicExpression as Expression, AlgebraicReference, RawPolyID as PolyID, -}; +use powdr_ast::analyzed::{AlgebraicExpression as Expression, AlgebraicReference, PolyID}; use powdr_number::{DegreeType, FieldElement}; use crate::witgen::Constraint; @@ -322,8 +320,8 @@ impl<'row, 'a, T: FieldElement> RowUpdater<'row, 'a, T> { fn get_cell_mut<'b>(&'b mut self, poly: &AlgebraicReference) -> &'b mut Cell<'a, T> { match poly.next { - false => &mut self.current[&poly.poly_id.raw], - true => &mut self.next[&poly.poly_id.raw], + false => &mut self.current[&poly.poly_id], + true => &mut self.next[&poly.poly_id], } } @@ -393,8 +391,8 @@ impl<'row, 'a, T: FieldElement> RowPair<'row, 'a, T> { /// [RowPair::from_single_row]. fn get_cell(&self, poly: &AlgebraicReference) -> &Cell { match (poly.next, self.next.as_ref()) { - (false, _) => &self.current[&poly.poly_id.raw], - (true, Some(next)) => &next[&poly.poly_id.raw], + (false, _) => &self.current[&poly.poly_id], + (true, Some(next)) => &next[&poly.poly_id], (true, None) => panic!("Tried to access next row, but it is not available."), } } diff --git a/executor/src/witgen/symbolic_witness_evaluator.rs b/executor/src/witgen/symbolic_witness_evaluator.rs index 4b2dc1048..6d9831d8b 100644 --- a/executor/src/witgen/symbolic_witness_evaluator.rs +++ b/executor/src/witgen/symbolic_witness_evaluator.rs @@ -45,7 +45,7 @@ where self.witness_access.value(poly) } else { // Constant polynomial (or something else) - let values = self.fixed_data.fixed_cols[&poly.poly_id.raw].values; + let values = self.fixed_data.fixed_cols[&poly.poly_id].values; let row = if poly.next { self.row + 1 } else { self.row } % (values.len() as DegreeType); Ok(values[row as usize].into()) diff --git a/executor/src/witgen/vm_processor.rs b/executor/src/witgen/vm_processor.rs index 59833c135..3be04b82e 100644 --- a/executor/src/witgen/vm_processor.rs +++ b/executor/src/witgen/vm_processor.rs @@ -1,8 +1,7 @@ use indicatif::{ProgressBar, ProgressStyle}; use itertools::Itertools; use powdr_ast::analyzed::{ - AlgebraicExpression as Expression, AlgebraicReference, Identity, IdentityKind, - RawPolyID as PolyID, + AlgebraicExpression as Expression, AlgebraicReference, Identity, IdentityKind, PolyID, }; use powdr_ast::indent; use powdr_number::{DegreeType, FieldElement}; @@ -72,8 +71,9 @@ impl<'a, 'b, 'c, T: FieldElement, Q: QueryCallback> VmProcessor<'a, 'b, 'c, T witnesses: &'c HashSet, data: FinalizableData<'a, T>, mutable_state: &'c mut MutableState<'a, 'b, T, Q>, - degree: u64, ) -> Self { + let degree = fixed_data.common_degree(witnesses.iter().cloned()); + let (identities_with_next, identities_without_next): (Vec<_>, Vec<_>) = identities .iter() .partition(|identity| identity.contains_next_ref()); diff --git a/pipeline/src/test_util.rs b/pipeline/src/test_util.rs index a90082e2d..041240d55 100644 --- a/pipeline/src/test_util.rs +++ b/pipeline/src/test_util.rs @@ -20,12 +20,12 @@ pub fn execute_test_file( file_name: &str, inputs: Vec, external_witness_values: Vec<(String, Vec)>, -) -> Result)>>, Vec> { +) -> Result<(), Vec> { Pipeline::default() .from_file(resolve_test_file(file_name)) .with_prover_inputs(inputs) .add_external_witness_values(external_witness_values) - .compute_witness() + .compute_witness().map(|_| ()) } pub fn verify_test_file( @@ -155,7 +155,7 @@ pub fn gen_halo2_proof(file_name: &str, inputs: Vec) { buffered_write_file(&setup_file_path, |writer| { powdr_backend::BackendType::Halo2 .factory::() - .generate_setup(pil.max_degree(), writer) + .generate_setup(pil.degree(), writer) .unwrap() }) .unwrap(); diff --git a/pipeline/tests/asm.rs b/pipeline/tests/asm.rs index 213853bfa..1ea8079fa 100644 --- a/pipeline/tests/asm.rs +++ b/pipeline/tests/asm.rs @@ -379,12 +379,12 @@ fn read_poly_files() { // check fixed cols (may have no fixed cols) let fixed = read_poly_set::(tmp_dir.as_path()); if !fixed.is_empty() { - assert_eq!(pil.max_degree(), fixed[0].1.len() as u64); + assert_eq!(pil.degree(), fixed[0].1.len() as u64); } // check witness cols (examples assumed to have at least one witness col) let witness = read_poly_set::(tmp_dir.as_path()); - assert_eq!(pil.max_degree(), witness[0].1.len() as u64); + assert_eq!(pil.degree(), witness[0].1.len() as u64); } } diff --git a/pipeline/tests/powdr_std.rs b/pipeline/tests/powdr_std.rs index 26f5bb37f..c927b2ad3 100644 --- a/pipeline/tests/powdr_std.rs +++ b/pipeline/tests/powdr_std.rs @@ -5,7 +5,8 @@ use powdr_number::{BigInt, Bn254Field, GoldilocksField}; use powdr_pil_analyzer::evaluator::Value; use powdr_pipeline::{ test_util::{ - evaluate_function, evaluate_integer_function, execute_test_file, gen_estark_proof, gen_halo2_proof, resolve_test_file, std_analyzed, test_halo2, verify_test_file + evaluate_function, evaluate_integer_function, execute_test_file, gen_estark_proof, + gen_halo2_proof, resolve_test_file, std_analyzed, test_halo2, verify_test_file, }, Pipeline, }; diff --git a/plonky3/src/circuit_builder.rs b/plonky3/src/circuit_builder.rs index 214316748..04c80e4d0 100644 --- a/plonky3/src/circuit_builder.rs +++ b/plonky3/src/circuit_builder.rs @@ -28,7 +28,7 @@ impl<'a, T: FieldElement> PowdrCircuit<'a, T> { pub fn generate_trace_rows(&self) -> RowMajorMatrix { // an iterator over all columns, committed then fixed let witness = self.witness().iter(); - let len = self.analyzed.max_degree(); + let len = self.analyzed.degree(); // for each row, get the value of each column let values = (0..len) @@ -106,13 +106,13 @@ impl<'a, T: FieldElement> PowdrCircuit<'a, T> { }; // witness columns indexes are unchanged, fixed ones are offset by `commitment_count` - let index = match poly_id.ptype() { + let index = match poly_id.ptype { PolynomialType::Committed => { assert!( - r.poly_id.id() < self.analyzed.commitment_count() as u64, + r.poly_id.id < self.analyzed.commitment_count() as u64, "Plonky3 expects `poly_id` to be contiguous" ); - r.poly_id.id() as usize + r.poly_id.id as usize } PolynomialType::Constant => { unreachable!( diff --git a/plonky3/src/prover/mod.rs b/plonky3/src/prover/mod.rs index 284f7674f..707580529 100644 --- a/plonky3/src/prover/mod.rs +++ b/plonky3/src/prover/mod.rs @@ -41,7 +41,7 @@ impl<'a, T: FieldElement> Plonky3Prover<'a, T> { let trace = circuit.generate_trace_rows(); - let config = get_config(self.analyzed.max_degree()); + let config = get_config(self.analyzed.degree()); let mut challenger = get_challenger(); @@ -62,7 +62,7 @@ impl<'a, T: FieldElement> Plonky3Prover<'a, T> { .map(|v| cast_to_goldilocks(*v)) .collect(); - let config = get_config(self.analyzed.max_degree()); + let config = get_config(self.analyzed.degree()); let mut challenger = get_challenger(); diff --git a/riscv/src/continuations.rs b/riscv/src/continuations.rs index ee3a4fa85..d912d69e8 100644 --- a/riscv/src/continuations.rs +++ b/riscv/src/continuations.rs @@ -71,7 +71,7 @@ where pipeline.compute_fixed_cols().unwrap(); // we can assume optimized_pil has been computed - let length = pipeline.compute_optimized_pil().unwrap().max_degree(); + let length = pipeline.compute_optimized_pil().unwrap().degree(); bootloader_inputs .into_iter() From 33cbb245b49ec53991b85714ae295399a2c24022 Mon Sep 17 00:00:00 2001 From: schaeff Date: Thu, 27 Jun 2024 16:22:01 +0200 Subject: [PATCH 22/39] clean --- executor/src/witgen/block_processor.rs | 1 - executor/src/witgen/generator.rs | 2 +- executor/src/witgen/machines/block_machine.rs | 2 +- .../src/witgen/machines/double_sorted_witness_machine.rs | 2 +- executor/src/witgen/machines/sorted_witness_machine.rs | 2 +- executor/src/witgen/machines/write_once_memory.rs | 2 +- executor/src/witgen/mod.rs | 6 +++--- executor/src/witgen/vm_processor.rs | 2 +- 8 files changed, 9 insertions(+), 10 deletions(-) diff --git a/executor/src/witgen/block_processor.rs b/executor/src/witgen/block_processor.rs index b8668e71f..3ba138235 100644 --- a/executor/src/witgen/block_processor.rs +++ b/executor/src/witgen/block_processor.rs @@ -154,7 +154,6 @@ mod tests { let mut fixed_lookup = FixedLookup::new(fixed_data.global_range_constraints().clone()); let mut machines = []; - // The degree is the max degree as we have a single machine let degree = fixed_data.analyzed.degree(); let columns = (0..fixed_data.witness_cols.len()) diff --git a/executor/src/witgen/generator.rs b/executor/src/witgen/generator.rs index a54b6fc0f..6fa07e7f9 100644 --- a/executor/src/witgen/generator.rs +++ b/executor/src/witgen/generator.rs @@ -122,7 +122,7 @@ impl<'a, T: FieldElement> Generator<'a, T> { let data = FinalizableData::new(&witnesses); Self { - degree: fixed_data.common_degree(witnesses.iter().cloned()), + degree: fixed_data.common_degree(&witnesses), connecting_identities: connecting_identities.clone(), name, fixed_data, diff --git a/executor/src/witgen/machines/block_machine.rs b/executor/src/witgen/machines/block_machine.rs index 4f74cf939..d06bfcc8b 100644 --- a/executor/src/witgen/machines/block_machine.rs +++ b/executor/src/witgen/machines/block_machine.rs @@ -129,7 +129,7 @@ impl<'a, T: FieldElement> BlockMachine<'a, T> { identities: &[&'a Identity>], witness_cols: &HashSet, ) -> Option { - let degree = fixed_data.common_degree(witness_cols.iter().cloned()); + let degree = fixed_data.common_degree(witness_cols); let (is_permutation, block_size, latch_row) = detect_connection_type_and_block_size(fixed_data, connecting_identities, degree)?; diff --git a/executor/src/witgen/machines/double_sorted_witness_machine.rs b/executor/src/witgen/machines/double_sorted_witness_machine.rs index 52f2565e6..4ba828cad 100644 --- a/executor/src/witgen/machines/double_sorted_witness_machine.rs +++ b/executor/src/witgen/machines/double_sorted_witness_machine.rs @@ -82,7 +82,7 @@ impl<'a, T: FieldElement> DoubleSortedWitnesses<'a, T> { connecting_identities: &BTreeMap>>, witness_cols: &HashSet, ) -> Option { - let degree = fixed_data.common_degree(witness_cols.iter().cloned()); + let degree = fixed_data.common_degree(witness_cols); // get the namespaces and column names let (mut namespaces, columns): (HashSet<_>, HashSet<_>) = witness_cols diff --git a/executor/src/witgen/machines/sorted_witness_machine.rs b/executor/src/witgen/machines/sorted_witness_machine.rs index fc7665fa9..91e1a2120 100644 --- a/executor/src/witgen/machines/sorted_witness_machine.rs +++ b/executor/src/witgen/machines/sorted_witness_machine.rs @@ -43,7 +43,7 @@ impl<'a, T: FieldElement> SortedWitnesses<'a, T> { identities: &[&Identity>], witnesses: &HashSet, ) -> Option { - let degree = fixed_data.common_degree(witnesses.iter().cloned()); + let degree = fixed_data.common_degree(witnesses); if identities.len() != 1 { return None; diff --git a/executor/src/witgen/machines/write_once_memory.rs b/executor/src/witgen/machines/write_once_memory.rs index 6fe6967f1..1a108ed3a 100644 --- a/executor/src/witgen/machines/write_once_memory.rs +++ b/executor/src/witgen/machines/write_once_memory.rs @@ -99,7 +99,7 @@ impl<'a, T: FieldElement> WriteOnceMemory<'a, T> { } }); - let degree = fixed_data.common_degree(key_polys.iter().chain(value_polys.iter()).cloned()); + let degree = fixed_data.common_degree(key_polys.iter().chain(value_polys.iter())); let mut key_to_index = BTreeMap::new(); for row in 0..degree { diff --git a/executor/src/witgen/mod.rs b/executor/src/witgen/mod.rs index a6a1a0377..58a22ad6d 100644 --- a/executor/src/witgen/mod.rs +++ b/executor/src/witgen/mod.rs @@ -285,7 +285,7 @@ pub struct FixedData<'a, T: FieldElement> { } impl<'a, T: FieldElement> FixedData<'a, T> { - pub fn common_degree(&self, ids: impl IntoIterator) -> DegreeType { + pub fn common_degree<'b>(&self, ids: impl IntoIterator) -> DegreeType { let ids: HashSet<_> = ids.into_iter().collect(); self.analyzed @@ -308,7 +308,7 @@ impl<'a, T: FieldElement> FixedData<'a, T> { .unwrap() } - pub fn degree(&self, id: PolyID) -> DegreeType { + pub fn degree(&self, id: &PolyID) -> DegreeType { self.common_degree(std::iter::once(id)) } @@ -426,7 +426,7 @@ impl<'a, T: FieldElement> FixedData<'a, T> { } fn external_witness(&self, row: DegreeType, column: &PolyID) -> Option { - let row = row % self.degree(*column); + let row = row % self.degree(column); self.witness_cols[column] .external_values .as_ref() diff --git a/executor/src/witgen/vm_processor.rs b/executor/src/witgen/vm_processor.rs index 3be04b82e..05f15c645 100644 --- a/executor/src/witgen/vm_processor.rs +++ b/executor/src/witgen/vm_processor.rs @@ -72,7 +72,7 @@ impl<'a, 'b, 'c, T: FieldElement, Q: QueryCallback> VmProcessor<'a, 'b, 'c, T data: FinalizableData<'a, T>, mutable_state: &'c mut MutableState<'a, 'b, T, Q>, ) -> Self { - let degree = fixed_data.common_degree(witnesses.iter().cloned()); + let degree = fixed_data.common_degree(witnesses); let (identities_with_next, identities_without_next): (Vec<_>, Vec<_>) = identities .iter() From 0bab7ac5d83aabd14167b27ff5d22f412c249024 Mon Sep 17 00:00:00 2001 From: schaeff Date: Thu, 27 Jun 2024 16:42:19 +0200 Subject: [PATCH 23/39] fix estark --- backend/src/estark/polygon_wrapper.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/estark/polygon_wrapper.rs b/backend/src/estark/polygon_wrapper.rs index b3ce86433..971730837 100644 --- a/backend/src/estark/polygon_wrapper.rs +++ b/backend/src/estark/polygon_wrapper.rs @@ -33,11 +33,11 @@ impl BackendFactory for Factory { } } -struct PolygonBackend(EStarkFilesCommon); +struct PolygonBackend<'b, F: FieldElement>(EStarkFilesCommon<'b, F>); // TODO: make both eStark backends interchangeable, from user perspective. // TODO: implement the other Backend trait methods. -impl<'a, F: FieldElement> Backend<'a, F> for PolygonBackend { +impl<'a: 'b, 'b, F: FieldElement> Backend<'a, F> for PolygonBackend<'b, F> { fn prove( &self, // Witness is taken from file written by the pipeline. From c71299bc72596a4542f8300ab9880cb43e8bc246 Mon Sep 17 00:00:00 2001 From: schaeff Date: Thu, 27 Jun 2024 16:58:13 +0200 Subject: [PATCH 24/39] lint --- backend/src/estark/polygon_wrapper.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/src/estark/polygon_wrapper.rs b/backend/src/estark/polygon_wrapper.rs index 971730837..6343015bb 100644 --- a/backend/src/estark/polygon_wrapper.rs +++ b/backend/src/estark/polygon_wrapper.rs @@ -60,8 +60,6 @@ impl<'a: 'b, 'b, F: FieldElement> Backend<'a, F> for PolygonBackend<'b, F> { let input_paths = self.0.write_files(witness, &output_dir)?; - let commits_path = output_dir.join("commits.bin"); - // Generate the proof. let proof_paths = pil_stark_prover::generate_proof( &input_paths.contraints, From 9f22b8256f31a168688d0b87f4a50d5fdf30c944 Mon Sep 17 00:00:00 2001 From: schaeff Date: Thu, 27 Jun 2024 17:54:10 +0200 Subject: [PATCH 25/39] fix common_degree --- executor/src/witgen/mod.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/executor/src/witgen/mod.rs b/executor/src/witgen/mod.rs index 58a22ad6d..c92a16545 100644 --- a/executor/src/witgen/mod.rs +++ b/executor/src/witgen/mod.rs @@ -289,10 +289,15 @@ impl<'a, T: FieldElement> FixedData<'a, T> { let ids: HashSet<_> = ids.into_iter().collect(); self.analyzed + // get all definitions .definitions .iter() - .flat_map(|(_, (symbol, _))| { - // get all polynomials and their degrees + // only keep polynomials + .filter_map(|(_, (symbol, _))| { + matches!(symbol.kind, SymbolKind::Poly(_)).then_some(symbol) + }) + // get all array elements and their degrees + .flat_map(|symbol| { symbol.array_elements().map(|(_, id)| { ( id, @@ -300,7 +305,9 @@ impl<'a, T: FieldElement> FixedData<'a, T> { ) }) }) + // only keep the ones matching our set .filter_map(|(id, degree)| ids.contains(&id).then_some(degree)) + // get the common degree .reduce(|acc, degree| { assert_eq!(acc, degree); acc From 5ad32e914fe40e28f809d008a14b4e1a7cd75890 Mon Sep 17 00:00:00 2001 From: schaeff Date: Thu, 27 Jun 2024 18:59:07 +0200 Subject: [PATCH 26/39] fix permutation test, remove empty p3 test --- pipeline/tests/powdr_std.rs | 2 +- plonky3/src/prover/mod.rs | 7 ------- std/protocols/permutation.asm | 4 +--- test_data/std/permutation_via_challenges.asm | 3 ++- test_data/std/permutation_via_challenges_ext.asm | 3 ++- 5 files changed, 6 insertions(+), 13 deletions(-) diff --git a/pipeline/tests/powdr_std.rs b/pipeline/tests/powdr_std.rs index adcdfb910..5844ecfec 100644 --- a/pipeline/tests/powdr_std.rs +++ b/pipeline/tests/powdr_std.rs @@ -94,7 +94,7 @@ fn permutation_via_challenges_bn() { } #[test] -#[should_panic = "Error reducing expression to constraint:\nExpression: std::protocols::permutation::permutation([main.z], main.permutation_constraint)\nError: FailedAssertion(\"The Goldilocks field is too small and needs to move to the extension field. Pass two accumulators instead!\")"] +#[should_panic = "Error reducing expression to constraint:\nExpression: std::protocols::permutation::permutation(main.is_first, [main.z], main.permutation_constraint)\nError: FailedAssertion(\"The Goldilocks field is too small and needs to move to the extension field. Pass two accumulators instead!\")"] fn permutation_via_challenges_gl() { let f = "std/permutation_via_challenges.asm"; Pipeline::::default() diff --git a/plonky3/src/prover/mod.rs b/plonky3/src/prover/mod.rs index 707580529..6fae09182 100644 --- a/plonky3/src/prover/mod.rs +++ b/plonky3/src/prover/mod.rs @@ -104,13 +104,6 @@ mod tests { run_test_goldilocks(content); } - #[test] - #[should_panic = "assertion failed: width >= 1"] - fn empty() { - let content = "namespace Global(8);"; - run_test_goldilocks(content); - } - #[test] fn add() { let content = r#" diff --git a/std/protocols/permutation.asm b/std/protocols/permutation.asm index 8a20be57b..48555a63c 100644 --- a/std/protocols/permutation.asm +++ b/std/protocols/permutation.asm @@ -18,8 +18,6 @@ use std::math::fp2::eval_ext; use std::math::fp2::from_base; use std::math::fp2::constrain_eq_ext; -let is_first: col = |i| if i == 0 { 1 } else { 0 }; - /// Get two phase-2 challenges to use in all permutation arguments. /// Note that this assumes that globally no other challenge of these IDs is used, /// and that challenges for multiple permutation arguments are re-used. @@ -117,7 +115,7 @@ let compute_next_z: Fp2, Constr -> fe[] = query |acc, permutation_constrai /// the wrapping behavior: The first accumulator is constrained to be 1, and the last /// accumulator is the same as the first one, because of wrapping. /// For small fields, this computation should happen in the extension field. -let permutation: expr[], Constr -> Constr[] = |acc, permutation_constraint| { +let permutation: expr, expr[], Constr -> Constr[] = |is_first, acc, permutation_constraint| { let (lhs_selector, lhs, rhs_selector, rhs) = unpack_permutation_constraint(permutation_constraint); diff --git a/test_data/std/permutation_via_challenges.asm b/test_data/std/permutation_via_challenges.asm index 75df8ff93..70396a255 100644 --- a/test_data/std/permutation_via_challenges.asm +++ b/test_data/std/permutation_via_challenges.asm @@ -20,6 +20,7 @@ machine Main with degree: 8 { // TODO: Functions currently cannot add witness columns at later stages, // so we have to manually create it here and pass it to permutation(). col witness stage(1) z; - permutation([z], permutation_constraint); + let is_first: col = |i| if i == 0 { 1 } else { 0 }; + permutation(is_first, [z], permutation_constraint); } diff --git a/test_data/std/permutation_via_challenges_ext.asm b/test_data/std/permutation_via_challenges_ext.asm index a0ea6b822..cf58593f4 100644 --- a/test_data/std/permutation_via_challenges_ext.asm +++ b/test_data/std/permutation_via_challenges_ext.asm @@ -23,7 +23,8 @@ machine Main with degree: 8 { // so we have to manually create it here and pass it to permutation(). col witness stage(1) z1; col witness stage(1) z2; - permutation([z1, z2], permutation_constraint); + let is_first: col = |i| if i == 0 { 1 } else { 0 }; + permutation(is_first, [z1, z2], permutation_constraint); // TODO: Helper columns, because we can't access the previous row in hints let hint = query |i| Query::Hint(compute_next_z(Fp2::Fp2(z1, z2), permutation_constraint)[i]); From ef455ac63b19366bb090a74d3fc554e4e6f5a688 Mon Sep 17 00:00:00 2001 From: schaeff Date: Fri, 28 Jun 2024 16:24:51 +0200 Subject: [PATCH 27/39] lint --- backend/src/estark/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/estark/mod.rs b/backend/src/estark/mod.rs index 34ce566f6..1a78b0a32 100644 --- a/backend/src/estark/mod.rs +++ b/backend/src/estark/mod.rs @@ -112,7 +112,7 @@ fn first_step_fixup( .collect::>() .into() } else { - fixed.into() + fixed }; (pil, patched_constants) From d24017ab616f4d1c87c87944253611b24a05d150 Mon Sep 17 00:00:00 2001 From: schaeff Date: Fri, 28 Jun 2024 18:54:34 +0200 Subject: [PATCH 28/39] remove speed bump --- executor/src/witgen/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/executor/src/witgen/mod.rs b/executor/src/witgen/mod.rs index c92a16545..bc71c1a2f 100644 --- a/executor/src/witgen/mod.rs +++ b/executor/src/witgen/mod.rs @@ -433,7 +433,6 @@ impl<'a, T: FieldElement> FixedData<'a, T> { } fn external_witness(&self, row: DegreeType, column: &PolyID) -> Option { - let row = row % self.degree(column); self.witness_cols[column] .external_values .as_ref() From 80a05ff44e5a3d2aa7a0edffe8c5477944b0512e Mon Sep 17 00:00:00 2001 From: schaeff Date: Fri, 28 Jun 2024 19:23:12 +0200 Subject: [PATCH 29/39] remove degree method --- executor/src/witgen/mod.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/executor/src/witgen/mod.rs b/executor/src/witgen/mod.rs index bc71c1a2f..be3cc58bf 100644 --- a/executor/src/witgen/mod.rs +++ b/executor/src/witgen/mod.rs @@ -315,10 +315,6 @@ impl<'a, T: FieldElement> FixedData<'a, T> { .unwrap() } - pub fn degree(&self, id: &PolyID) -> DegreeType { - self.common_degree(std::iter::once(id)) - } - pub fn new( analyzed: &'a Analyzed, fixed_col_values: &'a [(String, Vec)], From 17158a56f1a062ac817d60b2820de2da6baa1f49 Mon Sep 17 00:00:00 2001 From: schaeff Date: Mon, 1 Jul 2024 18:31:26 +0200 Subject: [PATCH 30/39] clean --- ast/src/analyzed/mod.rs | 9 ++++----- executor/src/witgen/machines/fixed_lookup_machine.rs | 11 +++-------- executor/src/witgen/mod.rs | 9 ++++----- 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/ast/src/analyzed/mod.rs b/ast/src/analyzed/mod.rs index 4abb94a56..3a4d88cf6 100644 --- a/ast/src/analyzed/mod.rs +++ b/ast/src/analyzed/mod.rs @@ -9,6 +9,7 @@ use std::iter; use std::ops::{self, ControlFlow}; use std::sync::Arc; +use itertools::Itertools; use powdr_number::{DegreeType, FieldElement}; use powdr_parser_util::SourceRef; use schemars::JsonSchema; @@ -52,15 +53,13 @@ impl Analyzed { self.definitions .values() .filter_map(|(symbol, _)| symbol.degree) - .reduce(|acc, degree| { - assert_eq!(acc, degree); - acc - }) + .unique() + .exactly_one() .unwrap() } /// Returns the set of all degrees in this [`Analyzed`]. - pub fn degrees(&self) -> HashSet { + pub fn degrees(&self) -> HashSet { self.definitions .values() .filter_map(|(symbol, _)| symbol.degree) diff --git a/executor/src/witgen/machines/fixed_lookup_machine.rs b/executor/src/witgen/machines/fixed_lookup_machine.rs index 4ae399e45..b87d255b1 100644 --- a/executor/src/witgen/machines/fixed_lookup_machine.rs +++ b/executor/src/witgen/machines/fixed_lookup_machine.rs @@ -113,14 +113,9 @@ impl IndexedColumns { .iter() .chain(output_column_values.iter()) .map(|values| values.len()) - .reduce(|acc, l| { - assert_eq!( - acc, l, - "all columns in a given lookup are expected to have the same degree" - ); - acc - }) - .unwrap(); + .unique() + .exactly_one() + .expect("all columns in a given lookup are expected to have the same degree"); let index: BTreeMap, IndexValue> = (0..degree) .fold( diff --git a/executor/src/witgen/mod.rs b/executor/src/witgen/mod.rs index 841bc5716..80c9455d6 100644 --- a/executor/src/witgen/mod.rs +++ b/executor/src/witgen/mod.rs @@ -1,6 +1,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::Arc; +use itertools::Itertools; use powdr_ast::analyzed::{ AlgebraicExpression, AlgebraicReference, Analyzed, Expression, FunctionValueDefinition, PolyID, PolynomialType, SymbolKind, TypedExpression, @@ -308,11 +309,9 @@ impl<'a, T: FieldElement> FixedData<'a, T> { // only keep the ones matching our set .filter_map(|(id, degree)| ids.contains(&id).then_some(degree)) // get the common degree - .reduce(|acc, degree| { - assert_eq!(acc, degree); - acc - }) - .unwrap() + .unique() + .exactly_one() + .unwrap_or_else(|_| panic!("expected all polynomials to have the same degree")) } pub fn new( From 890d239493c784b0ab0e4d8c3fec1b6891d87831 Mon Sep 17 00:00:00 2001 From: schaeff Date: Mon, 1 Jul 2024 19:21:38 +0200 Subject: [PATCH 31/39] clean --- executor/src/witgen/machines/block_machine.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/executor/src/witgen/machines/block_machine.rs b/executor/src/witgen/machines/block_machine.rs index ae22282ac..cb585ed73 100644 --- a/executor/src/witgen/machines/block_machine.rs +++ b/executor/src/witgen/machines/block_machine.rs @@ -1,6 +1,6 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::fmt::Display; -use std::iter; +use std::iter::{self, once}; use super::{EvalResult, FixedData, FixedLookup}; @@ -132,7 +132,7 @@ impl<'a, T: FieldElement> BlockMachine<'a, T> { let degree = fixed_data.common_degree(witness_cols); let (is_permutation, block_size, latch_row) = - detect_connection_type_and_block_size(fixed_data, connecting_identities, degree)?; + detect_connection_type_and_block_size(fixed_data, connecting_identities)?; for id in connecting_identities.values() { for r in id.right.expressions.iter() { @@ -181,7 +181,6 @@ impl<'a, T: FieldElement> BlockMachine<'a, T> { fn detect_connection_type_and_block_size<'a, T: FieldElement>( fixed_data: &'a FixedData<'a, T>, connecting_identities: &BTreeMap>, - degree: DegreeType, ) -> Option<(ConnectionType, usize, usize)> { // TODO we should check that the other constraints/fixed columns are also periodic. @@ -200,7 +199,7 @@ fn detect_connection_type_and_block_size<'a, T: FieldElement>( // We'd expect all RHS selectors to be fixed columns of the same period. connecting_identities .values() - .map(|id| try_to_period(&id.right.selector, fixed_data, degree)) + .map(|id| try_to_period(&id.right.selector, fixed_data)) .unique() .exactly_one() .ok()?? @@ -211,7 +210,7 @@ fn detect_connection_type_and_block_size<'a, T: FieldElement>( let find_max_period = |latch_candidates: BTreeSet>>| { latch_candidates .iter() - .filter_map(|e| try_to_period(e, fixed_data, degree)) + .filter_map(|e| try_to_period(e, fixed_data)) // If there is more than one period, the block size is the maximum period. .max_by_key(|&(_, period)| period) }; @@ -238,7 +237,6 @@ fn detect_connection_type_and_block_size<'a, T: FieldElement>( fn try_to_period( expr: &Option>, fixed_data: &FixedData, - degree: DegreeType, ) -> Option<(usize, usize)> { match expr { Some(expr) => { @@ -253,6 +251,8 @@ fn try_to_period( return None; } + let degree = fixed_data.common_degree(once(&poly.poly_id)); + let values = fixed_data.fixed_cols[&poly.poly_id].values; let offset = values.iter().position(|v| v.is_one())?; From f2f6d916617039cd26b3a90000d3e025a61ed65e Mon Sep 17 00:00:00 2001 From: schaeff Date: Mon, 1 Jul 2024 19:31:19 +0200 Subject: [PATCH 32/39] commit backend --- backend/src/estark/polygon_wrapper.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/src/estark/polygon_wrapper.rs b/backend/src/estark/polygon_wrapper.rs index b1e5a66e3..4758d5d6f 100644 --- a/backend/src/estark/polygon_wrapper.rs +++ b/backend/src/estark/polygon_wrapper.rs @@ -40,7 +40,6 @@ struct PolygonBackend(EStarkFilesCommon); impl<'a, F: FieldElement> Backend<'a, F> for PolygonBackend { fn prove( &self, - // Witness is taken from file written by the pipeline. witness: &[(String, Vec)], prev_proof: Option, // TODO: Implement challenges From 462eeb4cd5ff252a265dc64be7ed7bbebd72d8cb Mon Sep 17 00:00:00 2001 From: schaeff Date: Mon, 1 Jul 2024 19:36:15 +0200 Subject: [PATCH 33/39] fix test --- test_data/pil/different_degrees.pil | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_data/pil/different_degrees.pil b/test_data/pil/different_degrees.pil index d2462c6e5..128a5639f 100644 --- a/test_data/pil/different_degrees.pil +++ b/test_data/pil/different_degrees.pil @@ -19,6 +19,6 @@ namespace Main(2 * %N); col fixed CALL = [1, 0]*; (1 - CALL) * c = 0; - CALL {a, b, c} in {Add.a, Add.b, Add.c}; + CALL $ [a, b, c] in [Add.a, Add.b, Add.c]; From fa7157bc4673c59c471d47441112e5c0e3815136 Mon Sep 17 00:00:00 2001 From: schaeff Date: Mon, 1 Jul 2024 19:42:41 +0200 Subject: [PATCH 34/39] pull main --- executor/src/witgen/machines/block_machine.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor/src/witgen/machines/block_machine.rs b/executor/src/witgen/machines/block_machine.rs index ab273ec44..8b8495934 100644 --- a/executor/src/witgen/machines/block_machine.rs +++ b/executor/src/witgen/machines/block_machine.rs @@ -479,7 +479,7 @@ impl<'a, T: FieldElement> BlockMachine<'a, T> { if self.rows() > self.block_size as DegreeType { Some(RowIndex::from_degree( self.rows() - self.block_size as DegreeType + self.latch_row as DegreeType, - self.fixed_data.degree, + self.degree, )) } else { None From a3dd473992b571c20dc0deb40414382d5f22dc16 Mon Sep 17 00:00:00 2001 From: schaeff Date: Mon, 1 Jul 2024 20:23:15 +0200 Subject: [PATCH 35/39] fix lookup --- std/protocols/lookup.asm | 5 +---- test_data/std/lookup_via_challenges.asm | 3 ++- test_data/std/lookup_via_challenges_ext.asm | 3 ++- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/std/protocols/lookup.asm b/std/protocols/lookup.asm index 68ca77404..e77b04933 100644 --- a/std/protocols/lookup.asm +++ b/std/protocols/lookup.asm @@ -18,9 +18,6 @@ use std::math::fp2::eval_ext; use std::math::fp2::from_base; use std::math::fp2::constrain_eq_ext; - -let is_first: col = |i| if i == 0 { 1 } else { 0 }; - // challenges to be used in polynomial evaluation and folding different columns let alpha1: expr = challenge(0, 1); let alpha2: expr = challenge(0, 2); @@ -92,7 +89,7 @@ let compute_next_z: Fp2, Constr, expr -> fe[] = query |acc, lookup_constra // are done on the F_{p^2} extension field. // - lookup_constraint: The lookup constraint // - multiplicities: The multiplicities which shows how many times each RHS value appears in the LHS -let lookup: expr[], Constr, expr -> Constr[] = |acc, lookup_constraint, multiplicities| { +let lookup: expr, expr[], Constr, expr -> Constr[] = |is_first, acc, lookup_constraint, multiplicities| { let (lhs_selector, lhs, rhs_selector, rhs) = unpack_lookup_constraint(lookup_constraint); diff --git a/test_data/std/lookup_via_challenges.asm b/test_data/std/lookup_via_challenges.asm index 59ba20b32..012d489e5 100644 --- a/test_data/std/lookup_via_challenges.asm +++ b/test_data/std/lookup_via_challenges.asm @@ -22,6 +22,7 @@ machine Main with degree: 8 { // TODO: Functions currently cannot add witness columns at later stages, // so we have to manually create it here and pass it to permutation(). col witness stage(1) z; - lookup([z], lookup_constraint, m); + let is_first: col = |i| if i == 0 { 1 } else { 0 }; + lookup(is_first, [z], lookup_constraint, m); } \ No newline at end of file diff --git a/test_data/std/lookup_via_challenges_ext.asm b/test_data/std/lookup_via_challenges_ext.asm index 01b24cfeb..234349789 100644 --- a/test_data/std/lookup_via_challenges_ext.asm +++ b/test_data/std/lookup_via_challenges_ext.asm @@ -26,7 +26,8 @@ machine Main with degree: 8 { col witness stage(1) z1; col witness stage(1) z2; - lookup([z1, z2], lookup_constraint, m); + let is_first: col = |i| if i == 0 { 1 } else { 0 }; + lookup(is_first, [z1, z2], lookup_constraint, m); // TODO: Helper columns, because we can't access the previous row in hints let hint = query |i| Query::Hint(compute_next_z(Fp2::Fp2(z1, z2), lookup_constraint, m)[i]); From 109408b5656780ae8f89b3719af42eb93396d757 Mon Sep 17 00:00:00 2001 From: schaeff Date: Mon, 1 Jul 2024 22:56:31 +0200 Subject: [PATCH 36/39] fix test --- test_data/std/lookup_via_challenges_ext_simple.asm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_data/std/lookup_via_challenges_ext_simple.asm b/test_data/std/lookup_via_challenges_ext_simple.asm index 033d62213..1f54c98ae 100644 --- a/test_data/std/lookup_via_challenges_ext_simple.asm +++ b/test_data/std/lookup_via_challenges_ext_simple.asm @@ -19,7 +19,8 @@ machine Main with degree: 8 { col witness stage(1) z1; col witness stage(1) z2; - lookup([z1, z2], lookup_constraint, m); + let is_first: col = |i| if i == 0 { 1 } else { 0 }; + lookup(is_first, [z1, z2], lookup_constraint, m); // TODO: Helper columns, because we can't access the previous row in hints let hint = query |i| Query::Hint(compute_next_z(Fp2::Fp2(z1, z2), lookup_constraint, m)[i]); From 9977a13e3c7c32e4d17447a02e8b5dd069a940f3 Mon Sep 17 00:00:00 2001 From: schaeff Date: Tue, 2 Jul 2024 19:31:27 +0200 Subject: [PATCH 37/39] address review comments --- ast/src/analyzed/display.rs | 6 +++++- ast/src/analyzed/mod.rs | 4 ++-- backend/src/estark/bin_exporter.rs | 4 ++-- backend/src/estark/mod.rs | 14 +++++++------- executor/src/witgen/mod.rs | 8 ++++++++ pil-analyzer/src/pil_analyzer.rs | 26 ++++++++++++++------------ 6 files changed, 38 insertions(+), 24 deletions(-) diff --git a/ast/src/analyzed/display.rs b/ast/src/analyzed/display.rs index ff8a2a90b..a3cf304f5 100644 --- a/ast/src/analyzed/display.rs +++ b/ast/src/analyzed/display.rs @@ -21,7 +21,7 @@ use super::*; impl Display for Analyzed { fn fmt(&self, f: &mut Formatter<'_>) -> Result { - let mut current_namespace = AbsoluteSymbolPath::default(); + let (mut current_namespace, mut current_degree) = (AbsoluteSymbolPath::default(), None); let mut update_namespace = |name: &str, degree: Option, f: &mut Formatter<'_>| { let mut namespace = @@ -29,12 +29,16 @@ impl Display for Analyzed { let name = namespace.pop().unwrap(); if namespace != current_namespace { current_namespace = namespace; + current_degree = degree; writeln!( f, "namespace {}{};", current_namespace.relative_to(&Default::default()), degree.map(|d| format!("({d})")).unwrap_or_default() )?; + } else { + // If we're in the same namespace, the degree must match + assert_eq!(current_degree, degree); }; Ok((name, !current_namespace.is_empty())) }; diff --git a/ast/src/analyzed/mod.rs b/ast/src/analyzed/mod.rs index 3a4d88cf6..2c7ed4698 100644 --- a/ast/src/analyzed/mod.rs +++ b/ast/src/analyzed/mod.rs @@ -44,7 +44,7 @@ pub struct Analyzed { } impl Analyzed { - /// Returns the common degree in this [`Analyzed`]. + /// Returns the degree common among all symbols that have an explicit degree. /// /// # Panics /// @@ -58,7 +58,7 @@ impl Analyzed { .unwrap() } - /// Returns the set of all degrees in this [`Analyzed`]. + /// Returns the set of all explicit degrees in this [`Analyzed`]. pub fn degrees(&self) -> HashSet { self.definitions .values() diff --git a/backend/src/estark/bin_exporter.rs b/backend/src/estark/bin_exporter.rs index 6b9029ebe..c2148af78 100644 --- a/backend/src/estark/bin_exporter.rs +++ b/backend/src/estark/bin_exporter.rs @@ -41,8 +41,8 @@ fn write_polys_stream( } for i in 0..degree { - for (_name, constant) in polys { - let bytes = constant[i].to_bytes_le(); + for (_, values) in polys { + let bytes = values[i].to_bytes_le(); assert_eq!(bytes.len(), width); file.write_all(&bytes)?; } diff --git a/backend/src/estark/mod.rs b/backend/src/estark/mod.rs index 1a78b0a32..137ce144a 100644 --- a/backend/src/estark/mod.rs +++ b/backend/src/estark/mod.rs @@ -67,7 +67,7 @@ fn create_stark_struct(degree: DegreeType, hash_type: &str) -> StarkStruct { } } -type Constants = Vec<(String, Vec)>; +type Fixed = Vec<(String, Vec)>; /// eStark provers require a fixed column with the equivalent semantics to /// Polygon zkEVM's `L1` column. Powdr generated PIL will always have @@ -77,8 +77,8 @@ type Constants = Vec<(String, Vec)>; /// TODO Improve how this is done. fn first_step_fixup( pil: &Analyzed, - fixed: Arc>, -) -> (PIL, Arc>) { + fixed: Arc>, +) -> (PIL, Arc>) { let degree = pil.degree(); let mut pil: PIL = json_exporter::export(pil); @@ -123,7 +123,7 @@ struct EStarkFilesCommon { pil: PIL, /// If this field is present, it means the constants were patched with /// "main.first_step" column and must be written again to a file. - constants: Arc>, + constants: Arc>, output_dir: Option, proof_type: ProofType, } @@ -140,7 +140,7 @@ fn write_json_file(path: &Path, data: &T) -> Result<(), E impl<'a, F: FieldElement> EStarkFilesCommon { fn create( analyzed: &'a Analyzed, - fixed: Arc>, + fixed: Arc>, output_dir: Option, setup: Option<&mut dyn std::io::Read>, verification_key: Option<&mut dyn std::io::Read>, @@ -161,14 +161,14 @@ impl<'a, F: FieldElement> EStarkFilesCommon { } // Pre-process the PIL and fixed columns. - let (pil, constants) = first_step_fixup(analyzed, fixed); + let (pil, fixed) = first_step_fixup(analyzed, fixed); let proof_type: ProofType = ProofType::from(options); Ok(EStarkFilesCommon { degree: analyzed.degree(), pil, - constants, + constants: fixed, output_dir, proof_type, }) diff --git a/executor/src/witgen/mod.rs b/executor/src/witgen/mod.rs index 80c9455d6..29dc20d00 100644 --- a/executor/src/witgen/mod.rs +++ b/executor/src/witgen/mod.rs @@ -286,6 +286,14 @@ pub struct FixedData<'a, T: FieldElement> { } impl<'a, T: FieldElement> FixedData<'a, T> { + /// Returns the common degree of a set or polynomials + /// + /// # Panics + /// + /// Panics if: + /// - the degree is not unique + /// - the set of polynomials is empty + /// - a declared polynomial does not have an explicit degree pub fn common_degree<'b>(&self, ids: impl IntoIterator) -> DegreeType { let ids: HashSet<_> = ids.into_iter().collect(); diff --git a/pil-analyzer/src/pil_analyzer.rs b/pil-analyzer/src/pil_analyzer.rs index 3194547cd..d271c1b39 100644 --- a/pil-analyzer/src/pil_analyzer.rs +++ b/pil-analyzer/src/pil_analyzer.rs @@ -416,20 +416,22 @@ impl PILAnalyzer { } fn handle_namespace(&mut self, name: SymbolPath, degree: Option) { - if let Some(degree) = degree { - let degree = ExpressionProcessor::new(self.driver(), &Default::default()) - .process_expression(degree); + self.polynomial_degree = degree + .map(|degree| { + ExpressionProcessor::new(self.driver(), &Default::default()) + .process_expression(degree) + }) // TODO we should maybe implement a separate evaluator that is able to run before type checking // and is field-independent (only uses integers)? - let namespace_degree: u64 = u64::try_from( - evaluator::evaluate_expression::(°ree, &self.definitions) - .unwrap() - .try_to_integer() - .unwrap(), - ) - .unwrap(); - self.polynomial_degree = Some(namespace_degree); - } + .map(|degree| { + u64::try_from( + evaluator::evaluate_expression::(°ree, &self.definitions) + .unwrap() + .try_to_integer() + .unwrap(), + ) + .unwrap() + }); self.current_namespace = AbsoluteSymbolPath::default().join(name); } From ebb1561b43672ef510e9a9a6e6b7a99aabfa296c Mon Sep 17 00:00:00 2001 From: schaeff Date: Tue, 2 Jul 2024 20:10:13 +0200 Subject: [PATCH 38/39] fix condenser --- pil-analyzer/src/condenser.rs | 19 ++++++++++--------- pil-analyzer/src/pil_analyzer.rs | 1 - 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pil-analyzer/src/condenser.rs b/pil-analyzer/src/condenser.rs index 63b3dd86f..5adb15b4b 100644 --- a/pil-analyzer/src/condenser.rs +++ b/pil-analyzer/src/condenser.rs @@ -30,14 +30,13 @@ use crate::{ }; pub fn condense( - degree: Option, mut definitions: HashMap)>, mut public_declarations: HashMap, identities: &[Identity>], source_order: Vec, auto_added_symbols: HashSet, ) -> Analyzed { - let mut condenser = Condenser::new(&definitions, degree); + let mut condenser = Condenser::new(&definitions); // Counter needed to re-assign identity IDs. let mut counters = Counters::default(); @@ -53,7 +52,7 @@ pub fn condense( let mut namespace = AbsoluteSymbolPath::default().join(SymbolPath::from_str(name).unwrap()); namespace.pop(); - condenser.set_namespace(namespace); + condenser.set_namespace_and_degree(namespace, definitions[name].0.degree); } let statement = match s { StatementIdentifier::Identity(index) => { @@ -207,10 +206,7 @@ impl IdentityWithoutID { } impl<'a, T: FieldElement> Condenser<'a, T> { - pub fn new( - symbols: &'a HashMap)>, - degree: Option, - ) -> Self { + pub fn new(symbols: &'a HashMap)>) -> Self { let next_witness_id = symbols .values() .filter_map(|(sym, _)| match sym.kind { @@ -222,8 +218,8 @@ impl<'a, T: FieldElement> Condenser<'a, T> { .max() .unwrap_or_default(); Self { - degree, symbols, + degree: None, symbol_values: Default::default(), namespace: Default::default(), next_witness_id, @@ -266,8 +262,13 @@ impl<'a, T: FieldElement> Condenser<'a, T> { } /// Sets the current namespace which will be used for newly generated witness columns. - pub fn set_namespace(&mut self, namespace: AbsoluteSymbolPath) { + pub fn set_namespace_and_degree( + &mut self, + namespace: AbsoluteSymbolPath, + degree: Option, + ) { self.namespace = namespace; + self.degree = degree; } /// Returns the witness columns generated since the last call to this function. diff --git a/pil-analyzer/src/pil_analyzer.rs b/pil-analyzer/src/pil_analyzer.rs index d271c1b39..d70b0ba9c 100644 --- a/pil-analyzer/src/pil_analyzer.rs +++ b/pil-analyzer/src/pil_analyzer.rs @@ -328,7 +328,6 @@ impl PILAnalyzer { pub fn condense(self) -> Analyzed { condenser::condense( - self.polynomial_degree, self.definitions, self.public_declarations, &self.identities, From 3a0ccbe7f1721a02db916f2130a5c3d58884cb0e Mon Sep 17 00:00:00 2001 From: schaeff Date: Thu, 4 Jul 2024 14:10:38 +0200 Subject: [PATCH 39/39] fix empty pil test in p3 --- plonky3/src/circuit_builder.rs | 55 ++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/plonky3/src/circuit_builder.rs b/plonky3/src/circuit_builder.rs index e2922589d..8622c90d0 100644 --- a/plonky3/src/circuit_builder.rs +++ b/plonky3/src/circuit_builder.rs @@ -54,28 +54,39 @@ impl<'a, T: FieldElement> PowdrCircuit<'a, T> { let witness = self.witness().iter(); let publics = self.get_publics().into_iter(); - let len = self.analyzed.degree(); - - // for each row, get the value of each column - let values = (0..len) - .flat_map(move |i| { - // witness values - witness.clone().map(move |(_, v)| v[i as usize]).chain( - // publics rows: decrementor | inverse | selector - publics.clone().flat_map(move |(_, _, row_id)| { - let decr = T::from(row_id as u64) - T::from(i); - let inv_decr = if i as usize == row_id { - T::zero() - } else { - T::one() / decr - }; - let s = T::from(i as usize == row_id); - [decr, inv_decr, s] - }), - ) - }) - .map(cast_to_goldilocks) - .collect(); + let degrees = self.analyzed.degrees(); + + let values = match degrees.len() { + 1 => { + // for each row, get the value of each column + let degree = degrees.iter().next().unwrap(); + (0..*degree) + .flat_map(move |i| { + // witness values + witness.clone().map(move |(_, v)| v[i as usize]).chain( + // publics rows: decrementor | inverse | selector + publics.clone().flat_map(move |(_, _, row_id)| { + let decr = T::from(row_id as u64) - T::from(i); + let inv_decr = if i as usize == row_id { + T::zero() + } else { + T::one() / decr + }; + let s = T::from(i as usize == row_id); + [decr, inv_decr, s] + }), + ) + }) + .map(cast_to_goldilocks) + .collect() + } + 0 => { + // in this case, there are no columns, so there are no values + assert!(witness.clone().next().is_none()); + vec![] + } + _ => unreachable!(), + }; RowMajorMatrix::new(values, self.width()) } }