github/powdr
1
1
Fork 0
mirror of https://github.com/powdr-labs/powdr.git synced 2026-01-09 14:48:16 -05:00
Code Issues Packages Projects Releases Wiki Activity

zkEVM prover backend (#1211)

Browse Source 
Similar to #1193, but in here I am just interested in having it working
end-to-end, at least for a few cases, so that everybody can try it and
build upon.

<!--

Please follow this protocol when creating or reviewing PRs in this
repository:

- Leave the PR as draft until review is required.
- When reviewing a PR, every reviewer should assign themselves as soon
as they
start, so that other reviewers know the PR is covered. You should not be
discouraged from reviewing a PR with assignees, but you will know it is
not
  strictly needed.
- Unless the PR is very small, help the reviewers by not making forced
pushes, so
that GitHub properly tracks what has been changed since the last review;
use
  "merge" instead of "rebase". It can be squashed after approval.
- Once the comments have been addressed, explicitly let the reviewer
know the PR
  is ready again.

-->

---------

Co-authored-by: Leo <leo@powdrlabs.com>
This commit is contained in:
Lucas Clemente Vella
2024-04-18 10:16:09 +01:00
committed by GitHub
parent b2e0beef36
commit ac9dcfda87
27 changed files with 507 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/build-cache.yml vendored
View File

@@ -17,6 +17,8 @@ jobs:
submodules: recursive
- name: Install Rust toolchain 1.77 (with clippy and rustfmt)
run: rustup toolchain install 1.77-x86_64-unknown-linux-gnu && rustup component add clippy --toolchain 1.77-x86_64-unknown-linux-gnu && rustup component add rustfmt --toolchain 1.77-x86_64-unknown-linux-gnu
- name: Install EStarkPolygon prover dependencies
run: sudo apt-get install -y nlohmann-json3-dev libpqxx-dev nasm
- name: Lint
run: cargo clippy --all --all-targets --all-features --profile pr-tests -- -D warnings
- name: Lint

2
.github/workflows/nightly-tests.yml vendored
View File

@@ -72,6 +72,8 @@ jobs:
run: rustup target add riscv32imac-unknown-none-elf --toolchain nightly-2024-02-01-x86_64-unknown-linux-gnu
- name: Install stdlib
run: rustup component add rust-src --toolchain nightly-2024-02-01-x86_64-unknown-linux-gnu
- name: Install EStarkPolygon prover dependencies
run: sudo apt-get install -y nlohmann-json3-dev libpqxx-dev nasm
- name: Install pilcom
run: git clone https://github.com/0xPolygonHermez/pilcom.git && cd pilcom && npm install
- name: Check without Halo2

55
.github/workflows/pr-tests.yml vendored
View File

@@ -34,6 +34,8 @@ jobs:
${{ runner.os }}-cargo-pr-tests-
- name: Install Rust toolchain 1.77 (with clippy and rustfmt)
run: rustup toolchain install 1.77-x86_64-unknown-linux-gnu && rustup component add clippy --toolchain 1.77-x86_64-unknown-linux-gnu && rustup component add rustfmt --toolchain 1.77-x86_64-unknown-linux-gnu
- name: Install EStarkPolygon prover dependencies
run: sudo apt-get install -y nlohmann-json3-dev libpqxx-dev nasm
- name: Lint no default features
run: cargo clippy --all --all-targets --no-default-features --profile pr-tests -- -D warnings
- name: Lint all features
@@ -43,13 +45,17 @@ jobs:
- name: Build
run: cargo build --all-targets --all --all-features --profile pr-tests
- uses: taiki-e/install-action@nextest
- name: Archive EStarkPolygon prover built dependencies
run: tar --zstd -cf pil-stark-prover-deps.tar.zst target/pr-tests/build/pil-stark-prover-*/out
- name: Create tests archive
run: cargo nextest archive --archive-file tests.tar.zst --cargo-profile pr-tests --workspace --all-features
- name: Upload tests archive
- name: Upload build artifacts
uses: actions/upload-artifact@v2
with:
name: tests_archive
path: tests.tar.zst
path: |
tests.tar.zst
pil-stark-prover-deps.tar.zst
test_quick:
needs: build
@@ -81,10 +87,49 @@ jobs:
run: git clone https://github.com/0xPolygonHermez/pilcom.git && cd pilcom && npm install
- uses: taiki-e/install-action@nextest
- name: Run default tests
run: PILCOM=$(pwd)/pilcom/ cargo nextest run --archive-file tests.tar.zst --verbose
run: cargo nextest run --archive-file tests.tar.zst --verbose
env:
PILCOM: ${{ github.workspace }}/pilcom/
- name: Run examples
run: cargo run --example hello_world
test_estark_polygon:
needs: build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
submodules: recursive
- name: Download build artifacts
uses: actions/download-artifact@v2
with:
name: tests_archive
- name: ⚡ Cache nodejs
uses: actions/cache@v3
with:
path: |
~/pilcom/node_modules
key: ${{ runner.os }}-pilcom-node-modules
- name: Install Rust toolchain 1.77 (with clippy and rustfmt)
run: rustup toolchain install 1.77-x86_64-unknown-linux-gnu && rustup component add clippy --toolchain 1.77-x86_64-unknown-linux-gnu && rustup component add rustfmt --toolchain 1.77-x86_64-unknown-linux-gnu
- name: Install nightly
run: rustup toolchain install nightly-2024-02-01-x86_64-unknown-linux-gnu
- name: Install riscv target
run: rustup target add riscv32imac-unknown-none-elf --toolchain nightly-2024-02-01-x86_64-unknown-linux-gnu
- name: Install stdlib
run: rustup component add rust-src --toolchain nightly-2024-02-01-x86_64-unknown-linux-gnu
- name: Install pilcom
run: git clone https://github.com/0xPolygonHermez/pilcom.git && cd pilcom && npm install
- name: Install EStarkPolygon prover system dependency
run: sudo apt-get install -y nlohmann-json3-dev
- uses: taiki-e/install-action@nextest
- name: Unpack EStarkPolygon built dependencies
run: tar --zstd -xf pil-stark-prover-deps.tar.zst
- name: Run EStark Polygon test
run: cargo nextest run --archive-file tests.tar.zst --verbose --run-ignored=ignored-only --no-capture -E "test(=test_vec_median_estark_polygon)"
env:
PILCOM: ${{ github.workspace }}/pilcom/
test_slow:
strategy:
matrix:
@@ -127,5 +172,7 @@ jobs:
elif [[ "${{ matrix.test }}" == "subset2" ]]; then
TESTS="test(=keccak) | test(=vec_median) | test(=instruction_tests::addi) | test(=many_chunks) | test(=sum_serde)"
fi
PILCOM=$(pwd)/pilcom/ cargo nextest run --archive-file tests.tar.zst --verbose --run-ignored=ignored-only --no-capture -E "$TESTS"
cargo nextest run --archive-file tests.tar.zst --verbose --run-ignored=ignored-only --no-capture -E "$TESTS"
shell: bash
env:
PILCOM: ${{ github.workspace }}/pilcom/

2
README.md
View File

@@ -28,7 +28,7 @@ which is compiled to RISCV, then to powdr-asm and finally to powdr-PIL.
*powdr*-pil can be used to generate proofs using multiple backends, such as:
- Halo2: via [polyexen](https://github.com/Dhole/polyexen) and [snark-verifer](https://github.com/privacy-scaling-explorations/snark-verifier/)
- eSTARK: via [Eigen's starky](https://github.com/0xEigenLabs/eigen-zkvm/)
- eSTARK: via [Eigen's starky](https://github.com/0xEigenLabs/eigen-zkvm/) or via [our fork of Polygon's zkevm-prover](https://github.com/powdr-labs/zkevm-prover).
- SuperNova: ongoing work (https://github.com/powdr-labs/powdr/pull/453)
All stages are fully automatic, which means you do not need to write any

5
backend/Cargo.toml
View File

@@ -9,6 +9,7 @@ repository = { workspace = true }
[features]
halo2 = ["dep:powdr-halo2"]
estark-polygon = ["dep:pil-stark-prover"]
[dependencies]
powdr-ast = { path = "../ast" }
@@ -19,12 +20,14 @@ powdr-executor = { path = "../executor" }
strum = { version = "0.24.1", features = ["derive"] }
log = "0.4.17"
serde = "1.0"
serde_json = "1.0"
thiserror = "1.0.43"
starky = { git = "https://github.com/0xEigenLabs/eigen-zkvm.git", rev = "59d2152" }
pil-stark-prover = { git = "https://github.com/powdr-labs/pil-stark-prover.git", rev = "d119f0b402c37be160", optional = true }
mktemp = "0.5.0"
[dev-dependencies]
mktemp = "0.5.0"
test-log = "0.2.12"
env_logger = "0.10.0"
pretty_assertions = "1.4.0"

0
backend/src/pilstark/json_exporter/expression_counter.rs → backend/src/estark/json_exporter/expression_counter.rs
View File

0
backend/src/pilstark/json_exporter/mod.rs → backend/src/estark/json_exporter/mod.rs
View File

231
backend/src/estark/mod.rs Normal file
View File

@@ -0,0 +1,231 @@
mod json_exporter;
#[cfg(feature = "estark-polygon")]
pub mod polygon_wrapper;
pub mod starky_wrapper;
use std::{
fs::File,
io::{self, BufWriter, Write},
iter::{once, repeat},
path::{Path, PathBuf},
};
use crate::{Backend, BackendFactory, Error, Proof};
use powdr_ast::analyzed::Analyzed;
use powdr_executor::witgen::WitgenCallback;
use powdr_number::{write_polys_file, DegreeType, FieldElement};
use serde::Serialize;
use starky::types::{StarkStruct, Step, PIL};
fn create_stark_struct(degree: DegreeType) -> StarkStruct {
assert!(degree > 1);
let n_bits = (DegreeType::BITS - (degree - 1).leading_zeros()) as usize;
let n_bits_ext = n_bits + 1;
let steps = (2..=n_bits_ext)
.rev()
.step_by(4)
.map(|b| Step { nBits: b })
.collect();
StarkStruct {
nBits: n_bits,
nBitsExt: n_bits_ext,
nQueries: 2,
verificationHashType: "GL".to_owned(),
steps,
}
}
/// eStark provers require a fixed column with the equivalent semantics to
/// Polygon zkEVM's `L1` column. Powdr generated PIL will always have
/// `main.first_step`, but directly given PIL may not have it. This is a fixup
/// to inject such column if it doesn't exist.
///
/// TODO Improve how this is done.
fn first_step_fixup<'a, F: FieldElement>(
pil: &'a Analyzed<F>,
fixed: &'a [(String, Vec<F>)],
) -> (PIL, Vec<(String, Vec<F>)>) {
let degree = pil.degree();
let mut pil: PIL = json_exporter::export(pil);
let mut fixed = fixed.to_vec();
if !fixed.iter().any(|(k, _)| k == "main.first_step") {
use starky::types::Reference;
pil.nConstants += 1;
pil.references.insert(
"main.first_step".to_string(),
Reference {
polType: None,
type_: "constP".to_string(),
id: fixed.len(),
polDeg: degree as usize,
isArray: false,
elementType: None,
len: None,
},
);
fixed.push((
"main.first_step".to_string(),
once(F::one())
.chain(repeat(F::zero()))
.take(degree as usize)
.collect(),
));
}
(pil, fixed)
}
struct EStarkFilesCommon<'a, F: FieldElement> {
degree: DegreeType,
pil: PIL,
fixed: Vec<(String, Vec<F>)>,
output_dir: Option<&'a Path>,
}
fn buffered_write_file<R>(
path: &Path,
do_write: impl FnOnce(&mut BufWriter<File>) -> R,
) -> Result<R, io::Error> {
let mut writer = BufWriter::new(File::create(path)?);
let result = do_write(&mut writer);
writer.flush()?;
Ok(result)
}
fn write_json_file<T: ?Sized + Serialize>(path: &Path, data: &T) -> Result<(), Error> {
buffered_write_file(path, |writer| {
serde_json::to_writer(writer, data).map_err(|e| e.to_string())
})??;
Ok(())
}
fn write_polys_bin<F: FieldElement>(
path: &Path,
constants: &[(String, Vec<F>)],
) -> Result<(), Error> {
buffered_write_file(path, |writer| write_polys_file(writer, constants))??;
Ok(())
}
impl<'a, F: FieldElement> EStarkFilesCommon<'a, F> {
fn create(
analyzed: &'a Analyzed<F>,
fixed: &'a [(String, Vec<F>)],
output_dir: Option<&'a Path>,
setup: Option<&mut dyn std::io::Read>,
verification_key: Option<&mut dyn std::io::Read>,
) -> Result<Self, Error> {
if setup.is_some() {
return Err(Error::NoSetupAvailable);
}
if verification_key.is_some() {
return Err(Error::NoVerificationAvailable);
}
// Pre-process the PIL and fixed columns.
let (pil, fixed) = first_step_fixup(analyzed, fixed);
Ok(EStarkFilesCommon {
degree: analyzed.degree(),
pil,
fixed,
output_dir,
})
}
}
struct ProverInputFilePaths {
constants: PathBuf,
commits: PathBuf,
stark_struct: PathBuf,
contraints: PathBuf,
}
impl<'a, F: FieldElement> EStarkFilesCommon<'a, F> {
/// Write the files in the EStark Polygon format.
fn write_files(
&self,
output_dir: &Path,
witness: &[(String, Vec<F>)],
) -> Result<ProverInputFilePaths, Error> {
let paths = ProverInputFilePaths {
constants: output_dir.join("constants.bin"),
commits: output_dir.join("commits.bin"),
stark_struct: output_dir.join("starkstruct.json"),
contraints: output_dir.join("constraints.json"),
};
// Write the constants.
log::info!("Writing {}.", paths.constants.to_string_lossy());
write_polys_bin(&paths.constants, &self.fixed)?;
// Write the commits.
log::info!("Writing {}.", paths.commits.to_string_lossy());
write_polys_bin(&paths.commits, witness)?;
// Write the stark struct JSON.
log::info!("Writing {}.", paths.stark_struct.to_string_lossy());
write_json_file(&paths.stark_struct, &create_stark_struct(self.degree))?;
// Write the constraints in JSON.
log::info!("Writing {}.", paths.contraints.to_string_lossy());
write_json_file(&paths.contraints, &self.pil)?;
Ok(paths)
}
}
pub struct DumpFactory;
impl<F: FieldElement> BackendFactory<F> for DumpFactory {
fn create<'a>(
&self,
analyzed: &'a Analyzed<F>,
fixed: &'a [(String, Vec<F>)],
output_dir: Option<&'a Path>,
setup: Option<&mut dyn std::io::Read>,
verification_key: Option<&mut dyn std::io::Read>,
) -> Result<Box<dyn crate::Backend<'a, F> + 'a>, Error> {
Ok(Box::new(DumpBackend(EStarkFilesCommon::create(
analyzed,
fixed,
output_dir,
setup,
verification_key,
)?)))
}
}
/// A backend that just dumps the files to the output directory.
struct DumpBackend<'a, F: FieldElement>(EStarkFilesCommon<'a, F>);
impl<'a, F: FieldElement> Backend<'a, F> for DumpBackend<'a, F> {
fn prove(
&self,
witness: &[(String, Vec<F>)],
prev_proof: Option<Proof>,
// TODO: Implement challenges
_witgen_callback: WitgenCallback<F>,
) -> Result<Proof, Error> {
if prev_proof.is_some() {
return Err(Error::NoAggregationAvailable);
}
let output_dir = self
.0
.output_dir
.ok_or(Error::BackendError("output_dir is None".to_owned()))?;
self.0.write_files(output_dir, witness)?;
Ok(Vec::new())
}
}

83
backend/src/estark/polygon_wrapper.rs Normal file
View File

@@ -0,0 +1,83 @@
use std::{fs, path::Path};
use powdr_ast::analyzed::Analyzed;
use powdr_executor::witgen::WitgenCallback;
use powdr_number::FieldElement;
use crate::{Backend, BackendFactory, Error, Proof};
use super::EStarkFilesCommon;
pub struct Factory;
impl<F: FieldElement> BackendFactory<F> for Factory {
fn create<'a>(
&self,
analyzed: &'a Analyzed<F>,
fixed: &'a [(String, Vec<F>)],
output_dir: Option<&'a Path>,
setup: Option<&mut dyn std::io::Read>,
verification_key: Option<&mut dyn std::io::Read>,
) -> Result<Box<dyn crate::Backend<'a, F> + 'a>, Error> {
Ok(Box::new(PolygonBackend(EStarkFilesCommon::create(
analyzed,
fixed,
output_dir,
setup,
verification_key,
)?)))
}
}
struct PolygonBackend<'a, F: FieldElement>(EStarkFilesCommon<'a, F>);
// TODO: make both eStark backends interchangeable, from user perspective.
// TODO: implement the other Backend trait methods.
impl<'a, F: FieldElement> Backend<'a, F> for PolygonBackend<'a, F> {
fn prove(
&self,
witness: &[(String, Vec<F>)],
prev_proof: Option<Proof>,
// TODO: Implement challenges
_witgen_callback: WitgenCallback<F>,
) -> Result<Proof, Error> {
if prev_proof.is_some() {
return Err(Error::NoAggregationAvailable);
}
let tmp_dir;
let output_dir = if let Some(output_dir) = self.0.output_dir {
output_dir
} else {
tmp_dir = mktemp::Temp::new_dir()?;
tmp_dir.as_path()
};
let input_paths = self.0.write_files(output_dir, witness)?;
// Generate the proof.
let proof_paths = pil_stark_prover::generate_proof(
&input_paths.contraints,
&input_paths.stark_struct,
&input_paths.constants,
&input_paths.commits,
output_dir,
)
.map_err(|e| Error::BackendError(e.to_string()))?;
// Sanity check: verify the proof.
let publics_path = output_dir.join("publics.json");
// TODO: properly handle publics
fs::write(&publics_path, "[]")?;
pil_stark_prover::verify_proof(
&proof_paths.verification_key_json,
&proof_paths.starkinfo_json,
&proof_paths.proof_json,
&publics_path,
)
.map_err(|e| Error::BackendError(e.to_string()))?;
// Read the proof.
Ok(fs::read(&proof_paths.proof_json)?)
}
}

77
backend/src/pilstark/estark.rs → backend/src/estark/starky_wrapper.rs
View File

@@ -1,11 +1,10 @@
use std::io;
use std::iter::{once, repeat};
use std::time::Instant;
use crate::{pilstark, Backend, BackendFactory, Error};
use crate::{Backend, BackendFactory, Error};
use powdr_ast::analyzed::Analyzed;
use powdr_executor::witgen::WitgenCallback;
use powdr_number::{DegreeType, FieldElement, GoldilocksField, LargeInt};
use powdr_number::{FieldElement, GoldilocksField, LargeInt};
use starky::{
merklehash::MerkleTreeGL,
@@ -15,12 +14,14 @@ use starky::{
stark_verify::stark_verify,
traits::FieldExtension,
transcript::TranscriptGL,
types::{StarkStruct, Step, PIL},
types::{StarkStruct, PIL},
};
pub struct EStarkFactory;
use super::{create_stark_struct, first_step_fixup};
impl<F: FieldElement> BackendFactory<F> for EStarkFactory {
pub struct Factory;
impl<F: FieldElement> BackendFactory<F> for Factory {
fn create<'a>(
&self,
pil: &'a Analyzed<F>,
@@ -38,26 +39,9 @@ impl<F: FieldElement> BackendFactory<F> for EStarkFactory {
return Err(Error::NoSetupAvailable);
}
let degree = pil.degree();
assert!(degree > 1);
let n_bits = (DegreeType::BITS - (degree - 1).leading_zeros()) as usize;
let n_bits_ext = n_bits + 1;
let params = create_stark_struct(pil.degree());
let steps = (2..=n_bits_ext)
.rev()
.step_by(4)
.map(|b| Step { nBits: b })
.collect();
let params = StarkStruct {
nBits: n_bits,
nBitsExt: n_bits_ext,
nQueries: 2,
verificationHashType: "GL".to_owned(),
steps,
};
let (pil_json, fixed) = pil_json(pil, fixed);
let (pil_json, fixed) = first_step_fixup(pil, fixed);
let const_pols = to_starky_pols_array(&fixed, &pil_json, PolKind::Constant);
let setup = if let Some(vkey) = verification_key {
@@ -75,49 +59,6 @@ impl<F: FieldElement> BackendFactory<F> for EStarkFactory {
}
}
fn pil_json<'a, F: FieldElement>(
pil: &'a Analyzed<F>,
fixed: &'a [(String, Vec<F>)],
) -> (PIL, Vec<(String, Vec<F>)>) {
let degree = pil.degree();
let mut pil: PIL = pilstark::json_exporter::export(pil);
// TODO starky requires a fixed column with the equivalent
// semantics to Polygon zkEVM's `L1` column.
// It takes the name of that column via the API.
// Powdr generated PIL will always have `main.first_step`,
// but directly given PIL may not have it.
// This is a hack to inject such column if it doesn't exist.
// It should be eventually improved.
let mut fixed = fixed.to_vec();
if !fixed.iter().any(|(k, _)| k == "main.first_step") {
use starky::types::Reference;
pil.nConstants += 1;
pil.references.insert(
"main.first_step".to_string(),
Reference {
polType: None,
type_: "constP".to_string(),
id: fixed.len(),
polDeg: degree as usize,
isArray: false,
elementType: None,
len: None,
},
);
fixed.push((
"main.first_step".to_string(),
once(F::one())
.chain(repeat(F::zero()))
.take(degree as usize)
.collect(),
));
}
(pil, fixed)
}
fn create_stark_setup(
mut pil: PIL,
const_pols: &PolsArray,

27
backend/src/lib.rs
View File

@@ -1,8 +1,8 @@
#![deny(clippy::print_stdout)]
mod estark;
#[cfg(feature = "halo2")]
mod halo2_impl;
mod pilstark;
use powdr_ast::analyzed::Analyzed;
use powdr_executor::witgen::WitgenCallback;
@@ -18,10 +18,13 @@ pub enum BackendType {
#[cfg(feature = "halo2")]
#[strum(serialize = "halo2-mock")]
Halo2Mock,
#[strum(serialize = "estark")]
EStark,
#[strum(serialize = "pil-stark-cli")]
PilStarkCli,
#[cfg(feature = "estark-polygon")]
#[strum(serialize = "estark-polygon")]
EStarkPolygon,
#[strum(serialize = "estark-starky")]
EStarkStarky,
#[strum(serialize = "estark-dump")]
EStarkDump,
}
impl BackendType {
@@ -30,16 +33,22 @@ impl BackendType {
const HALO2_FACTORY: halo2_impl::Halo2ProverFactory = halo2_impl::Halo2ProverFactory;
#[cfg(feature = "halo2")]
const HALO2_MOCK_FACTORY: halo2_impl::Halo2MockFactory = halo2_impl::Halo2MockFactory;
const ESTARK_FACTORY: pilstark::estark::EStarkFactory = pilstark::estark::EStarkFactory;
const PIL_STARK_CLI_FACTORY: pilstark::PilStarkCliFactory = pilstark::PilStarkCliFactory;
#[cfg(feature = "estark-polygon")]
const ESTARK_POLYGON_FACTORY: estark::polygon_wrapper::Factory =
estark::polygon_wrapper::Factory;
const ESTARK_STARKY_FACTORY: estark::starky_wrapper::Factory =
estark::starky_wrapper::Factory;
const ESTARK_DUMP_FACTORY: estark::DumpFactory = estark::DumpFactory;
match self {
#[cfg(feature = "halo2")]
BackendType::Halo2 => &HALO2_FACTORY,
#[cfg(feature = "halo2")]
BackendType::Halo2Mock => &HALO2_MOCK_FACTORY,
BackendType::EStark => &ESTARK_FACTORY,
BackendType::PilStarkCli => &PIL_STARK_CLI_FACTORY,
#[cfg(feature = "estark-polygon")]
BackendType::EStarkPolygon => &ESTARK_POLYGON_FACTORY,
BackendType::EStarkStarky => &ESTARK_STARKY_FACTORY,
BackendType::EStarkDump => &ESTARK_DUMP_FACTORY,
}
}
}

72
backend/src/pilstark/mod.rs
View File

@@ -1,72 +0,0 @@
pub mod estark;
mod json_exporter;
use std::{
fs::File,
io::{BufWriter, Write},
path::Path,
};
use crate::{Backend, BackendFactory, Error, Proof};
use powdr_ast::analyzed::Analyzed;
use powdr_executor::witgen::WitgenCallback;
use powdr_number::FieldElement;
pub struct PilStarkCliFactory;
impl<F: FieldElement> BackendFactory<F> for PilStarkCliFactory {
fn create<'a>(
&self,
analyzed: &'a Analyzed<F>,
_fixed: &'a [(String, Vec<F>)],
output_dir: Option<&'a Path>,
setup: Option<&mut dyn std::io::Read>,
verification_key: Option<&mut dyn std::io::Read>,
) -> Result<Box<dyn crate::Backend<'a, F> + 'a>, Error> {
if setup.is_some() {
return Err(Error::NoSetupAvailable);
}
if verification_key.is_some() {
return Err(Error::NoVerificationAvailable);
}
Ok(Box::new(PilStarkCli {
analyzed,
output_dir,
}))
}
}
pub struct PilStarkCli<'a, F: FieldElement> {
analyzed: &'a Analyzed<F>,
output_dir: Option<&'a Path>,
}
impl<'a, F: FieldElement> Backend<'a, F> for PilStarkCli<'a, F> {
fn prove(
&self,
_witness: &[(String, Vec<F>)],
prev_proof: Option<Proof>,
// TODO: Implement challenges
_witgen_callback: WitgenCallback<F>,
) -> Result<Proof, Error> {
if prev_proof.is_some() {
return Err(Error::NoAggregationAvailable);
}
// Write the constraints in the format expected by the prover-cpp
if let Some(output_dir) = self.output_dir {
let path = output_dir.join("constraints.json");
let mut writer = BufWriter::new(File::create(path)?);
serde_json::to_writer(&mut writer, &json_exporter::export(self.analyzed))
.map_err(|e| e.to_string())?;
writer.flush()?;
} else {
// If we were going to call the prover-cpp, we could write the
// constraints.json to a temporary directory in case no output_dir
// is provided.
}
// TODO: actually use prover-cpp to generate the proof
Ok(Vec::new())
}
}

13
book/src/installation.md
View File

@@ -10,6 +10,19 @@ The easiest way to install both is with [`rustup.rs`](https://rustup.rs/).
On Windows, you will also need a recent version of [Visual Studio](https://visualstudio.microsoft.com/downloads/),
installed with the "Desktop Development With C++" Workloads option.
If you want to enable `estark-polygon` feature, you also need the following
runtime dependencies installed on the system:
- `gcc`
- `nlohmann-json3-dev`
Yes, gcc and headers at runtime, this is not a typo.
You will also need the following build time dependencies:
- `make`
- `pkg-config`
- `libpqxx-dev`
- `nasm`
## Building
Using a single Cargo command (enable the halo2 backend to use it with the cli):

4
cli/Cargo.toml
View File

@@ -11,6 +11,7 @@ default-run = "powdr"
[features]
default = [] # halo2 is disabled by default
halo2 = ["powdr-backend/halo2", "powdr-pipeline/halo2"]
estark-polygon = ["powdr-backend/estark-polygon", "powdr-pipeline/estark-polygon", "powdr-riscv/estark-polygon"]
[dependencies]
powdr-backend = { path = "../backend" }
@@ -30,6 +31,9 @@ clap-markdown = "0.1.3"
[dev-dependencies]
tempfile = "3.6"
test-log = "0.2.12"
env_logger = "0.10.0"
[[bin]]
name = "powdr"
path = "src/main.rs"

3
cli/src/main.rs
View File

@@ -912,6 +912,7 @@ fn optimize_and_output<T: FieldElement>(file: &str) {
mod test {
use crate::{run_command, Commands, CsvRenderModeCLI, FieldArgument};
use powdr_backend::BackendType;
use test_log::test;
#[test]
fn simple_sum() {
@@ -930,7 +931,7 @@ mod test {
inputs: "3,2,1,2".into(),
force: false,
pilo: false,
prove_with: Some(BackendType::PilStarkCli),
prove_with: Some(BackendType::EStarkDump),
export_csv: true,
csv_mode: CsvRenderModeCLI::Hex,
just_execute: false,

15
number/src/serialize.rs
View File

@@ -1,4 +1,4 @@
use std::io::{Read, Write};
use std::io::{self, Read, Write};
use ark_serialize::{CanonicalDeserialize, CanonicalSerialize, Compress, Validate};
use csv::{Reader, Writer};
@@ -89,11 +89,14 @@ fn ceil_div(num: usize, div: usize) -> usize {
(num + div - 1) / div
}
pub fn write_polys_file<T: FieldElement>(file: &mut impl Write, polys: &[(String, Vec<T>)]) {
pub fn write_polys_file<T: FieldElement>(
file: &mut impl Write,
polys: &[(String, Vec<T>)],
) -> Result<(), io::Error> {
let width = ceil_div(T::BITS as usize, 64) * 8;
if polys.is_empty() {
return;
return Ok(());
}
// TODO maybe the witness should have a proper type that
@@ -107,9 +110,11 @@ pub fn write_polys_file<T: FieldElement>(file: &mut impl Write, polys: &[(String
for (_name, constant) in polys {
let bytes = constant[i].to_bytes_le();
assert_eq!(bytes.len(), width);
file.write_all(&bytes).unwrap();
file.write_all(&bytes)?;
}
}
Ok(())
}
pub fn read_polys_file<T: FieldElement>(
@@ -188,7 +193,7 @@ mod tests {
let (polys, degree) = test_polys();
write_polys_file(&mut buf, &polys);
write_polys_file(&mut buf, &polys).unwrap();
let (read_polys, read_degree) = read_polys_file::<Bn254Field>(
&mut Cursor::new(buf),
&["a".to_string(), "b".to_string()],

1
pipeline/Cargo.toml
View File

@@ -9,6 +9,7 @@ repository = { workspace = true }
[features]
halo2 = ["powdr-backend/halo2"]
estark-polygon = ["powdr-backend/estark-polygon"]
[dependencies]
powdr-airgen = { path = "../airgen" }

24
pipeline/src/pipeline.rs
View File

@@ -2,7 +2,7 @@ use std::{
borrow::Borrow,
fmt::Display,
fs,
io::{self, BufReader, BufWriter},
io::{self, BufReader},
marker::Send,
path::{Path, PathBuf},
rc::Rc,
@@ -24,12 +24,12 @@ use powdr_executor::{
chain_callbacks, unused_query_callback, QueryCallback, WitgenCallback, WitnessGenerator,
},
};
use powdr_number::{write_polys_csv_file, write_polys_file, CsvRenderMode, FieldElement};
use powdr_number::{write_polys_csv_file, CsvRenderMode, FieldElement};
use powdr_schemas::SerializedAnalyzed;
use crate::{
inputs_to_query_callback, serde_data_to_query_callback,
util::{try_read_poly_set, write_or_panic, FixedPolySet, WitnessPolySet},
util::{try_read_poly_set, FixedPolySet, WitnessPolySet},
};
type Columns<T> = Vec<(String, Vec<T>)>;
@@ -373,7 +373,7 @@ impl<T: FieldElement> Pipeline<T> {
pub fn read_constants(mut self, directory: &Path) -> Self {
let pil = self.compute_optimized_pil().unwrap();
let fixed = try_read_poly_set::<FixedPolySet, T>(&pil, directory, self.name())
let fixed = try_read_poly_set::<FixedPolySet, T>(&pil, directory)
.map(|(fixed, degree_fixed)| {
assert_eq!(pil.degree.unwrap(), degree_fixed);
fixed
@@ -393,7 +393,7 @@ impl<T: FieldElement> Pipeline<T> {
pub fn read_witness(mut self, directory: &Path) -> Self {
let pil = self.compute_optimized_pil().unwrap();
let witness = try_read_poly_set::<WitnessPolySet, T>(&pil, directory, self.name())
let witness = try_read_poly_set::<WitnessPolySet, T>(&pil, directory)
.map(|(witness, degree_witness)| {
assert_eq!(pil.degree.unwrap(), degree_witness);
witness
@@ -487,24 +487,11 @@ impl<T: FieldElement> Pipeline<T> {
Ok(())
}
fn maybe_write_constants(&self, constants: &[(String, Vec<T>)]) -> Result<(), Vec<String>> {
if let Some(path) = self.path_if_should_write(|name| format!("{name}_constants.bin"))? {
let writer = BufWriter::new(fs::File::create(path).unwrap());
write_or_panic(writer, |writer| write_polys_file(writer, constants));
}
Ok(())
}
fn maybe_write_witness(
&self,
fixed: &[(String, Vec<T>)],
witness: &[(String, Vec<T>)],
) -> Result<(), Vec<String>> {
if let Some(path) = self.path_if_should_write(|name| format!("{name}_commits.bin"))? {
let file = BufWriter::new(fs::File::create(path).unwrap());
write_or_panic(file, |file| write_polys_file(file, witness));
}
if self.arguments.export_witness_csv {
if let Some(path) = self.path_if_should_write(|name| format!("{name}_columns.csv"))? {
let columns = fixed.iter().chain(witness.iter()).collect::<Vec<_>>();
@@ -792,7 +779,6 @@ impl<T: FieldElement> Pipeline<T> {
let start = Instant::now();
let fixed_cols = constant_evaluator::generate(&pil);
self.maybe_write_constants(&fixed_cols)?;
self.log(&format!("Took {}", start.elapsed().as_secs_f32()));
self.artifact.fixed_cols = Some(Rc::new(fixed_cols));

19
pipeline/src/test_util.rs
View File

@@ -28,7 +28,7 @@ pub fn verify_test_file(
.from_file(resolve_test_file(file_name))
.with_prover_inputs(inputs)
.add_external_witness_values(external_witness_values);
verify_pipeline(pipeline)
verify_pipeline(pipeline, BackendType::EStarkDump)
}
pub fn verify_asm_string<S: serde::Serialize + Send + Sync + 'static>(
@@ -47,11 +47,14 @@ pub fn verify_asm_string<S: serde::Serialize + Send + Sync + 'static>(
pipeline = pipeline.add_data_vec(&data);
}
verify_pipeline(pipeline).unwrap();
verify_pipeline(pipeline, BackendType::EStarkDump).unwrap();
}
pub fn verify_pipeline(pipeline: Pipeline<GoldilocksField>) -> Result<(), String> {
let mut pipeline = pipeline.with_backend(BackendType::PilStarkCli);
pub fn verify_pipeline(
pipeline: Pipeline<GoldilocksField>,
backend: BackendType,
) -> Result<(), String> {
let mut pipeline = pipeline.with_backend(backend);
let tmp_dir = mktemp::Temp::new_dir().unwrap();
if pipeline.output_dir().is_none() {
@@ -60,7 +63,7 @@ pub fn verify_pipeline(pipeline: Pipeline<GoldilocksField>) -> Result<(), String
pipeline.compute_proof().unwrap();
verify(pipeline.output_dir().unwrap(), pipeline.name(), None)
verify(pipeline.output_dir().unwrap())
}
pub fn gen_estark_proof(file_name: &str, inputs: Vec<GoldilocksField>) {
@@ -69,7 +72,7 @@ pub fn gen_estark_proof(file_name: &str, inputs: Vec<GoldilocksField>) {
.with_tmp_output(&tmp_dir)
.from_file(resolve_test_file(file_name))
.with_prover_inputs(inputs)
.with_backend(powdr_backend::BackendType::EStark);
.with_backend(powdr_backend::BackendType::EStarkStarky);
pipeline.clone().compute_proof().unwrap();
@@ -228,7 +231,7 @@ pub fn assert_proofs_fail_for_invalid_witnesses_pilcom(
.from_file(resolve_test_file(file_name))
.set_witness(convert_witness(witness));
assert!(verify_pipeline(pipeline.clone()).is_err());
assert!(verify_pipeline(pipeline.clone(), BackendType::EStarkDump).is_err());
}
pub fn assert_proofs_fail_for_invalid_witnesses_estark(
@@ -241,7 +244,7 @@ pub fn assert_proofs_fail_for_invalid_witnesses_estark(
assert!(pipeline
.clone()
.with_backend(powdr_backend::BackendType::EStark)
.with_backend(powdr_backend::BackendType::EStarkStarky)
.compute_proof()
.is_err());
}

5
pipeline/src/util.rs
View File

@@ -39,7 +39,6 @@ impl PolySet for WitnessPolySet {
pub fn try_read_poly_set<P: PolySet, T: FieldElement>(
pil: &Analyzed<T>,
dir: &Path,
name: &str,
) -> Option<(Vec<(String, Vec<T>)>, DegreeType)> {
let column_names: Vec<String> = P::get_polys(pil)
.iter()
@@ -48,9 +47,9 @@ pub fn try_read_poly_set<P: PolySet, T: FieldElement>(
.collect();
(!column_names.is_empty()).then(|| {
let fname = format!("{name}_{}", P::FILE_NAME);
let path = dir.join(P::FILE_NAME);
read_polys_file(
&mut BufReader::new(File::open(dir.join(fname)).unwrap()),
&mut BufReader::new(File::open(path).unwrap()),
&column_names,
)
})

11
pipeline/src/verify.rs
View File

@@ -1,16 +1,11 @@
use std::{path::Path, process::Command};
pub fn verify(temp_dir: &Path, name: &str, constants_name: Option<&str>) -> Result<(), String> {
pub fn verify(temp_dir: &Path) -> Result<(), String> {
let pilcom = std::env::var("PILCOM")
.expect("Please set the PILCOM environment variable to the path to the pilcom repository.");
let constants_name = constants_name.unwrap_or(name);
let constants_file = format!(
"{}/{constants_name}_constants.bin",
temp_dir.to_str().unwrap()
);
let commits_file = format!("{}/{name}_commits.bin", temp_dir.to_str().unwrap());
let constants_file = format!("{}/constants.bin", temp_dir.to_str().unwrap());
let commits_file = format!("{}/commits.bin", temp_dir.to_str().unwrap());
let constraints_file = format!("{}/constraints.json", temp_dir.to_str().unwrap());
let verifier_output = Command::new("node")

11
pipeline/tests/asm.rs
View File

@@ -1,3 +1,4 @@
use powdr_backend::BackendType;
use powdr_number::{Bn254Field, FieldElement, GoldilocksField};
use powdr_pipeline::{
test_util::{gen_estark_proof, resolve_test_file, test_halo2, verify_test_file},
@@ -341,14 +342,14 @@ fn read_poly_files() {
// generate poly files
let mut pipeline = Pipeline::<Bn254Field>::default()
.from_file(resolve_test_file(f))
.with_output(tmp_dir.to_path_buf(), true);
.with_output(tmp_dir.to_path_buf(), true)
.with_backend(BackendType::EStarkDump);
pipeline.compute_witness().unwrap();
let name = pipeline.name().to_string();
let pil = pipeline.compute_optimized_pil().unwrap();
pipeline.compute_proof().unwrap();
// check fixed cols (may have no fixed cols)
if let Some((fixed, degree)) =
try_read_poly_set::<FixedPolySet, _>(&pil, tmp_dir.as_path(), &name)
if let Some((fixed, degree)) = try_read_poly_set::<FixedPolySet, _>(&pil, tmp_dir.as_path())
{
assert_eq!(pil.degree(), degree);
assert_eq!(pil.degree(), fixed[0].1.len() as u64);
@@ -356,7 +357,7 @@ fn read_poly_files() {
// check witness cols (examples assumed to have at least one witness col)
let (witness, degree) =
try_read_poly_set::<WitnessPolySet, _>(&pil, tmp_dir.as_path(), &name).unwrap();
try_read_poly_set::<WitnessPolySet, _>(&pil, tmp_dir.as_path()).unwrap();
assert_eq!(pil.degree(), degree);
assert_eq!(pil.degree(), witness[0].1.len() as u64);
}

3
powdr/Cargo.toml
View File

@@ -18,5 +18,6 @@ powdr-riscv = { path = "../riscv" }
powdr-riscv-executor = { path = "../riscv-executor" }
[features]
default = ["halo2"] # halo2 is enabled by default
default = ["halo2"]
halo2 = ["powdr-backend/halo2", "powdr-pipeline/halo2"]
estark-polygon = ["powdr-backend/estark-polygon", "powdr-pipeline/estark-polygon", "powdr-riscv/estark-polygon"]

1
riscv/Cargo.toml
View File

@@ -11,6 +11,7 @@ repository = { workspace = true }
[features]
default = [] # complex-tests is disabled by default
complex-tests = []
estark-polygon = ["powdr-pipeline/estark-polygon"]
[dependencies]
powdr-ast = { path = "../ast" }

4
riscv/tests/common/mod.rs
View File

@@ -1,3 +1,4 @@
use powdr_backend::BackendType;
use powdr_number::GoldilocksField;
use powdr_pipeline::{test_util::verify_pipeline, Pipeline};
use std::path::PathBuf;
@@ -8,6 +9,7 @@ pub fn verify_riscv_asm_string<S: serde::Serialize + Send + Sync + 'static>(
contents: &str,
inputs: Vec<GoldilocksField>,
data: Option<Vec<(u32, S)>>,
backend: BackendType,
) {
let temp_dir = mktemp::Temp::new_dir().unwrap().release();
@@ -30,5 +32,5 @@ pub fn verify_riscv_asm_string<S: serde::Serialize + Send + Sync + 'static>(
usize::MAX,
powdr_riscv_executor::ExecMode::Fast,
);
verify_pipeline(pipeline).unwrap();
verify_pipeline(pipeline, backend).unwrap();
}

9
riscv/tests/instructions.rs
View File

@@ -2,6 +2,7 @@ mod common;
mod instruction_tests {
use crate::common::verify_riscv_asm_string;
use powdr_backend::BackendType;
use powdr_number::GoldilocksField;
use powdr_riscv::compiler::compile;
use powdr_riscv::Runtime;
@@ -15,7 +16,13 @@ mod instruction_tests {
false,
);
verify_riscv_asm_string::<()>(&format!("{name}.asm"), &powdr_asm, Default::default(), None);
verify_riscv_asm_string::<()>(
&format!("{name}.asm"),
&powdr_asm,
Default::default(),
None,
BackendType::EStarkDump,
);
}
include!(concat!(env!("OUT_DIR"), "/instruction_tests.rs"));

39
riscv/tests/riscv.rs
View File

@@ -34,9 +34,9 @@ pub fn test_continuations(case: &str) {
let pipeline_callback = |pipeline: Pipeline<GoldilocksField>| -> Result<(), ()> {
// Can't use `verify_pipeline`, because the pipeline was renamed in the middle of after
// computing the constants file.
let mut pipeline = pipeline.with_backend(BackendType::PilStarkCli);
let mut pipeline = pipeline.with_backend(BackendType::EStarkDump);
pipeline.compute_proof().unwrap();
verify(pipeline.output_dir().unwrap(), pipeline.name(), Some(case)).unwrap();
verify(pipeline.output_dir().unwrap()).unwrap();
Ok(())
};
let bootloader_inputs = rust_continuations_dry_run(&mut pipeline);
@@ -126,6 +126,22 @@ fn keccak() {
verify_riscv_crate(case, Default::default(), &Runtime::base());
}
#[cfg(feature = "estark-polygon")]
#[test]
#[ignore = "Too slow"]
fn test_vec_median_estark_polygon() {
let case = "vec_median";
verify_riscv_crate_with_backend(
case,
[5, 11, 15, 75, 6, 5, 1, 4, 7, 3, 2, 9, 2]
.into_iter()
.map(|x| x.into())
.collect(),
&Runtime::base(),
BackendType::EStarkPolygon,
);
}
#[test]
#[ignore = "Too slow"]
fn vec_median() {
@@ -234,8 +250,17 @@ fn many_chunks_memory() {
}
fn verify_riscv_crate(case: &str, inputs: Vec<GoldilocksField>, runtime: &Runtime) {
verify_riscv_crate_with_backend(case, inputs, runtime, BackendType::EStarkDump)
}
fn verify_riscv_crate_with_backend(
case: &str,
inputs: Vec<GoldilocksField>,
runtime: &Runtime,
backend: BackendType,
) {
let powdr_asm = compile_riscv_crate::<GoldilocksField>(case, runtime);
verify_riscv_asm_string::<()>(&format!("{case}.asm"), &powdr_asm, inputs, None);
verify_riscv_asm_string::<()>(&format!("{case}.asm"), &powdr_asm, inputs, None, backend);
}
fn verify_riscv_crate_with_data<S: serde::Serialize + Send + Sync + 'static>(
@@ -246,7 +271,13 @@ fn verify_riscv_crate_with_data<S: serde::Serialize + Send + Sync + 'static>(
) {
let powdr_asm = compile_riscv_crate::<GoldilocksField>(case, runtime);
verify_riscv_asm_string(&format!("{case}.asm"), &powdr_asm, inputs, Some(data));
verify_riscv_asm_string(
&format!("{case}.asm"),
&powdr_asm,
inputs,
Some(data),
BackendType::EStarkDump,
);
}
fn compile_riscv_crate<T: FieldElement>(case: &str, runtime: &Runtime) -> String {