diff --git a/coordinator/internal/types/prover.go b/coordinator/internal/types/prover.go index 0957755fd..9b9bbdb71 100644 --- a/coordinator/internal/types/prover.go +++ b/coordinator/internal/types/prover.go @@ -27,6 +27,8 @@ const ( ProverTypeChunk // ProverTypeBatch signals it's a batch prover, which can prove batch_tasks and bundle_tasks ProverTypeBatch + // ProverTypeOpenVM signals it's an openvm prover + ProverTypeOpenVM ) // MakeProverType make ProverType from ProofType diff --git a/prover/src/config.rs b/prover/src/config.rs index 4e3c1f2cc..8071cecd8 100644 --- a/prover/src/config.rs +++ b/prover/src/config.rs @@ -1,17 +1,16 @@ -use anyhow::{bail, Result}; +use anyhow::Result; use serde::{Deserialize, Serialize}; -use std::fs::File; +use std::{collections::HashMap, fs::File}; use crate::types::ProverType; -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct CircuitConfig { pub hard_fork_name: String, - pub params_path: String, - pub assets_path: String, + pub workspace_path: String, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct CoordinatorConfig { pub base_url: String, pub retry_count: u32, @@ -19,20 +18,19 @@ pub struct CoordinatorConfig { pub connection_timeout_sec: u64, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct L2GethConfig { pub endpoint: String, } -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, Clone)] pub struct Config { pub prover_name: String, pub keystore_path: String, pub keystore_password: String, pub db_path: String, pub prover_type: ProverType, - pub low_version_circuit: CircuitConfig, - pub high_version_circuit: CircuitConfig, + pub circuits: HashMap, pub coordinator: CoordinatorConfig, pub l2geth: Option, } @@ -50,53 +48,3 @@ impl Config { Config::from_reader(&file) } } - -static SCROLL_PROVER_ASSETS_DIR_ENV_NAME: &str = "SCROLL_PROVER_ASSETS_DIR"; -static mut SCROLL_PROVER_ASSETS_DIRS: Vec = vec![]; - -#[derive(Debug)] -pub struct AssetsDirEnvConfig {} - -impl AssetsDirEnvConfig { - pub fn init() -> Result<()> { - let value = std::env::var(SCROLL_PROVER_ASSETS_DIR_ENV_NAME)?; - let dirs: Vec<&str> = value.split(',').collect(); - if dirs.len() != 2 { - bail!("env variable SCROLL_PROVER_ASSETS_DIR value must be 2 parts seperated by comma.") - } - unsafe { - SCROLL_PROVER_ASSETS_DIRS = dirs.into_iter().map(|s| s.to_string()).collect(); - log::info!( - "init SCROLL_PROVER_ASSETS_DIRS: {:?}", - SCROLL_PROVER_ASSETS_DIRS - ); - } - Ok(()) - } - - pub fn enable_first() { - unsafe { - log::info!( - "set env {SCROLL_PROVER_ASSETS_DIR_ENV_NAME} to {}", - &SCROLL_PROVER_ASSETS_DIRS[0] - ); - std::env::set_var( - SCROLL_PROVER_ASSETS_DIR_ENV_NAME, - &SCROLL_PROVER_ASSETS_DIRS[0], - ); - } - } - - pub fn enable_second() { - unsafe { - log::info!( - "set env {SCROLL_PROVER_ASSETS_DIR_ENV_NAME} to {}", - &SCROLL_PROVER_ASSETS_DIRS[1] - ); - std::env::set_var( - SCROLL_PROVER_ASSETS_DIR_ENV_NAME, - &SCROLL_PROVER_ASSETS_DIRS[1], - ); - } - } -} diff --git a/prover/src/coordinator_client.rs b/prover/src/coordinator_client.rs index 46067d7cc..674ff586d 100644 --- a/prover/src/coordinator_client.rs +++ b/prover/src/coordinator_client.rs @@ -4,7 +4,6 @@ pub mod listener; pub mod types; use anyhow::{bail, Context, Ok, Result}; -use std::rc::Rc; use api::Api; use errors::*; @@ -16,20 +15,20 @@ use crate::{config::Config, key_signer::KeySigner}; pub use errors::ProofStatusNotOKError; -pub struct CoordinatorClient<'a> { +pub struct CoordinatorClient { api: Api, token: Option, - config: &'a Config, - key_signer: Rc, + config: Config, + key_signer: KeySigner, rt: Runtime, listener: Box, vks: Vec, } -impl<'a> CoordinatorClient<'a> { +impl CoordinatorClient { pub fn new( - config: &'a Config, - key_signer: Rc, + config: Config, + key_signer: KeySigner, listener: Box, vks: Vec, ) -> Result { diff --git a/prover/src/geth_client.rs b/prover/src/geth_client.rs index e617d8eba..c05941b5c 100644 --- a/prover/src/geth_client.rs +++ b/prover/src/geth_client.rs @@ -1,11 +1,7 @@ -use crate::types::CommonHash; use anyhow::Result; use ethers_core::types::BlockNumber; use tokio::runtime::Runtime; -use serde::{de::DeserializeOwned, Serialize}; -use std::fmt::Debug; - use ethers_providers::{Http, Provider}; pub struct GethClient { @@ -28,24 +24,6 @@ impl GethClient { }) } - pub fn get_block_trace_by_hash(&mut self, hash: &CommonHash) -> Result - where - T: Serialize + DeserializeOwned + Debug + Send, - { - log::info!( - "{}: calling get_block_trace_by_hash, hash: {:#?}", - self.id, - hash - ); - - let trace_future = self - .provider - .request("scroll_getBlockTraceByNumberOrHash", [format!("{hash:#x}")]); - - let trace = self.rt.block_on(trace_future)?; - Ok(trace) - } - pub fn block_number(&mut self) -> Result { log::info!("{}: calling block_number", self.id); diff --git a/prover/src/main.rs b/prover/src/main.rs index 75553187a..f8cce357e 100644 --- a/prover/src/main.rs +++ b/prover/src/main.rs @@ -15,7 +15,7 @@ mod zk_circuits_handler; use anyhow::Result; use clap::{ArgAction, Parser}; -use config::{AssetsDirEnvConfig, Config}; +use config::Config; use prover::Prover; use std::rc::Rc; use task_cache::{ClearCacheCoordinatorListener, TaskCache}; @@ -49,29 +49,23 @@ fn start() -> Result<()> { utils::log_init(args.log_file); let config: Config = Config::from_file(args.config_file)?; - - if let Err(e) = AssetsDirEnvConfig::init() { - log::error!("AssetsDirEnvConfig init failed: {:#}", e); - std::process::exit(-2); - } - let task_cache = Rc::new(TaskCache::new(&config.db_path)?); let coordinator_listener = Box::new(ClearCacheCoordinatorListener { task_cache: task_cache.clone(), }); - let prover = Prover::new(&config, coordinator_listener)?; + let prover = Prover::new(config.clone(), coordinator_listener)?; log::info!( - "prover start successfully. name: {}, type: {:?}, publickey: {}, version: {}", + "starting prover. name: {}, type: {:?}, publickey: {}, version: {}", config.prover_name, config.prover_type, - prover.get_public_key(), + prover.public_key, version::get_version(), ); - let task_processor = TaskProcessor::new(&prover, task_cache); + let task_processor = TaskProcessor::new(prover, task_cache); task_processor.start(); diff --git a/prover/src/prover.rs b/prover/src/prover.rs index 7de83906e..f3b4c20dd 100644 --- a/prover/src/prover.rs +++ b/prover/src/prover.rs @@ -1,91 +1,76 @@ use anyhow::{bail, Context, Error, Ok, Result}; -use ethers_core::types::U64; - -use std::{cell::RefCell, rc::Rc}; use crate::{ config::Config, coordinator_client::{listener::Listener, types::*, CoordinatorClient}, geth_client::GethClient, key_signer::KeySigner, - types::{ProofFailureType, ProofStatus, ProverType}, - utils::get_task_types, - zk_circuits_handler::{CircuitsHandler, CircuitsHandlerProvider}, + types::{ProofFailureType, ProofStatus, TaskType}, + zk_circuits_handler::{euclid::EuclidHandler, CircuitsHandler}, }; use super::types::{ProofDetail, Task}; -pub struct Prover<'a> { - config: &'a Config, - key_signer: Rc, - circuits_handler_provider: RefCell>, - coordinator_client: RefCell>, - geth_client: Option>>, +pub struct Prover { + config: Config, + pub public_key: String, + coordinator_client: CoordinatorClient, + geth_client: GethClient, + + active_handler: Option<(String, Box)>, } -impl<'a> Prover<'a> { - pub fn new(config: &'a Config, coordinator_listener: Box) -> Result { - let prover_type = config.prover_type; +impl Prover { + pub fn new(config: Config, coordinator_listener: Box) -> Result { let keystore_path = &config.keystore_path; let keystore_password = &config.keystore_password; - let geth_client = if config.prover_type == ProverType::Chunk { - Some(Rc::new(RefCell::new( - GethClient::new( - &config.prover_name, - &config.l2geth.as_ref().unwrap().endpoint, - ) - .context("failed to create l2 geth_client")?, - ))) - } else { - None - }; + let geth_client = GethClient::new( + &config.prover_name, + &config.l2geth.as_ref().unwrap().endpoint, + ) + .context("failed to create l2 geth_client")?; - let provider = CircuitsHandlerProvider::new(prover_type, config, geth_client.clone()) - .context("failed to create circuits handler provider")?; + let key_signer = KeySigner::new(keystore_path, keystore_password)?; + let public_key = key_signer.get_public_key(); - let vks = provider.init_vks(prover_type, config, geth_client.clone()); - - let key_signer = Rc::new(KeySigner::new(keystore_path, keystore_password)?); - let coordinator_client = - CoordinatorClient::new(config, Rc::clone(&key_signer), coordinator_listener, vks) - .context("failed to create coordinator_client")?; + let coordinator_client = CoordinatorClient::new( + config.clone(), + key_signer, + coordinator_listener, + vec![], /* todo: vks */ + ) + .context("failed to create coordinator_client")?; let prover = Prover { config, - key_signer: Rc::clone(&key_signer), - circuits_handler_provider: RefCell::new(provider), - coordinator_client: RefCell::new(coordinator_client), + public_key, + coordinator_client, geth_client, + active_handler: None, }; Ok(prover) } - pub fn get_public_key(&self) -> String { - self.key_signer.get_public_key() - } - - pub fn fetch_task(&self) -> Result { + pub fn fetch_task(&mut self) -> Result { log::info!("[prover] start to fetch_task"); let mut req = GetTaskRequest { - task_types: get_task_types(self.config.prover_type), + task_types: vec![TaskType::Chunk, TaskType::Batch, TaskType::Bundle], prover_height: None, }; - if self.config.prover_type == ProverType::Chunk { - let latest_block_number = self.get_latest_block_number_value()?; - if let Some(v) = latest_block_number { - if v.as_u64() == 0 { - bail!("omit to prove task of the genesis block") - } - req.prover_height = Some(v.as_u64()); - } else { - log::error!("[prover] failed to fetch latest confirmed block number, got None"); - bail!("failed to fetch latest confirmed block number, got None") + let latest_block_number = self.geth_client.block_number()?.as_number(); + if let Some(v) = latest_block_number { + if v.as_u64() == 0 { + bail!("omit to prove task of the genesis block") } + req.prover_height = Some(v.as_u64()); + } else { + log::error!("[prover] failed to fetch latest confirmed block number, got None"); + bail!("failed to fetch latest confirmed block number, got None") } - let resp = self.coordinator_client.borrow_mut().get_task(&req)?; + let resp = self.coordinator_client.get_task(&req)?; match resp.data { Some(d) => Ok(Task::from(d)), @@ -95,28 +80,43 @@ impl<'a> Prover<'a> { } } - pub fn prove_task(&self, task: &Task) -> Result { - log::info!("[prover] start to prove_task, task id: {}", task.id); - let handler: Rc> = self - .circuits_handler_provider - .borrow_mut() - .get_circuits_handler(&task.hard_fork_name) - .context("failed to get circuit handler")?; - self.do_prove(task, handler) + fn set_active_handler(&mut self, hard_fork_name: &str) { + if let Some(handler) = &self.active_handler { + if handler.0 == hard_fork_name { + return; + } + } + + // if we got assigned a task for an unknown hard fork, there is something wrong in the + // coordinator + let config = self.config.circuits.get(hard_fork_name).unwrap(); + + let handler = Box::new(match hard_fork_name { + "euclid" => EuclidHandler::new(&config.workspace_path), + _ => unreachable!(), + }) as Box; + self.active_handler = Some((hard_fork_name.to_string(), handler)); } - fn do_prove(&self, task: &Task, handler: Rc>) -> Result { + pub fn prove_task(&mut self, task: &Task) -> Result { + log::info!("[prover] start to prove_task, task id: {}", task.id); let mut proof_detail = ProofDetail { id: task.id.clone(), proof_type: task.task_type, ..Default::default() }; - proof_detail.proof_data = handler.get_proof_data(task.task_type, task)?; + self.set_active_handler(&task.hard_fork_name); + proof_detail.proof_data = self + .active_handler + .as_ref() + .unwrap() + .1 + .get_proof_data(task, &self.geth_client)?; Ok(proof_detail) } - pub fn submit_proof(&self, proof_detail: ProofDetail, task: &Task) -> Result<()> { + pub fn submit_proof(&mut self, proof_detail: ProofDetail, task: &Task) -> Result<()> { log::info!( "[prover] start to submit_proof, task id: {}", proof_detail.id @@ -131,11 +131,11 @@ impl<'a> Prover<'a> { ..Default::default() }; - self.do_submit(&request) + self.coordinator_client.submit_proof(&request).map(|_r| ()) } pub fn submit_error( - &self, + &mut self, task: &Task, failure_type: ProofFailureType, error: Error, @@ -150,21 +150,7 @@ impl<'a> Prover<'a> { failure_msg: Some(format!("{:#}", error)), ..Default::default() }; - self.do_submit(&request) - } - fn do_submit(&self, request: &SubmitProofRequest) -> Result<()> { - self.coordinator_client.borrow_mut().submit_proof(request)?; - Ok(()) - } - - fn get_latest_block_number_value(&self) -> Result> { - let number = self - .geth_client - .as_ref() - .unwrap() - .borrow_mut() - .block_number()?; - Ok(number.as_number()) + self.coordinator_client.submit_proof(&request).map(|_r| ()) } } diff --git a/prover/src/task_processor.rs b/prover/src/task_processor.rs index df4629d5b..ba061e91e 100644 --- a/prover/src/task_processor.rs +++ b/prover/src/task_processor.rs @@ -2,20 +2,20 @@ use super::{coordinator_client::ProofStatusNotOKError, prover::Prover, task_cach use anyhow::{Context, Result}; use std::rc::Rc; -pub struct TaskProcessor<'a> { - prover: &'a Prover<'a>, +pub struct TaskProcessor { + prover: Prover, task_cache: Rc, } -impl<'a> TaskProcessor<'a> { - pub fn new(prover: &'a Prover<'a>, task_cache: Rc) -> Self { +impl TaskProcessor { + pub fn new(prover: Prover, task_cache: Rc) -> Self { TaskProcessor { prover, task_cache } } - pub fn start(&self) { + pub fn start(mut self) { loop { log::info!("start a new round."); - if let Err(err) = self.prove_and_submit() { + if let Err(err) = &self.prove_and_submit() { if err.is::() { log::info!("proof status not ok, downgrade level to info."); } else { @@ -27,7 +27,7 @@ impl<'a> TaskProcessor<'a> { } } - fn prove_and_submit(&self) -> Result<()> { + fn prove_and_submit(&mut self) -> Result<()> { let task_from_cache = self .task_cache .get_last_task() diff --git a/prover/src/types.rs b/prover/src/types.rs index 513995d6d..4aa073913 100644 --- a/prover/src/types.rs +++ b/prover/src/types.rs @@ -56,15 +56,17 @@ impl Default for TaskType { #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum ProverType { - Chunk, - Batch, + Deprecated1, + Deprecated2, + OpenVM, } impl ProverType { fn from_u8(v: u8) -> Self { match v { - 1 => ProverType::Chunk, - 2 => ProverType::Batch, + 1 => ProverType::Deprecated1, + 2 => ProverType::Deprecated2, + 3 => ProverType::OpenVM, _ => { panic!("invalid prover_type") } @@ -73,8 +75,9 @@ impl ProverType { pub fn to_u8(self) -> u8 { match self { - ProverType::Chunk => 1, - ProverType::Batch => 2, + ProverType::Deprecated1 => 1, + ProverType::Deprecated2 => 2, + ProverType::OpenVM => 3, } } } @@ -84,10 +87,7 @@ impl Serialize for ProverType { where S: Serializer, { - match *self { - ProverType::Chunk => serializer.serialize_u8(1), - ProverType::Batch => serializer.serialize_u8(2), - } + serializer.serialize_u8(self.to_u8()) } } diff --git a/prover/src/utils.rs b/prover/src/utils.rs index 18be4ac7a..87b7137e3 100644 --- a/prover/src/utils.rs +++ b/prover/src/utils.rs @@ -1,8 +1,6 @@ use env_logger::Env; use std::{fs::OpenOptions, sync::Once}; -use crate::types::{ProverType, TaskType}; - static LOG_INIT: Once = Once::new(); /// Initialize log @@ -23,10 +21,3 @@ pub fn log_init(log_file: Option) { builder.init(); }); } - -pub fn get_task_types(prover_type: ProverType) -> Vec { - match prover_type { - ProverType::Chunk => vec![TaskType::Chunk], - ProverType::Batch => vec![TaskType::Batch, TaskType::Bundle], - } -} diff --git a/prover/src/zk_circuits_handler.rs b/prover/src/zk_circuits_handler.rs index d1a8eb38c..4b5708a94 100644 --- a/prover/src/zk_circuits_handler.rs +++ b/prover/src/zk_circuits_handler.rs @@ -1,19 +1,10 @@ -mod common; -mod darwin; -mod darwin_v2; +pub mod euclid; -use super::geth_client::GethClient; use crate::{ - config::{AssetsDirEnvConfig, Config}, - types::{ProverType, Task, TaskType}, - utils::get_task_types, + geth_client::GethClient, + types::{Task, TaskType}, }; -use anyhow::{bail, Result}; -use darwin::DarwinHandler; -use darwin_v2::DarwinV2Handler; -use std::{cell::RefCell, collections::HashMap, rc::Rc}; - -type HardForkName = String; +use anyhow::Result; pub mod utils { pub fn encode_vk(vk: Vec) -> String { @@ -24,151 +15,5 @@ pub mod utils { pub trait CircuitsHandler { fn get_vk(&self, task_type: TaskType) -> Option>; - fn get_proof_data(&self, task_type: TaskType, task: &Task) -> Result; -} - -type CircuitsHandlerBuilder = fn( - prover_type: ProverType, - config: &Config, - geth_client: Option>>, -) -> Result>; - -pub struct CircuitsHandlerProvider<'a> { - prover_type: ProverType, - config: &'a Config, - geth_client: Option>>, - circuits_handler_builder_map: HashMap, - - current_fork_name: Option, - current_circuit: Option>>, -} - -impl<'a> CircuitsHandlerProvider<'a> { - pub fn new( - prover_type: ProverType, - config: &'a Config, - geth_client: Option>>, - ) -> Result { - let mut m: HashMap = HashMap::new(); - - fn handler_builder( - prover_type: ProverType, - config: &Config, - geth_client: Option>>, - ) -> Result> { - log::info!( - "now init zk circuits handler, hard_fork_name: {}", - &config.low_version_circuit.hard_fork_name - ); - AssetsDirEnvConfig::enable_first(); - DarwinHandler::new( - prover_type, - &config.low_version_circuit.params_path, - &config.low_version_circuit.assets_path, - geth_client, - ) - .map(|handler| Box::new(handler) as Box) - } - m.insert( - config.low_version_circuit.hard_fork_name.clone(), - handler_builder, - ); - - fn next_handler_builder( - prover_type: ProverType, - config: &Config, - geth_client: Option>>, - ) -> Result> { - log::info!( - "now init zk circuits handler, hard_fork_name: {}", - &config.high_version_circuit.hard_fork_name - ); - AssetsDirEnvConfig::enable_second(); - DarwinV2Handler::new( - prover_type, - &config.high_version_circuit.params_path, - &config.high_version_circuit.assets_path, - geth_client, - ) - .map(|handler| Box::new(handler) as Box) - } - - m.insert( - config.high_version_circuit.hard_fork_name.clone(), - next_handler_builder, - ); - - let provider = CircuitsHandlerProvider { - prover_type, - config, - geth_client, - circuits_handler_builder_map: m, - current_fork_name: None, - current_circuit: None, - }; - - Ok(provider) - } - - pub fn get_circuits_handler( - &mut self, - hard_fork_name: &String, - ) -> Result>> { - match &self.current_fork_name { - Some(fork_name) if fork_name == hard_fork_name => { - log::info!("get circuits handler from cache"); - if let Some(handler) = &self.current_circuit { - Ok(handler.clone()) - } else { - bail!("missing cached handler, there must be something wrong.") - } - } - _ => { - log::info!( - "failed to get circuits handler from cache, create a new one: {hard_fork_name}" - ); - if let Some(builder) = self.circuits_handler_builder_map.get(hard_fork_name) { - log::info!("building circuits handler for {hard_fork_name}"); - let handler = builder(self.prover_type, self.config, self.geth_client.clone()) - .expect("failed to build circuits handler"); - self.current_fork_name = Some(hard_fork_name.clone()); - let rc_handler = Rc::new(handler); - self.current_circuit = Some(rc_handler.clone()); - Ok(rc_handler) - } else { - bail!("missing builder, there must be something wrong.") - } - } - } - } - - pub fn init_vks( - &self, - prover_type: ProverType, - config: &'a Config, - geth_client: Option>>, - ) -> Vec { - self.circuits_handler_builder_map - .iter() - .flat_map(|(hard_fork_name, build)| { - let handler = build(prover_type, config, geth_client.clone()) - .expect("failed to build circuits handler"); - - get_task_types(prover_type) - .into_iter() - .map(|task_type| { - let vk = handler - .get_vk(task_type) - .map_or("".to_string(), utils::encode_vk); - log::info!( - "vk for {hard_fork_name}, is {vk}, task_type: {:?}", - task_type - ); - vk - }) - .filter(|vk| !vk.is_empty()) - .collect::>() - }) - .collect::>() - } + fn get_proof_data(&self, task: &Task, geth_client: &GethClient) -> Result; } diff --git a/prover/src/zk_circuits_handler/common.rs b/prover/src/zk_circuits_handler/common.rs deleted file mode 100644 index e88628ad6..000000000 --- a/prover/src/zk_circuits_handler/common.rs +++ /dev/null @@ -1,33 +0,0 @@ -use std::{collections::BTreeMap, rc::Rc}; - -use crate::types::ProverType; - -use once_cell::sync::OnceCell; - -use halo2_proofs::{halo2curves::bn256::Bn256, poly::kzg::commitment::ParamsKZG}; - -static mut PARAMS_MAP: OnceCell>>> = OnceCell::new(); - -pub fn get_params_map_instance<'a, F>(load_params_func: F) -> &'a BTreeMap> -where - F: FnOnce() -> BTreeMap>, -{ - unsafe { - PARAMS_MAP.get_or_init(|| { - let params_map = load_params_func(); - Rc::new(params_map) - }) - } -} - -pub fn get_degrees(prover_types: &std::collections::HashSet, f: F) -> Vec -where - F: FnMut(&ProverType) -> Vec, -{ - prover_types - .iter() - .flat_map(f) - .collect::>() - .into_iter() - .collect() -} diff --git a/prover/src/zk_circuits_handler/darwin.rs b/prover/src/zk_circuits_handler/darwin.rs deleted file mode 100644 index 96618f950..000000000 --- a/prover/src/zk_circuits_handler/darwin.rs +++ /dev/null @@ -1,467 +0,0 @@ -use super::{common::*, CircuitsHandler}; -use crate::{ - geth_client::GethClient, - types::{ProverType, TaskType}, -}; -use anyhow::{bail, Context, Ok, Result}; -use once_cell::sync::Lazy; -use serde::Deserialize; - -use crate::types::{CommonHash, Task}; -use std::{cell::RefCell, cmp::Ordering, env, rc::Rc}; - -use prover_darwin::{ - aggregator::Prover as BatchProver, - check_chunk_hashes, - common::Prover as CommonProver, - config::{AGG_DEGREES, ZKEVM_DEGREES}, - zkevm::Prover as ChunkProver, - BatchProof, BatchProvingTask, BlockTrace, BundleProof, BundleProvingTask, ChunkInfo, - ChunkProof, ChunkProvingTask, -}; - -// Only used for debugging. -static OUTPUT_DIR: Lazy> = Lazy::new(|| env::var("PROVER_OUTPUT_DIR").ok()); - -#[derive(Debug, Clone, Deserialize)] -pub struct BatchTaskDetail { - pub chunk_infos: Vec, - #[serde(flatten)] - pub batch_proving_task: BatchProvingTask, -} - -type BundleTaskDetail = BundleProvingTask; - -#[derive(Debug, Clone, Deserialize)] -pub struct ChunkTaskDetail { - pub block_hashes: Vec, -} - -fn get_block_number(block_trace: &BlockTrace) -> Option { - block_trace.header.number.map(|n| n.as_u64()) -} - -#[derive(Default)] -pub struct DarwinHandler { - chunk_prover: Option>>, - batch_prover: Option>>, - - geth_client: Option>>, -} - -impl DarwinHandler { - pub fn new_multi( - prover_types: Vec, - params_dir: &str, - assets_dir: &str, - geth_client: Option>>, - ) -> Result { - let class_name = std::intrinsics::type_name::(); - let prover_types_set = prover_types - .into_iter() - .collect::>(); - let mut handler = Self { - batch_prover: None, - chunk_prover: None, - geth_client, - }; - let degrees: Vec = get_degrees(&prover_types_set, |prover_type| match prover_type { - ProverType::Chunk => ZKEVM_DEGREES.clone(), - ProverType::Batch => AGG_DEGREES.clone(), - }); - let params_map = get_params_map_instance(|| { - log::info!( - "calling get_params_map from {}, prover_types: {:?}, degrees: {:?}", - class_name, - prover_types_set, - degrees - ); - CommonProver::load_params_map(params_dir, °rees) - }); - for prover_type in prover_types_set { - match prover_type { - ProverType::Chunk => { - handler.chunk_prover = Some(RefCell::new(ChunkProver::from_params_and_assets( - params_map, assets_dir, - ))); - } - ProverType::Batch => { - handler.batch_prover = Some(RefCell::new(BatchProver::from_params_and_assets( - params_map, assets_dir, - ))) - } - } - } - Ok(handler) - } - - pub fn new( - prover_type: ProverType, - params_dir: &str, - assets_dir: &str, - geth_client: Option>>, - ) -> Result { - Self::new_multi(vec![prover_type], params_dir, assets_dir, geth_client) - } - - fn gen_chunk_proof_raw(&self, chunk_trace: Vec) -> Result { - if let Some(prover) = self.chunk_prover.as_ref() { - let chunk = ChunkProvingTask::from(chunk_trace); - - let chunk_proof = - prover - .borrow_mut() - .gen_chunk_proof(chunk, None, None, self.get_output_dir())?; - - return Ok(chunk_proof); - } - unreachable!("please check errors in proof_type logic") - } - - fn gen_chunk_proof(&self, task: &crate::types::Task) -> Result { - let chunk_trace = self.gen_chunk_traces(task)?; - let chunk_proof = self.gen_chunk_proof_raw(chunk_trace)?; - Ok(serde_json::to_string(&chunk_proof)?) - } - - fn gen_batch_proof_raw(&self, batch_task_detail: BatchTaskDetail) -> Result { - if let Some(prover) = self.batch_prover.as_ref() { - let chunk_hashes_proofs: Vec<(ChunkInfo, ChunkProof)> = batch_task_detail - .chunk_infos - .clone() - .into_iter() - .zip(batch_task_detail.batch_proving_task.chunk_proofs.clone()) - .collect(); - - let chunk_proofs: Vec = - chunk_hashes_proofs.iter().map(|t| t.1.clone()).collect(); - - let is_valid = prover.borrow_mut().check_protocol_of_chunks(&chunk_proofs); - - if !is_valid { - bail!("non-match chunk protocol") - } - check_chunk_hashes("", &chunk_hashes_proofs).context("failed to check chunk info")?; - let batch_proof = prover.borrow_mut().gen_batch_proof( - batch_task_detail.batch_proving_task, - None, - self.get_output_dir(), - )?; - - return Ok(batch_proof); - } - unreachable!("please check errors in proof_type logic") - } - - fn gen_batch_proof(&self, task: &crate::types::Task) -> Result { - log::info!("[circuit] gen_batch_proof for task {}", task.id); - - let batch_task_detail: BatchTaskDetail = serde_json::from_str(&task.task_data)?; - let batch_proof = self.gen_batch_proof_raw(batch_task_detail)?; - Ok(serde_json::to_string(&batch_proof)?) - } - - fn gen_bundle_proof_raw(&self, bundle_task_detail: BundleTaskDetail) -> Result { - if let Some(prover) = self.batch_prover.as_ref() { - let bundle_proof = prover.borrow_mut().gen_bundle_proof( - bundle_task_detail, - None, - self.get_output_dir(), - )?; - - return Ok(bundle_proof); - } - unreachable!("please check errors in proof_type logic") - } - - fn gen_bundle_proof(&self, task: &crate::types::Task) -> Result { - log::info!("[circuit] gen_bundle_proof for task {}", task.id); - let bundle_task_detail: BundleTaskDetail = serde_json::from_str(&task.task_data)?; - let bundle_proof = self.gen_bundle_proof_raw(bundle_task_detail)?; - Ok(serde_json::to_string(&bundle_proof)?) - } - - fn get_output_dir(&self) -> Option<&str> { - OUTPUT_DIR.as_deref() - } - - fn gen_chunk_traces(&self, task: &Task) -> Result> { - let chunk_task_detail: ChunkTaskDetail = serde_json::from_str(&task.task_data)?; - self.get_sorted_traces_by_hashes(&chunk_task_detail.block_hashes) - } - - fn get_sorted_traces_by_hashes(&self, block_hashes: &[CommonHash]) -> Result> { - if block_hashes.is_empty() { - log::error!("[prover] failed to get sorted traces: block_hashes are empty"); - bail!("block_hashes are empty") - } - - let mut block_traces = Vec::new(); - for hash in block_hashes.iter() { - let trace = self - .geth_client - .as_ref() - .unwrap() - .borrow_mut() - .get_block_trace_by_hash(hash)?; - block_traces.push(trace); - } - - block_traces.sort_by(|a, b| { - if get_block_number(a).is_none() { - Ordering::Less - } else if get_block_number(b).is_none() { - Ordering::Greater - } else { - get_block_number(a) - .unwrap() - .cmp(&get_block_number(b).unwrap()) - } - }); - - let block_numbers: Vec = block_traces - .iter() - .map(|trace| get_block_number(trace).unwrap_or(0)) - .collect(); - let mut i = 0; - while i < block_numbers.len() - 1 { - if block_numbers[i] + 1 != block_numbers[i + 1] { - log::error!( - "[prover] block numbers are not continuous, got {} and {}", - block_numbers[i], - block_numbers[i + 1] - ); - bail!( - "block numbers are not continuous, got {} and {}", - block_numbers[i], - block_numbers[i + 1] - ) - } - i += 1; - } - - Ok(block_traces) - } -} - -impl CircuitsHandler for DarwinHandler { - fn get_vk(&self, task_type: TaskType) -> Option> { - match task_type { - TaskType::Chunk => self - .chunk_prover - .as_ref() - .and_then(|prover| prover.borrow().get_vk()), - TaskType::Batch => self - .batch_prover - .as_ref() - .and_then(|prover| prover.borrow().get_batch_vk()), - TaskType::Bundle => self - .batch_prover - .as_ref() - .and_then(|prover| prover.borrow().get_bundle_vk()), - _ => unreachable!(), - } - } - - fn get_proof_data(&self, task_type: TaskType, task: &crate::types::Task) -> Result { - match task_type { - TaskType::Chunk => self.gen_chunk_proof(task), - TaskType::Batch => self.gen_batch_proof(task), - TaskType::Bundle => self.gen_bundle_proof(task), - _ => unreachable!(), - } - } -} - -// =================================== tests module ======================================== - -#[cfg(test)] -mod tests { - use super::*; - use crate::zk_circuits_handler::utils::encode_vk; - use prover_darwin::utils::chunk_trace_to_witness_block; - use std::{path::PathBuf, sync::LazyLock}; - - #[ctor::ctor] - fn init() { - crate::utils::log_init(None); - log::info!("logger initialized"); - } - - static DEFAULT_WORK_DIR: &str = "/assets"; - static WORK_DIR: LazyLock = LazyLock::new(|| { - std::env::var("DARWIN_TEST_DIR") - .unwrap_or(String::from(DEFAULT_WORK_DIR)) - .trim_end_matches('/') - .to_string() - }); - static PARAMS_PATH: LazyLock = LazyLock::new(|| format!("{}/test_params", *WORK_DIR)); - static ASSETS_PATH: LazyLock = LazyLock::new(|| format!("{}/test_assets", *WORK_DIR)); - static PROOF_DUMP_PATH: LazyLock = - LazyLock::new(|| format!("{}/proof_data", *WORK_DIR)); - static BATCH_DIR_PATH: LazyLock = - LazyLock::new(|| format!("{}/traces/batch_24", *WORK_DIR)); - static BATCH_VK_PATH: LazyLock = - LazyLock::new(|| format!("{}/test_assets/vk_batch.vkey", *WORK_DIR)); - static CHUNK_VK_PATH: LazyLock = - LazyLock::new(|| format!("{}/test_assets/vk_chunk.vkey", *WORK_DIR)); - - #[test] - fn it_works() { - let result = true; - assert!(result); - } - - #[test] - fn test_circuits() -> Result<()> { - let bi_handler = DarwinHandler::new_multi( - vec![ProverType::Chunk, ProverType::Batch], - &PARAMS_PATH, - &ASSETS_PATH, - None, - )?; - - let chunk_handler = bi_handler; - let chunk_vk = chunk_handler.get_vk(TaskType::Chunk).unwrap(); - - check_vk(TaskType::Chunk, chunk_vk, "chunk vk must be available"); - let chunk_dir_paths = get_chunk_dir_paths()?; - log::info!("chunk_dir_paths, {:?}", chunk_dir_paths); - let mut chunk_infos = vec![]; - let mut chunk_proofs = vec![]; - for (id, chunk_path) in chunk_dir_paths.into_iter().enumerate() { - let chunk_id = format!("chunk_proof{}", id + 1); - log::info!("start to process {chunk_id}"); - let chunk_trace = read_chunk_trace(chunk_path)?; - - let chunk_info = traces_to_chunk_info(chunk_trace.clone())?; - chunk_infos.push(chunk_info); - - log::info!("start to prove {chunk_id}"); - let chunk_proof = chunk_handler.gen_chunk_proof_raw(chunk_trace)?; - let proof_data = serde_json::to_string(&chunk_proof)?; - dump_proof(chunk_id, proof_data)?; - chunk_proofs.push(chunk_proof); - } - - let batch_handler = chunk_handler; - let batch_vk = batch_handler.get_vk(TaskType::Batch).unwrap(); - check_vk(TaskType::Batch, batch_vk, "batch vk must be available"); - let batch_task_detail = make_batch_task_detail(chunk_infos, chunk_proofs); - log::info!("start to prove batch"); - let batch_proof = batch_handler.gen_batch_proof_raw(batch_task_detail)?; - let proof_data = serde_json::to_string(&batch_proof)?; - dump_proof("batch_proof".to_string(), proof_data)?; - - Ok(()) - } - - fn make_batch_task_detail(_: Vec, _: Vec) -> BatchTaskDetail { - todo!(); - // BatchTaskDetail { - // chunk_infos, - // batch_proving_task: BatchProvingTask { - // parent_batch_hash: todo!(), - // parent_state_root: todo!(), - // batch_header: todo!(), - // chunk_proofs, - // }, - // } - } - - fn check_vk(proof_type: TaskType, vk: Vec, info: &str) { - log::info!("check_vk, {:?}", proof_type); - let vk_from_file = read_vk(proof_type).unwrap(); - assert_eq!(vk_from_file, encode_vk(vk), "{info}") - } - - fn read_vk(proof_type: TaskType) -> Result { - log::info!("read_vk, {:?}", proof_type); - let vk_file = match proof_type { - TaskType::Chunk => CHUNK_VK_PATH.clone(), - TaskType::Batch => BATCH_VK_PATH.clone(), - TaskType::Bundle => todo!(), - TaskType::Undefined => unreachable!(), - }; - - let data = std::fs::read(vk_file)?; - Ok(encode_vk(data)) - } - - fn read_chunk_trace(path: PathBuf) -> Result> { - log::info!("read_chunk_trace, {:?}", path); - let mut chunk_trace: Vec = vec![]; - - fn read_block_trace(file: &PathBuf) -> Result { - let f = std::fs::File::open(file)?; - Ok(serde_json::from_reader(&f)?) - } - - if path.is_dir() { - let entries = std::fs::read_dir(&path)?; - let mut files: Vec = entries - .into_iter() - .filter_map(|e| { - if e.is_err() { - return None; - } - let entry = e.unwrap(); - if entry.path().is_dir() { - return None; - } - if let Result::Ok(file_name) = entry.file_name().into_string() { - Some(file_name) - } else { - None - } - }) - .collect(); - files.sort(); - - log::info!("files in chunk {:?} is {:?}", path, files); - for file in files { - let block_trace = read_block_trace(&path.join(file))?; - chunk_trace.push(block_trace); - } - } else { - let block_trace = read_block_trace(&path)?; - chunk_trace.push(block_trace); - } - Ok(chunk_trace) - } - - fn get_chunk_dir_paths() -> Result> { - let batch_path = PathBuf::from(BATCH_DIR_PATH.clone()); - let entries = std::fs::read_dir(&batch_path)?; - let mut files: Vec = entries - .filter_map(|e| { - if e.is_err() { - return None; - } - let entry = e.unwrap(); - if entry.path().is_dir() { - if let Result::Ok(file_name) = entry.file_name().into_string() { - Some(file_name) - } else { - None - } - } else { - None - } - }) - .collect(); - files.sort(); - log::info!("files in batch {:?} is {:?}", batch_path, files); - Ok(files.into_iter().map(|f| batch_path.join(f)).collect()) - } - - fn traces_to_chunk_info(chunk_trace: Vec) -> Result { - let witness_block = chunk_trace_to_witness_block(chunk_trace)?; - Ok(ChunkInfo::from_witness_block(&witness_block, false)) - } - - fn dump_proof(id: String, proof_data: String) -> Result<()> { - let dump_path = PathBuf::from(PROOF_DUMP_PATH.clone()); - Ok(std::fs::write(dump_path.join(id), proof_data)?) - } -} diff --git a/prover/src/zk_circuits_handler/darwin_v2.rs b/prover/src/zk_circuits_handler/darwin_v2.rs deleted file mode 100644 index fce3871d0..000000000 --- a/prover/src/zk_circuits_handler/darwin_v2.rs +++ /dev/null @@ -1,525 +0,0 @@ -use super::{common::*, CircuitsHandler}; -use crate::{ - geth_client::GethClient, - types::{ProverType, TaskType}, -}; -use anyhow::{bail, Context, Ok, Result}; -use once_cell::sync::Lazy; -use serde::Deserialize; - -use crate::types::{CommonHash, Task}; -use std::{cell::RefCell, cmp::Ordering, env, rc::Rc}; - -use prover_darwin_v2::{ - aggregator::Prover as BatchProver, - check_chunk_hashes, - common::Prover as CommonProver, - config::{AGG_DEGREES, ZKEVM_DEGREES}, - zkevm::Prover as ChunkProver, - BatchProof, BatchProvingTask, BlockTrace, BundleProof, BundleProvingTask, ChunkInfo, - ChunkProof, ChunkProvingTask, -}; - -// Only used for debugging. -static OUTPUT_DIR: Lazy> = Lazy::new(|| env::var("PROVER_OUTPUT_DIR").ok()); - -#[derive(Debug, Clone, Deserialize)] -pub struct BatchTaskDetail { - pub chunk_infos: Vec, - #[serde(flatten)] - pub batch_proving_task: BatchProvingTask, -} - -type BundleTaskDetail = BundleProvingTask; - -#[derive(Debug, Clone, Deserialize)] -pub struct ChunkTaskDetail { - pub block_hashes: Vec, -} - -fn get_block_number(block_trace: &BlockTrace) -> Option { - block_trace.header.number.map(|n| n.as_u64()) -} - -#[derive(Default)] -pub struct DarwinV2Handler { - chunk_prover: Option>>, - batch_prover: Option>>, - - geth_client: Option>>, -} - -impl DarwinV2Handler { - pub fn new_multi( - prover_types: Vec, - params_dir: &str, - assets_dir: &str, - geth_client: Option>>, - ) -> Result { - let class_name = std::intrinsics::type_name::(); - let prover_types_set = prover_types - .into_iter() - .collect::>(); - let mut handler = Self { - batch_prover: None, - chunk_prover: None, - geth_client, - }; - let degrees: Vec = get_degrees(&prover_types_set, |prover_type| match prover_type { - ProverType::Chunk => ZKEVM_DEGREES.clone(), - ProverType::Batch => AGG_DEGREES.clone(), - }); - let params_map = get_params_map_instance(|| { - log::info!( - "calling get_params_map from {}, prover_types: {:?}, degrees: {:?}", - class_name, - prover_types_set, - degrees - ); - CommonProver::load_params_map(params_dir, °rees) - }); - for prover_type in prover_types_set { - match prover_type { - ProverType::Chunk => { - handler.chunk_prover = Some(RefCell::new(ChunkProver::from_params_and_assets( - params_map, assets_dir, - ))); - } - ProverType::Batch => { - handler.batch_prover = Some(RefCell::new(BatchProver::from_params_and_assets( - params_map, assets_dir, - ))) - } - } - } - Ok(handler) - } - - pub fn new( - prover_type: ProverType, - params_dir: &str, - assets_dir: &str, - geth_client: Option>>, - ) -> Result { - Self::new_multi(vec![prover_type], params_dir, assets_dir, geth_client) - } - - fn gen_chunk_proof_raw(&self, chunk_trace: Vec) -> Result { - if let Some(prover) = self.chunk_prover.as_ref() { - let chunk = ChunkProvingTask::from(chunk_trace); - - let chunk_proof = - prover - .borrow_mut() - .gen_chunk_proof(chunk, None, None, self.get_output_dir())?; - - return Ok(chunk_proof); - } - unreachable!("please check errors in proof_type logic") - } - - fn gen_chunk_proof(&self, task: &crate::types::Task) -> Result { - let chunk_trace = self.gen_chunk_traces(task)?; - let chunk_proof = self.gen_chunk_proof_raw(chunk_trace)?; - Ok(serde_json::to_string(&chunk_proof)?) - } - - fn gen_batch_proof_raw(&self, batch_task_detail: BatchTaskDetail) -> Result { - if let Some(prover) = self.batch_prover.as_ref() { - let chunk_hashes_proofs: Vec<(ChunkInfo, ChunkProof)> = batch_task_detail - .chunk_infos - .clone() - .into_iter() - .zip(batch_task_detail.batch_proving_task.chunk_proofs.clone()) - .collect(); - - let chunk_proofs: Vec = - chunk_hashes_proofs.iter().map(|t| t.1.clone()).collect(); - - let is_valid = prover.borrow_mut().check_protocol_of_chunks(&chunk_proofs); - - if !is_valid { - bail!("non-match chunk protocol") - } - check_chunk_hashes("", &chunk_hashes_proofs).context("failed to check chunk info")?; - let batch_proof = prover.borrow_mut().gen_batch_proof( - batch_task_detail.batch_proving_task, - None, - self.get_output_dir(), - )?; - - return Ok(batch_proof); - } - unreachable!("please check errors in proof_type logic") - } - - fn gen_batch_proof(&self, task: &crate::types::Task) -> Result { - log::info!("[circuit] gen_batch_proof for task {}", task.id); - - let batch_task_detail: BatchTaskDetail = serde_json::from_str(&task.task_data)?; - let batch_proof = self.gen_batch_proof_raw(batch_task_detail)?; - Ok(serde_json::to_string(&batch_proof)?) - } - - fn gen_bundle_proof_raw(&self, bundle_task_detail: BundleTaskDetail) -> Result { - if let Some(prover) = self.batch_prover.as_ref() { - let bundle_proof = prover.borrow_mut().gen_bundle_proof( - bundle_task_detail, - None, - self.get_output_dir(), - )?; - - return Ok(bundle_proof); - } - unreachable!("please check errors in proof_type logic") - } - - fn gen_bundle_proof(&self, task: &crate::types::Task) -> Result { - log::info!("[circuit] gen_bundle_proof for task {}", task.id); - let bundle_task_detail: BundleTaskDetail = serde_json::from_str(&task.task_data)?; - let bundle_proof = self.gen_bundle_proof_raw(bundle_task_detail)?; - Ok(serde_json::to_string(&bundle_proof)?) - } - - fn get_output_dir(&self) -> Option<&str> { - OUTPUT_DIR.as_deref() - } - - fn gen_chunk_traces(&self, task: &Task) -> Result> { - let chunk_task_detail: ChunkTaskDetail = serde_json::from_str(&task.task_data)?; - self.get_sorted_traces_by_hashes(&chunk_task_detail.block_hashes) - } - - fn get_sorted_traces_by_hashes(&self, block_hashes: &[CommonHash]) -> Result> { - if block_hashes.is_empty() { - log::error!("[prover] failed to get sorted traces: block_hashes are empty"); - bail!("block_hashes are empty") - } - - let mut block_traces = Vec::new(); - for hash in block_hashes.iter() { - let trace = self - .geth_client - .as_ref() - .unwrap() - .borrow_mut() - .get_block_trace_by_hash(hash)?; - block_traces.push(trace); - } - - block_traces.sort_by(|a, b| { - if get_block_number(a).is_none() { - Ordering::Less - } else if get_block_number(b).is_none() { - Ordering::Greater - } else { - get_block_number(a) - .unwrap() - .cmp(&get_block_number(b).unwrap()) - } - }); - - let block_numbers: Vec = block_traces - .iter() - .map(|trace| get_block_number(trace).unwrap_or(0)) - .collect(); - let mut i = 0; - while i < block_numbers.len() - 1 { - if block_numbers[i] + 1 != block_numbers[i + 1] { - log::error!( - "[prover] block numbers are not continuous, got {} and {}", - block_numbers[i], - block_numbers[i + 1] - ); - bail!( - "block numbers are not continuous, got {} and {}", - block_numbers[i], - block_numbers[i + 1] - ) - } - i += 1; - } - - Ok(block_traces) - } -} - -impl CircuitsHandler for DarwinV2Handler { - fn get_vk(&self, task_type: TaskType) -> Option> { - match task_type { - TaskType::Chunk => self - .chunk_prover - .as_ref() - .and_then(|prover| prover.borrow().get_vk()), - TaskType::Batch => self - .batch_prover - .as_ref() - .and_then(|prover| prover.borrow().get_batch_vk()), - TaskType::Bundle => self - .batch_prover - .as_ref() - .and_then(|prover| prover.borrow().get_bundle_vk()), - _ => unreachable!(), - } - } - - fn get_proof_data(&self, task_type: TaskType, task: &crate::types::Task) -> Result { - match task_type { - TaskType::Chunk => self.gen_chunk_proof(task), - TaskType::Batch => self.gen_batch_proof(task), - TaskType::Bundle => self.gen_bundle_proof(task), - _ => unreachable!(), - } - } -} - -// =================================== tests module ======================================== - -#[cfg(test)] -mod tests { - use super::*; - use crate::zk_circuits_handler::utils::encode_vk; - use ethers_core::types::H256; - use prover_darwin_v2::{ - aggregator::eip4844, utils::chunk_trace_to_witness_block, BatchData, BatchHeader, - MAX_AGG_SNARKS, - }; - use std::{path::PathBuf, sync::LazyLock}; - - #[ctor::ctor] - fn init() { - crate::utils::log_init(None); - log::info!("logger initialized"); - } - - static DEFAULT_WORK_DIR: &str = "/assets"; - static WORK_DIR: LazyLock = LazyLock::new(|| { - std::env::var("DARWIN_V2_TEST_DIR") - .unwrap_or(String::from(DEFAULT_WORK_DIR)) - .trim_end_matches('/') - .to_string() - }); - static PARAMS_PATH: LazyLock = LazyLock::new(|| format!("{}/test_params", *WORK_DIR)); - static ASSETS_PATH: LazyLock = LazyLock::new(|| format!("{}/test_assets", *WORK_DIR)); - static PROOF_DUMP_PATH: LazyLock = - LazyLock::new(|| format!("{}/proof_data", *WORK_DIR)); - static BATCH_DIR_PATH: LazyLock = - LazyLock::new(|| format!("{}/traces/batch_24", *WORK_DIR)); - static BATCH_VK_PATH: LazyLock = - LazyLock::new(|| format!("{}/test_assets/vk_batch.vkey", *WORK_DIR)); - static CHUNK_VK_PATH: LazyLock = - LazyLock::new(|| format!("{}/test_assets/vk_chunk.vkey", *WORK_DIR)); - - #[test] - fn it_works() { - let result = true; - assert!(result); - } - - #[test] - fn test_circuits() -> Result<()> { - let bi_handler = DarwinV2Handler::new_multi( - vec![ProverType::Chunk, ProverType::Batch], - &PARAMS_PATH, - &ASSETS_PATH, - None, - )?; - - let chunk_handler = bi_handler; - let chunk_vk = chunk_handler.get_vk(TaskType::Chunk).unwrap(); - - check_vk(TaskType::Chunk, chunk_vk, "chunk vk must be available"); - let chunk_dir_paths = get_chunk_dir_paths()?; - log::info!("chunk_dir_paths, {:?}", chunk_dir_paths); - let mut chunk_traces = vec![]; - let mut chunk_infos = vec![]; - let mut chunk_proofs = vec![]; - for (id, chunk_path) in chunk_dir_paths.into_iter().enumerate() { - let chunk_id = format!("chunk_proof{}", id + 1); - log::info!("start to process {chunk_id}"); - let chunk_trace = read_chunk_trace(chunk_path)?; - chunk_traces.push(chunk_trace.clone()); - let chunk_info = traces_to_chunk_info(chunk_trace.clone())?; - chunk_infos.push(chunk_info); - - log::info!("start to prove {chunk_id}"); - let chunk_proof = chunk_handler.gen_chunk_proof_raw(chunk_trace)?; - let proof_data = serde_json::to_string(&chunk_proof)?; - dump_proof(chunk_id, proof_data)?; - chunk_proofs.push(chunk_proof); - } - - let batch_handler = chunk_handler; - let batch_vk = batch_handler.get_vk(TaskType::Batch).unwrap(); - check_vk(TaskType::Batch, batch_vk, "batch vk must be available"); - let batch_task_detail = make_batch_task_detail(chunk_traces, chunk_proofs, None); - log::info!("start to prove batch"); - let batch_proof = batch_handler.gen_batch_proof_raw(batch_task_detail)?; - let proof_data = serde_json::to_string(&batch_proof)?; - dump_proof("batch_proof".to_string(), proof_data)?; - - Ok(()) - } - - // copied from https://github.com/scroll-tech/scroll-prover/blob/main/integration/src/prove.rs - fn get_blob_from_chunks(chunks: &[ChunkInfo]) -> Vec { - let num_chunks = chunks.len(); - - let padded_chunk = - ChunkInfo::mock_padded_chunk_info_for_testing(chunks.last().as_ref().unwrap()); - let chunks_with_padding = [ - chunks.to_vec(), - vec![padded_chunk; MAX_AGG_SNARKS - num_chunks], - ] - .concat(); - let batch_data = BatchData::<{ MAX_AGG_SNARKS }>::new(chunks.len(), &chunks_with_padding); - let batch_bytes = batch_data.get_batch_data_bytes(); - let blob_bytes = eip4844::get_blob_bytes(&batch_bytes); - log::info!("blob_bytes len {}", blob_bytes.len()); - blob_bytes - } - - // TODO: chunk_infos can be extracted from chunk_proofs. - // Still needed? - fn make_batch_task_detail( - chunk_traces: Vec>, - chunk_proofs: Vec, - last_batcher_header: Option>, - ) -> BatchTaskDetail { - // dummy parent batch hash - let dummy_parent_batch_hash = H256([ - 0xab, 0xac, 0xad, 0xae, 0xaf, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - ]); - let chunk_infos: Vec<_> = chunk_proofs.iter().map(|p| p.chunk_info.clone()).collect(); - - let l1_message_popped = chunk_traces - .iter() - .flatten() - .map(|chunk| chunk.num_l1_txs()) - .sum(); - let last_block_timestamp = chunk_traces.last().map_or(0, |block_traces| { - block_traces - .last() - .map_or(0, |block_trace| block_trace.header.timestamp.as_u64()) - }); - - let blob_bytes = get_blob_from_chunks(&chunk_infos); - let batch_header = BatchHeader::construct_from_chunks( - last_batcher_header.map_or(4, |header| header.version), - last_batcher_header.map_or(123, |header| header.batch_index + 1), - l1_message_popped, - last_batcher_header.map_or(l1_message_popped, |header| { - header.total_l1_message_popped + l1_message_popped - }), - last_batcher_header.map_or(dummy_parent_batch_hash, |header| header.batch_hash()), - last_block_timestamp, - &chunk_infos, - &blob_bytes, - ); - BatchTaskDetail { - chunk_infos, - batch_proving_task: BatchProvingTask { - chunk_proofs, - batch_header, - blob_bytes, - }, - } - } - - fn check_vk(proof_type: TaskType, vk: Vec, info: &str) { - log::info!("check_vk, {:?}", proof_type); - let vk_from_file = read_vk(proof_type).unwrap(); - assert_eq!(vk_from_file, encode_vk(vk), "{info}") - } - - fn read_vk(proof_type: TaskType) -> Result { - log::info!("read_vk, {:?}", proof_type); - let vk_file = match proof_type { - TaskType::Chunk => CHUNK_VK_PATH.clone(), - TaskType::Batch => BATCH_VK_PATH.clone(), - TaskType::Bundle => todo!(), - TaskType::Undefined => unreachable!(), - }; - - let data = std::fs::read(vk_file)?; - Ok(encode_vk(data)) - } - - fn read_chunk_trace(path: PathBuf) -> Result> { - log::info!("read_chunk_trace, {:?}", path); - let mut chunk_trace: Vec = vec![]; - - fn read_block_trace(file: &PathBuf) -> Result { - let f = std::fs::File::open(file)?; - Ok(serde_json::from_reader(&f)?) - } - - if path.is_dir() { - let entries = std::fs::read_dir(&path)?; - let mut files: Vec = entries - .into_iter() - .filter_map(|e| { - if e.is_err() { - return None; - } - let entry = e.unwrap(); - if entry.path().is_dir() { - return None; - } - if let Result::Ok(file_name) = entry.file_name().into_string() { - Some(file_name) - } else { - None - } - }) - .collect(); - files.sort(); - - log::info!("files in chunk {:?} is {:?}", path, files); - for file in files { - let block_trace = read_block_trace(&path.join(file))?; - chunk_trace.push(block_trace); - } - } else { - let block_trace = read_block_trace(&path)?; - chunk_trace.push(block_trace); - } - Ok(chunk_trace) - } - - fn get_chunk_dir_paths() -> Result> { - let batch_path = PathBuf::from(BATCH_DIR_PATH.clone()); - let entries = std::fs::read_dir(&batch_path)?; - let mut files: Vec = entries - .filter_map(|e| { - if e.is_err() { - return None; - } - let entry = e.unwrap(); - if entry.path().is_dir() { - if let Result::Ok(file_name) = entry.file_name().into_string() { - Some(file_name) - } else { - None - } - } else { - None - } - }) - .collect(); - files.sort(); - log::info!("files in batch {:?} is {:?}", batch_path, files); - Ok(files.into_iter().map(|f| batch_path.join(f)).collect()) - } - - fn traces_to_chunk_info(chunk_trace: Vec) -> Result { - let witness_block = chunk_trace_to_witness_block(chunk_trace)?; - Ok(ChunkInfo::from_witness_block(&witness_block, false)) - } - - fn dump_proof(id: String, proof_data: String) -> Result<()> { - let dump_path = PathBuf::from(PROOF_DUMP_PATH.clone()); - Ok(std::fs::write(dump_path.join(id), proof_data)?) - } -}