github/scroll
1
1
Fork 0
mirror of https://github.com/scroll-tech/scroll.git synced 2026-04-23 03:00:50 -04:00
Code Issues Packages Projects Releases Wiki Activity

deprecate halo2 provers, start openvm prover

Browse Source
This commit is contained in:
Ömer Faruk Irmak
2025-01-27 13:39:18 +03:00
parent ca8d930bd6
commit 421afe9c30
13 changed files with 111 additions and 1393 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
coordinator/internal/types/prover.go
View File

@@ -27,6 +27,8 @@ const (
ProverTypeChunk
// ProverTypeBatch signals it's a batch prover, which can prove batch_tasks and bundle_tasks
ProverTypeBatch
// ProverTypeOpenVM signals it's an openvm prover
ProverTypeOpenVM
)
// MakeProverType make ProverType from ProofType

68
prover/src/config.rs
View File

@@ -1,17 +1,16 @@
use anyhow::{bail, Result};
use anyhow::Result;
use serde::{Deserialize, Serialize};
use std::fs::File;
use std::{collections::HashMap, fs::File};
use crate::types::ProverType;
#[derive(Debug, Serialize, Deserialize)]
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct CircuitConfig {
pub hard_fork_name: String,
pub params_path: String,
pub assets_path: String,
pub workspace_path: String,
}
#[derive(Debug, Serialize, Deserialize)]
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct CoordinatorConfig {
pub base_url: String,
pub retry_count: u32,
@@ -19,20 +18,19 @@ pub struct CoordinatorConfig {
pub connection_timeout_sec: u64,
}
#[derive(Debug, Serialize, Deserialize)]
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct L2GethConfig {
pub endpoint: String,
}
#[derive(Debug, Deserialize)]
#[derive(Debug, Deserialize, Clone)]
pub struct Config {
pub prover_name: String,
pub keystore_path: String,
pub keystore_password: String,
pub db_path: String,
pub prover_type: ProverType,
pub low_version_circuit: CircuitConfig,
pub high_version_circuit: CircuitConfig,
pub circuits: HashMap<String, CircuitConfig>,
pub coordinator: CoordinatorConfig,
pub l2geth: Option<L2GethConfig>,
}
@@ -50,53 +48,3 @@ impl Config {
Config::from_reader(&file)
}
}
static SCROLL_PROVER_ASSETS_DIR_ENV_NAME: &str = "SCROLL_PROVER_ASSETS_DIR";
static mut SCROLL_PROVER_ASSETS_DIRS: Vec<String> = vec![];
#[derive(Debug)]
pub struct AssetsDirEnvConfig {}
impl AssetsDirEnvConfig {
pub fn init() -> Result<()> {
let value = std::env::var(SCROLL_PROVER_ASSETS_DIR_ENV_NAME)?;
let dirs: Vec<&str> = value.split(',').collect();
if dirs.len() != 2 {
bail!("env variable SCROLL_PROVER_ASSETS_DIR value must be 2 parts seperated by comma.")
}
unsafe {
SCROLL_PROVER_ASSETS_DIRS = dirs.into_iter().map(|s| s.to_string()).collect();
log::info!(
"init SCROLL_PROVER_ASSETS_DIRS: {:?}",
SCROLL_PROVER_ASSETS_DIRS
);
}
Ok(())
}
pub fn enable_first() {
unsafe {
log::info!(
"set env {SCROLL_PROVER_ASSETS_DIR_ENV_NAME} to {}",
&SCROLL_PROVER_ASSETS_DIRS[0]
);
std::env::set_var(
SCROLL_PROVER_ASSETS_DIR_ENV_NAME,
&SCROLL_PROVER_ASSETS_DIRS[0],
);
}
}
pub fn enable_second() {
unsafe {
log::info!(
"set env {SCROLL_PROVER_ASSETS_DIR_ENV_NAME} to {}",
&SCROLL_PROVER_ASSETS_DIRS[1]
);
std::env::set_var(
SCROLL_PROVER_ASSETS_DIR_ENV_NAME,
&SCROLL_PROVER_ASSETS_DIRS[1],
);
}
}
}

13
prover/src/coordinator_client.rs
View File

@@ -4,7 +4,6 @@ pub mod listener;
pub mod types;
use anyhow::{bail, Context, Ok, Result};
use std::rc::Rc;
use api::Api;
use errors::*;
@@ -16,20 +15,20 @@ use crate::{config::Config, key_signer::KeySigner};
pub use errors::ProofStatusNotOKError;
pub struct CoordinatorClient<'a> {
pub struct CoordinatorClient {
api: Api,
token: Option<String>,
config: &'a Config,
key_signer: Rc<KeySigner>,
config: Config,
key_signer: KeySigner,
rt: Runtime,
listener: Box<dyn Listener>,
vks: Vec<String>,
}
impl<'a> CoordinatorClient<'a> {
impl CoordinatorClient {
pub fn new(
config: &'a Config,
key_signer: Rc<KeySigner>,
config: Config,
key_signer: KeySigner,
listener: Box<dyn Listener>,
vks: Vec<String>,
) -> Result<Self> {

22
prover/src/geth_client.rs
View File

@@ -1,11 +1,7 @@
use crate::types::CommonHash;
use anyhow::Result;
use ethers_core::types::BlockNumber;
use tokio::runtime::Runtime;
use serde::{de::DeserializeOwned, Serialize};
use std::fmt::Debug;
use ethers_providers::{Http, Provider};
pub struct GethClient {
@@ -28,24 +24,6 @@ impl GethClient {
})
}
pub fn get_block_trace_by_hash<T>(&mut self, hash: &CommonHash) -> Result<T>
where
T: Serialize + DeserializeOwned + Debug + Send,
{
log::info!(
"{}: calling get_block_trace_by_hash, hash: {:#?}",
self.id,
hash
);
let trace_future = self
.provider
.request("scroll_getBlockTraceByNumberOrHash", [format!("{hash:#x}")]);
let trace = self.rt.block_on(trace_future)?;
Ok(trace)
}
pub fn block_number(&mut self) -> Result<BlockNumber> {
log::info!("{}: calling block_number", self.id);

16
prover/src/main.rs
View File

@@ -15,7 +15,7 @@ mod zk_circuits_handler;
use anyhow::Result;
use clap::{ArgAction, Parser};
use config::{AssetsDirEnvConfig, Config};
use config::Config;
use prover::Prover;
use std::rc::Rc;
use task_cache::{ClearCacheCoordinatorListener, TaskCache};
@@ -49,29 +49,23 @@ fn start() -> Result<()> {
utils::log_init(args.log_file);
let config: Config = Config::from_file(args.config_file)?;
if let Err(e) = AssetsDirEnvConfig::init() {
log::error!("AssetsDirEnvConfig init failed: {:#}", e);
std::process::exit(-2);
}
let task_cache = Rc::new(TaskCache::new(&config.db_path)?);
let coordinator_listener = Box::new(ClearCacheCoordinatorListener {
task_cache: task_cache.clone(),
});
let prover = Prover::new(&config, coordinator_listener)?;
let prover = Prover::new(config.clone(), coordinator_listener)?;
log::info!(
"prover start successfully. name: {}, type: {:?}, publickey: {}, version: {}",
"starting prover. name: {}, type: {:?}, publickey: {}, version: {}",
config.prover_name,
config.prover_type,
prover.get_public_key(),
prover.public_key,
version::get_version(),
);
let task_processor = TaskProcessor::new(&prover, task_cache);
let task_processor = TaskProcessor::new(prover, task_cache);
task_processor.start();

150
prover/src/prover.rs
View File

@@ -1,91 +1,76 @@
use anyhow::{bail, Context, Error, Ok, Result};
use ethers_core::types::U64;
use std::{cell::RefCell, rc::Rc};
use crate::{
config::Config,
coordinator_client::{listener::Listener, types::*, CoordinatorClient},
geth_client::GethClient,
key_signer::KeySigner,
types::{ProofFailureType, ProofStatus, ProverType},
utils::get_task_types,
zk_circuits_handler::{CircuitsHandler, CircuitsHandlerProvider},
types::{ProofFailureType, ProofStatus, TaskType},
zk_circuits_handler::{euclid::EuclidHandler, CircuitsHandler},
};
use super::types::{ProofDetail, Task};
pub struct Prover<'a> {
config: &'a Config,
key_signer: Rc<KeySigner>,
circuits_handler_provider: RefCell<CircuitsHandlerProvider<'a>>,
coordinator_client: RefCell<CoordinatorClient<'a>>,
geth_client: Option<Rc<RefCell<GethClient>>>,
pub struct Prover {
config: Config,
pub public_key: String,
coordinator_client: CoordinatorClient,
geth_client: GethClient,
active_handler: Option<(String, Box<dyn CircuitsHandler>)>,
}
impl<'a> Prover<'a> {
pub fn new(config: &'a Config, coordinator_listener: Box<dyn Listener>) -> Result<Self> {
let prover_type = config.prover_type;
impl Prover {
pub fn new(config: Config, coordinator_listener: Box<dyn Listener>) -> Result<Self> {
let keystore_path = &config.keystore_path;
let keystore_password = &config.keystore_password;
let geth_client = if config.prover_type == ProverType::Chunk {
Some(Rc::new(RefCell::new(
GethClient::new(
&config.prover_name,
&config.l2geth.as_ref().unwrap().endpoint,
)
.context("failed to create l2 geth_client")?,
)))
} else {
None
};
let geth_client = GethClient::new(
&config.prover_name,
&config.l2geth.as_ref().unwrap().endpoint,
)
.context("failed to create l2 geth_client")?;
let provider = CircuitsHandlerProvider::new(prover_type, config, geth_client.clone())
.context("failed to create circuits handler provider")?;
let key_signer = KeySigner::new(keystore_path, keystore_password)?;
let public_key = key_signer.get_public_key();
let vks = provider.init_vks(prover_type, config, geth_client.clone());
let key_signer = Rc::new(KeySigner::new(keystore_path, keystore_password)?);
let coordinator_client =
CoordinatorClient::new(config, Rc::clone(&key_signer), coordinator_listener, vks)
.context("failed to create coordinator_client")?;
let coordinator_client = CoordinatorClient::new(
config.clone(),
key_signer,
coordinator_listener,
vec![], /* todo: vks */
)
.context("failed to create coordinator_client")?;
let prover = Prover {
config,
key_signer: Rc::clone(&key_signer),
circuits_handler_provider: RefCell::new(provider),
coordinator_client: RefCell::new(coordinator_client),
public_key,
coordinator_client,
geth_client,
active_handler: None,
};
Ok(prover)
}
pub fn get_public_key(&self) -> String {
self.key_signer.get_public_key()
}
pub fn fetch_task(&self) -> Result<Task> {
pub fn fetch_task(&mut self) -> Result<Task> {
log::info!("[prover] start to fetch_task");
let mut req = GetTaskRequest {
task_types: get_task_types(self.config.prover_type),
task_types: vec![TaskType::Chunk, TaskType::Batch, TaskType::Bundle],
prover_height: None,
};
if self.config.prover_type == ProverType::Chunk {
let latest_block_number = self.get_latest_block_number_value()?;
if let Some(v) = latest_block_number {
if v.as_u64() == 0 {
bail!("omit to prove task of the genesis block")
}
req.prover_height = Some(v.as_u64());
} else {
log::error!("[prover] failed to fetch latest confirmed block number, got None");
bail!("failed to fetch latest confirmed block number, got None")
let latest_block_number = self.geth_client.block_number()?.as_number();
if let Some(v) = latest_block_number {
if v.as_u64() == 0 {
bail!("omit to prove task of the genesis block")
}
req.prover_height = Some(v.as_u64());
} else {
log::error!("[prover] failed to fetch latest confirmed block number, got None");
bail!("failed to fetch latest confirmed block number, got None")
}
let resp = self.coordinator_client.borrow_mut().get_task(&req)?;
let resp = self.coordinator_client.get_task(&req)?;
match resp.data {
Some(d) => Ok(Task::from(d)),
@@ -95,28 +80,43 @@ impl<'a> Prover<'a> {
}
}
pub fn prove_task(&self, task: &Task) -> Result<ProofDetail> {
log::info!("[prover] start to prove_task, task id: {}", task.id);
let handler: Rc<Box<dyn CircuitsHandler>> = self
.circuits_handler_provider
.borrow_mut()
.get_circuits_handler(&task.hard_fork_name)
.context("failed to get circuit handler")?;
self.do_prove(task, handler)
fn set_active_handler(&mut self, hard_fork_name: &str) {
if let Some(handler) = &self.active_handler {
if handler.0 == hard_fork_name {
return;
}
}
// if we got assigned a task for an unknown hard fork, there is something wrong in the
// coordinator
let config = self.config.circuits.get(hard_fork_name).unwrap();
let handler = Box::new(match hard_fork_name {
"euclid" => EuclidHandler::new(&config.workspace_path),
_ => unreachable!(),
}) as Box<dyn CircuitsHandler>;
self.active_handler = Some((hard_fork_name.to_string(), handler));
}
fn do_prove(&self, task: &Task, handler: Rc<Box<dyn CircuitsHandler>>) -> Result<ProofDetail> {
pub fn prove_task(&mut self, task: &Task) -> Result<ProofDetail> {
log::info!("[prover] start to prove_task, task id: {}", task.id);
let mut proof_detail = ProofDetail {
id: task.id.clone(),
proof_type: task.task_type,
..Default::default()
};
proof_detail.proof_data = handler.get_proof_data(task.task_type, task)?;
self.set_active_handler(&task.hard_fork_name);
proof_detail.proof_data = self
.active_handler
.as_ref()
.unwrap()
.1
.get_proof_data(task, &self.geth_client)?;
Ok(proof_detail)
}
pub fn submit_proof(&self, proof_detail: ProofDetail, task: &Task) -> Result<()> {
pub fn submit_proof(&mut self, proof_detail: ProofDetail, task: &Task) -> Result<()> {
log::info!(
"[prover] start to submit_proof, task id: {}",
proof_detail.id
@@ -131,11 +131,11 @@ impl<'a> Prover<'a> {
..Default::default()
};
self.do_submit(&request)
self.coordinator_client.submit_proof(&request).map(|_r| ())
}
pub fn submit_error(
&self,
&mut self,
task: &Task,
failure_type: ProofFailureType,
error: Error,
@@ -150,21 +150,7 @@ impl<'a> Prover<'a> {
failure_msg: Some(format!("{:#}", error)),
..Default::default()
};
self.do_submit(&request)
}
fn do_submit(&self, request: &SubmitProofRequest) -> Result<()> {
self.coordinator_client.borrow_mut().submit_proof(request)?;
Ok(())
}
fn get_latest_block_number_value(&self) -> Result<Option<U64>> {
let number = self
.geth_client
.as_ref()
.unwrap()
.borrow_mut()
.block_number()?;
Ok(number.as_number())
self.coordinator_client.submit_proof(&request).map(|_r| ())
}
}

14
prover/src/task_processor.rs
View File

@@ -2,20 +2,20 @@ use super::{coordinator_client::ProofStatusNotOKError, prover::Prover, task_cach
use anyhow::{Context, Result};
use std::rc::Rc;
pub struct TaskProcessor<'a> {
prover: &'a Prover<'a>,
pub struct TaskProcessor {
prover: Prover,
task_cache: Rc<TaskCache>,
}
impl<'a> TaskProcessor<'a> {
pub fn new(prover: &'a Prover<'a>, task_cache: Rc<TaskCache>) -> Self {
impl TaskProcessor {
pub fn new(prover: Prover, task_cache: Rc<TaskCache>) -> Self {
TaskProcessor { prover, task_cache }
}
pub fn start(&self) {
pub fn start(mut self) {
loop {
log::info!("start a new round.");
if let Err(err) = self.prove_and_submit() {
if let Err(err) = &self.prove_and_submit() {
if err.is::<ProofStatusNotOKError>() {
log::info!("proof status not ok, downgrade level to info.");
} else {
@@ -27,7 +27,7 @@ impl<'a> TaskProcessor<'a> {
}
}
fn prove_and_submit(&self) -> Result<()> {
fn prove_and_submit(&mut self) -> Result<()> {
let task_from_cache = self
.task_cache
.get_last_task()

20
prover/src/types.rs
View File

@@ -56,15 +56,17 @@ impl Default for TaskType {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum ProverType {
Chunk,
Batch,
Deprecated1,
Deprecated2,
OpenVM,
}
impl ProverType {
fn from_u8(v: u8) -> Self {
match v {
1 => ProverType::Chunk,
2 => ProverType::Batch,
1 => ProverType::Deprecated1,
2 => ProverType::Deprecated2,
3 => ProverType::OpenVM,
_ => {
panic!("invalid prover_type")
}
@@ -73,8 +75,9 @@ impl ProverType {
pub fn to_u8(self) -> u8 {
match self {
ProverType::Chunk => 1,
ProverType::Batch => 2,
ProverType::Deprecated1 => 1,
ProverType::Deprecated2 => 2,
ProverType::OpenVM => 3,
}
}
}
@@ -84,10 +87,7 @@ impl Serialize for ProverType {
where
S: Serializer,
{
match *self {
ProverType::Chunk => serializer.serialize_u8(1),
ProverType::Batch => serializer.serialize_u8(2),
}
serializer.serialize_u8(self.to_u8())
}
}

9
prover/src/utils.rs
View File

@@ -1,8 +1,6 @@
use env_logger::Env;
use std::{fs::OpenOptions, sync::Once};
use crate::types::{ProverType, TaskType};
static LOG_INIT: Once = Once::new();
/// Initialize log
@@ -23,10 +21,3 @@ pub fn log_init(log_file: Option<String>) {
builder.init();
});
}
pub fn get_task_types(prover_type: ProverType) -> Vec<TaskType> {
match prover_type {
ProverType::Chunk => vec![TaskType::Chunk],
ProverType::Batch => vec![TaskType::Batch, TaskType::Bundle],
}
}

165
prover/src/zk_circuits_handler.rs
View File

@@ -1,19 +1,10 @@
mod common;
mod darwin;
mod darwin_v2;
pub mod euclid;
use super::geth_client::GethClient;
use crate::{
config::{AssetsDirEnvConfig, Config},
types::{ProverType, Task, TaskType},
utils::get_task_types,
geth_client::GethClient,
types::{Task, TaskType},
};
use anyhow::{bail, Result};
use darwin::DarwinHandler;
use darwin_v2::DarwinV2Handler;
use std::{cell::RefCell, collections::HashMap, rc::Rc};
type HardForkName = String;
use anyhow::Result;
pub mod utils {
pub fn encode_vk(vk: Vec<u8>) -> String {
@@ -24,151 +15,5 @@ pub mod utils {
pub trait CircuitsHandler {
fn get_vk(&self, task_type: TaskType) -> Option<Vec<u8>>;
fn get_proof_data(&self, task_type: TaskType, task: &Task) -> Result<String>;
}
type CircuitsHandlerBuilder = fn(
prover_type: ProverType,
config: &Config,
geth_client: Option<Rc<RefCell<GethClient>>>,
) -> Result<Box<dyn CircuitsHandler>>;
pub struct CircuitsHandlerProvider<'a> {
prover_type: ProverType,
config: &'a Config,
geth_client: Option<Rc<RefCell<GethClient>>>,
circuits_handler_builder_map: HashMap<HardForkName, CircuitsHandlerBuilder>,
current_fork_name: Option<HardForkName>,
current_circuit: Option<Rc<Box<dyn CircuitsHandler>>>,
}
impl<'a> CircuitsHandlerProvider<'a> {
pub fn new(
prover_type: ProverType,
config: &'a Config,
geth_client: Option<Rc<RefCell<GethClient>>>,
) -> Result<Self> {
let mut m: HashMap<HardForkName, CircuitsHandlerBuilder> = HashMap::new();
fn handler_builder(
prover_type: ProverType,
config: &Config,
geth_client: Option<Rc<RefCell<GethClient>>>,
) -> Result<Box<dyn CircuitsHandler>> {
log::info!(
"now init zk circuits handler, hard_fork_name: {}",
&config.low_version_circuit.hard_fork_name
);
AssetsDirEnvConfig::enable_first();
DarwinHandler::new(
prover_type,
&config.low_version_circuit.params_path,
&config.low_version_circuit.assets_path,
geth_client,
)
.map(|handler| Box::new(handler) as Box<dyn CircuitsHandler>)
}
m.insert(
config.low_version_circuit.hard_fork_name.clone(),
handler_builder,
);
fn next_handler_builder(
prover_type: ProverType,
config: &Config,
geth_client: Option<Rc<RefCell<GethClient>>>,
) -> Result<Box<dyn CircuitsHandler>> {
log::info!(
"now init zk circuits handler, hard_fork_name: {}",
&config.high_version_circuit.hard_fork_name
);
AssetsDirEnvConfig::enable_second();
DarwinV2Handler::new(
prover_type,
&config.high_version_circuit.params_path,
&config.high_version_circuit.assets_path,
geth_client,
)
.map(|handler| Box::new(handler) as Box<dyn CircuitsHandler>)
}
m.insert(
config.high_version_circuit.hard_fork_name.clone(),
next_handler_builder,
);
let provider = CircuitsHandlerProvider {
prover_type,
config,
geth_client,
circuits_handler_builder_map: m,
current_fork_name: None,
current_circuit: None,
};
Ok(provider)
}
pub fn get_circuits_handler(
&mut self,
hard_fork_name: &String,
) -> Result<Rc<Box<dyn CircuitsHandler>>> {
match &self.current_fork_name {
Some(fork_name) if fork_name == hard_fork_name => {
log::info!("get circuits handler from cache");
if let Some(handler) = &self.current_circuit {
Ok(handler.clone())
} else {
bail!("missing cached handler, there must be something wrong.")
}
}
_ => {
log::info!(
"failed to get circuits handler from cache, create a new one: {hard_fork_name}"
);
if let Some(builder) = self.circuits_handler_builder_map.get(hard_fork_name) {
log::info!("building circuits handler for {hard_fork_name}");
let handler = builder(self.prover_type, self.config, self.geth_client.clone())
.expect("failed to build circuits handler");
self.current_fork_name = Some(hard_fork_name.clone());
let rc_handler = Rc::new(handler);
self.current_circuit = Some(rc_handler.clone());
Ok(rc_handler)
} else {
bail!("missing builder, there must be something wrong.")
}
}
}
}
pub fn init_vks(
&self,
prover_type: ProverType,
config: &'a Config,
geth_client: Option<Rc<RefCell<GethClient>>>,
) -> Vec<String> {
self.circuits_handler_builder_map
.iter()
.flat_map(|(hard_fork_name, build)| {
let handler = build(prover_type, config, geth_client.clone())
.expect("failed to build circuits handler");
get_task_types(prover_type)
.into_iter()
.map(|task_type| {
let vk = handler
.get_vk(task_type)
.map_or("".to_string(), utils::encode_vk);
log::info!(
"vk for {hard_fork_name}, is {vk}, task_type: {:?}",
task_type
);
vk
})
.filter(|vk| !vk.is_empty())
.collect::<Vec<String>>()
})
.collect::<Vec<String>>()
}
fn get_proof_data(&self, task: &Task, geth_client: &GethClient) -> Result<String>;
}

33
prover/src/zk_circuits_handler/common.rs
View File

@@ -1,33 +0,0 @@
use std::{collections::BTreeMap, rc::Rc};
use crate::types::ProverType;
use once_cell::sync::OnceCell;
use halo2_proofs::{halo2curves::bn256::Bn256, poly::kzg::commitment::ParamsKZG};
static mut PARAMS_MAP: OnceCell<Rc<BTreeMap<u32, ParamsKZG<Bn256>>>> = OnceCell::new();
pub fn get_params_map_instance<'a, F>(load_params_func: F) -> &'a BTreeMap<u32, ParamsKZG<Bn256>>
where
F: FnOnce() -> BTreeMap<u32, ParamsKZG<Bn256>>,
{
unsafe {
PARAMS_MAP.get_or_init(|| {
let params_map = load_params_func();
Rc::new(params_map)
})
}
}
pub fn get_degrees<F>(prover_types: &std::collections::HashSet<ProverType>, f: F) -> Vec<u32>
where
F: FnMut(&ProverType) -> Vec<u32>,
{
prover_types
.iter()
.flat_map(f)
.collect::<std::collections::HashSet<u32>>()
.into_iter()
.collect()
}

467
prover/src/zk_circuits_handler/darwin.rs
View File

@@ -1,467 +0,0 @@
use super::{common::*, CircuitsHandler};
use crate::{
geth_client::GethClient,
types::{ProverType, TaskType},
};
use anyhow::{bail, Context, Ok, Result};
use once_cell::sync::Lazy;
use serde::Deserialize;
use crate::types::{CommonHash, Task};
use std::{cell::RefCell, cmp::Ordering, env, rc::Rc};
use prover_darwin::{
aggregator::Prover as BatchProver,
check_chunk_hashes,
common::Prover as CommonProver,
config::{AGG_DEGREES, ZKEVM_DEGREES},
zkevm::Prover as ChunkProver,
BatchProof, BatchProvingTask, BlockTrace, BundleProof, BundleProvingTask, ChunkInfo,
ChunkProof, ChunkProvingTask,
};
// Only used for debugging.
static OUTPUT_DIR: Lazy<Option<String>> = Lazy::new(|| env::var("PROVER_OUTPUT_DIR").ok());
#[derive(Debug, Clone, Deserialize)]
pub struct BatchTaskDetail {
pub chunk_infos: Vec<ChunkInfo>,
#[serde(flatten)]
pub batch_proving_task: BatchProvingTask,
}
type BundleTaskDetail = BundleProvingTask;
#[derive(Debug, Clone, Deserialize)]
pub struct ChunkTaskDetail {
pub block_hashes: Vec<CommonHash>,
}
fn get_block_number(block_trace: &BlockTrace) -> Option<u64> {
block_trace.header.number.map(|n| n.as_u64())
}
#[derive(Default)]
pub struct DarwinHandler {
chunk_prover: Option<RefCell<ChunkProver<'static>>>,
batch_prover: Option<RefCell<BatchProver<'static>>>,
geth_client: Option<Rc<RefCell<GethClient>>>,
}
impl DarwinHandler {
pub fn new_multi(
prover_types: Vec<ProverType>,
params_dir: &str,
assets_dir: &str,
geth_client: Option<Rc<RefCell<GethClient>>>,
) -> Result<Self> {
let class_name = std::intrinsics::type_name::<Self>();
let prover_types_set = prover_types
.into_iter()
.collect::<std::collections::HashSet<ProverType>>();
let mut handler = Self {
batch_prover: None,
chunk_prover: None,
geth_client,
};
let degrees: Vec<u32> = get_degrees(&prover_types_set, |prover_type| match prover_type {
ProverType::Chunk => ZKEVM_DEGREES.clone(),
ProverType::Batch => AGG_DEGREES.clone(),
});
let params_map = get_params_map_instance(|| {
log::info!(
"calling get_params_map from {}, prover_types: {:?}, degrees: {:?}",
class_name,
prover_types_set,
degrees
);
CommonProver::load_params_map(params_dir, &degrees)
});
for prover_type in prover_types_set {
match prover_type {
ProverType::Chunk => {
handler.chunk_prover = Some(RefCell::new(ChunkProver::from_params_and_assets(
params_map, assets_dir,
)));
}
ProverType::Batch => {
handler.batch_prover = Some(RefCell::new(BatchProver::from_params_and_assets(
params_map, assets_dir,
)))
}
}
}
Ok(handler)
}
pub fn new(
prover_type: ProverType,
params_dir: &str,
assets_dir: &str,
geth_client: Option<Rc<RefCell<GethClient>>>,
) -> Result<Self> {
Self::new_multi(vec![prover_type], params_dir, assets_dir, geth_client)
}
fn gen_chunk_proof_raw(&self, chunk_trace: Vec<BlockTrace>) -> Result<ChunkProof> {
if let Some(prover) = self.chunk_prover.as_ref() {
let chunk = ChunkProvingTask::from(chunk_trace);
let chunk_proof =
prover
.borrow_mut()
.gen_chunk_proof(chunk, None, None, self.get_output_dir())?;
return Ok(chunk_proof);
}
unreachable!("please check errors in proof_type logic")
}
fn gen_chunk_proof(&self, task: &crate::types::Task) -> Result<String> {
let chunk_trace = self.gen_chunk_traces(task)?;
let chunk_proof = self.gen_chunk_proof_raw(chunk_trace)?;
Ok(serde_json::to_string(&chunk_proof)?)
}
fn gen_batch_proof_raw(&self, batch_task_detail: BatchTaskDetail) -> Result<BatchProof> {
if let Some(prover) = self.batch_prover.as_ref() {
let chunk_hashes_proofs: Vec<(ChunkInfo, ChunkProof)> = batch_task_detail
.chunk_infos
.clone()
.into_iter()
.zip(batch_task_detail.batch_proving_task.chunk_proofs.clone())
.collect();
let chunk_proofs: Vec<ChunkProof> =
chunk_hashes_proofs.iter().map(|t| t.1.clone()).collect();
let is_valid = prover.borrow_mut().check_protocol_of_chunks(&chunk_proofs);
if !is_valid {
bail!("non-match chunk protocol")
}
check_chunk_hashes("", &chunk_hashes_proofs).context("failed to check chunk info")?;
let batch_proof = prover.borrow_mut().gen_batch_proof(
batch_task_detail.batch_proving_task,
None,
self.get_output_dir(),
)?;
return Ok(batch_proof);
}
unreachable!("please check errors in proof_type logic")
}
fn gen_batch_proof(&self, task: &crate::types::Task) -> Result<String> {
log::info!("[circuit] gen_batch_proof for task {}", task.id);
let batch_task_detail: BatchTaskDetail = serde_json::from_str(&task.task_data)?;
let batch_proof = self.gen_batch_proof_raw(batch_task_detail)?;
Ok(serde_json::to_string(&batch_proof)?)
}
fn gen_bundle_proof_raw(&self, bundle_task_detail: BundleTaskDetail) -> Result<BundleProof> {
if let Some(prover) = self.batch_prover.as_ref() {
let bundle_proof = prover.borrow_mut().gen_bundle_proof(
bundle_task_detail,
None,
self.get_output_dir(),
)?;
return Ok(bundle_proof);
}
unreachable!("please check errors in proof_type logic")
}
fn gen_bundle_proof(&self, task: &crate::types::Task) -> Result<String> {
log::info!("[circuit] gen_bundle_proof for task {}", task.id);
let bundle_task_detail: BundleTaskDetail = serde_json::from_str(&task.task_data)?;
let bundle_proof = self.gen_bundle_proof_raw(bundle_task_detail)?;
Ok(serde_json::to_string(&bundle_proof)?)
}
fn get_output_dir(&self) -> Option<&str> {
OUTPUT_DIR.as_deref()
}
fn gen_chunk_traces(&self, task: &Task) -> Result<Vec<BlockTrace>> {
let chunk_task_detail: ChunkTaskDetail = serde_json::from_str(&task.task_data)?;
self.get_sorted_traces_by_hashes(&chunk_task_detail.block_hashes)
}
fn get_sorted_traces_by_hashes(&self, block_hashes: &[CommonHash]) -> Result<Vec<BlockTrace>> {
if block_hashes.is_empty() {
log::error!("[prover] failed to get sorted traces: block_hashes are empty");
bail!("block_hashes are empty")
}
let mut block_traces = Vec::new();
for hash in block_hashes.iter() {
let trace = self
.geth_client
.as_ref()
.unwrap()
.borrow_mut()
.get_block_trace_by_hash(hash)?;
block_traces.push(trace);
}
block_traces.sort_by(|a, b| {
if get_block_number(a).is_none() {
Ordering::Less
} else if get_block_number(b).is_none() {
Ordering::Greater
} else {
get_block_number(a)
.unwrap()
.cmp(&get_block_number(b).unwrap())
}
});
let block_numbers: Vec<u64> = block_traces
.iter()
.map(|trace| get_block_number(trace).unwrap_or(0))
.collect();
let mut i = 0;
while i < block_numbers.len() - 1 {
if block_numbers[i] + 1 != block_numbers[i + 1] {
log::error!(
"[prover] block numbers are not continuous, got {} and {}",
block_numbers[i],
block_numbers[i + 1]
);
bail!(
"block numbers are not continuous, got {} and {}",
block_numbers[i],
block_numbers[i + 1]
)
}
i += 1;
}
Ok(block_traces)
}
}
impl CircuitsHandler for DarwinHandler {
fn get_vk(&self, task_type: TaskType) -> Option<Vec<u8>> {
match task_type {
TaskType::Chunk => self
.chunk_prover
.as_ref()
.and_then(|prover| prover.borrow().get_vk()),
TaskType::Batch => self
.batch_prover
.as_ref()
.and_then(|prover| prover.borrow().get_batch_vk()),
TaskType::Bundle => self
.batch_prover
.as_ref()
.and_then(|prover| prover.borrow().get_bundle_vk()),
_ => unreachable!(),
}
}
fn get_proof_data(&self, task_type: TaskType, task: &crate::types::Task) -> Result<String> {
match task_type {
TaskType::Chunk => self.gen_chunk_proof(task),
TaskType::Batch => self.gen_batch_proof(task),
TaskType::Bundle => self.gen_bundle_proof(task),
_ => unreachable!(),
}
}
}
// =================================== tests module ========================================
#[cfg(test)]
mod tests {
use super::*;
use crate::zk_circuits_handler::utils::encode_vk;
use prover_darwin::utils::chunk_trace_to_witness_block;
use std::{path::PathBuf, sync::LazyLock};
#[ctor::ctor]
fn init() {
crate::utils::log_init(None);
log::info!("logger initialized");
}
static DEFAULT_WORK_DIR: &str = "/assets";
static WORK_DIR: LazyLock<String> = LazyLock::new(|| {
std::env::var("DARWIN_TEST_DIR")
.unwrap_or(String::from(DEFAULT_WORK_DIR))
.trim_end_matches('/')
.to_string()
});
static PARAMS_PATH: LazyLock<String> = LazyLock::new(|| format!("{}/test_params", *WORK_DIR));
static ASSETS_PATH: LazyLock<String> = LazyLock::new(|| format!("{}/test_assets", *WORK_DIR));
static PROOF_DUMP_PATH: LazyLock<String> =
LazyLock::new(|| format!("{}/proof_data", *WORK_DIR));
static BATCH_DIR_PATH: LazyLock<String> =
LazyLock::new(|| format!("{}/traces/batch_24", *WORK_DIR));
static BATCH_VK_PATH: LazyLock<String> =
LazyLock::new(|| format!("{}/test_assets/vk_batch.vkey", *WORK_DIR));
static CHUNK_VK_PATH: LazyLock<String> =
LazyLock::new(|| format!("{}/test_assets/vk_chunk.vkey", *WORK_DIR));
#[test]
fn it_works() {
let result = true;
assert!(result);
}
#[test]
fn test_circuits() -> Result<()> {
let bi_handler = DarwinHandler::new_multi(
vec![ProverType::Chunk, ProverType::Batch],
&PARAMS_PATH,
&ASSETS_PATH,
None,
)?;
let chunk_handler = bi_handler;
let chunk_vk = chunk_handler.get_vk(TaskType::Chunk).unwrap();
check_vk(TaskType::Chunk, chunk_vk, "chunk vk must be available");
let chunk_dir_paths = get_chunk_dir_paths()?;
log::info!("chunk_dir_paths, {:?}", chunk_dir_paths);
let mut chunk_infos = vec![];
let mut chunk_proofs = vec![];
for (id, chunk_path) in chunk_dir_paths.into_iter().enumerate() {
let chunk_id = format!("chunk_proof{}", id + 1);
log::info!("start to process {chunk_id}");
let chunk_trace = read_chunk_trace(chunk_path)?;
let chunk_info = traces_to_chunk_info(chunk_trace.clone())?;
chunk_infos.push(chunk_info);
log::info!("start to prove {chunk_id}");
let chunk_proof = chunk_handler.gen_chunk_proof_raw(chunk_trace)?;
let proof_data = serde_json::to_string(&chunk_proof)?;
dump_proof(chunk_id, proof_data)?;
chunk_proofs.push(chunk_proof);
}
let batch_handler = chunk_handler;
let batch_vk = batch_handler.get_vk(TaskType::Batch).unwrap();
check_vk(TaskType::Batch, batch_vk, "batch vk must be available");
let batch_task_detail = make_batch_task_detail(chunk_infos, chunk_proofs);
log::info!("start to prove batch");
let batch_proof = batch_handler.gen_batch_proof_raw(batch_task_detail)?;
let proof_data = serde_json::to_string(&batch_proof)?;
dump_proof("batch_proof".to_string(), proof_data)?;
Ok(())
}
fn make_batch_task_detail(_: Vec<ChunkInfo>, _: Vec<ChunkProof>) -> BatchTaskDetail {
todo!();
// BatchTaskDetail {
// chunk_infos,
// batch_proving_task: BatchProvingTask {
// parent_batch_hash: todo!(),
// parent_state_root: todo!(),
// batch_header: todo!(),
// chunk_proofs,
// },
// }
}
fn check_vk(proof_type: TaskType, vk: Vec<u8>, info: &str) {
log::info!("check_vk, {:?}", proof_type);
let vk_from_file = read_vk(proof_type).unwrap();
assert_eq!(vk_from_file, encode_vk(vk), "{info}")
}
fn read_vk(proof_type: TaskType) -> Result<String> {
log::info!("read_vk, {:?}", proof_type);
let vk_file = match proof_type {
TaskType::Chunk => CHUNK_VK_PATH.clone(),
TaskType::Batch => BATCH_VK_PATH.clone(),
TaskType::Bundle => todo!(),
TaskType::Undefined => unreachable!(),
};
let data = std::fs::read(vk_file)?;
Ok(encode_vk(data))
}
fn read_chunk_trace(path: PathBuf) -> Result<Vec<BlockTrace>> {
log::info!("read_chunk_trace, {:?}", path);
let mut chunk_trace: Vec<BlockTrace> = vec![];
fn read_block_trace(file: &PathBuf) -> Result<BlockTrace> {
let f = std::fs::File::open(file)?;
Ok(serde_json::from_reader(&f)?)
}
if path.is_dir() {
let entries = std::fs::read_dir(&path)?;
let mut files: Vec<String> = entries
.into_iter()
.filter_map(|e| {
if e.is_err() {
return None;
}
let entry = e.unwrap();
if entry.path().is_dir() {
return None;
}
if let Result::Ok(file_name) = entry.file_name().into_string() {
Some(file_name)
} else {
None
}
})
.collect();
files.sort();
log::info!("files in chunk {:?} is {:?}", path, files);
for file in files {
let block_trace = read_block_trace(&path.join(file))?;
chunk_trace.push(block_trace);
}
} else {
let block_trace = read_block_trace(&path)?;
chunk_trace.push(block_trace);
}
Ok(chunk_trace)
}
fn get_chunk_dir_paths() -> Result<Vec<PathBuf>> {
let batch_path = PathBuf::from(BATCH_DIR_PATH.clone());
let entries = std::fs::read_dir(&batch_path)?;
let mut files: Vec<String> = entries
.filter_map(|e| {
if e.is_err() {
return None;
}
let entry = e.unwrap();
if entry.path().is_dir() {
if let Result::Ok(file_name) = entry.file_name().into_string() {
Some(file_name)
} else {
None
}
} else {
None
}
})
.collect();
files.sort();
log::info!("files in batch {:?} is {:?}", batch_path, files);
Ok(files.into_iter().map(|f| batch_path.join(f)).collect())
}
fn traces_to_chunk_info(chunk_trace: Vec<BlockTrace>) -> Result<ChunkInfo> {
let witness_block = chunk_trace_to_witness_block(chunk_trace)?;
Ok(ChunkInfo::from_witness_block(&witness_block, false))
}
fn dump_proof(id: String, proof_data: String) -> Result<()> {
let dump_path = PathBuf::from(PROOF_DUMP_PATH.clone());
Ok(std::fs::write(dump_path.join(id), proof_data)?)
}
}

525
prover/src/zk_circuits_handler/darwin_v2.rs
View File

@@ -1,525 +0,0 @@
use super::{common::*, CircuitsHandler};
use crate::{
geth_client::GethClient,
types::{ProverType, TaskType},
};
use anyhow::{bail, Context, Ok, Result};
use once_cell::sync::Lazy;
use serde::Deserialize;
use crate::types::{CommonHash, Task};
use std::{cell::RefCell, cmp::Ordering, env, rc::Rc};
use prover_darwin_v2::{
aggregator::Prover as BatchProver,
check_chunk_hashes,
common::Prover as CommonProver,
config::{AGG_DEGREES, ZKEVM_DEGREES},
zkevm::Prover as ChunkProver,
BatchProof, BatchProvingTask, BlockTrace, BundleProof, BundleProvingTask, ChunkInfo,
ChunkProof, ChunkProvingTask,
};
// Only used for debugging.
static OUTPUT_DIR: Lazy<Option<String>> = Lazy::new(|| env::var("PROVER_OUTPUT_DIR").ok());
#[derive(Debug, Clone, Deserialize)]
pub struct BatchTaskDetail {
pub chunk_infos: Vec<ChunkInfo>,
#[serde(flatten)]
pub batch_proving_task: BatchProvingTask,
}
type BundleTaskDetail = BundleProvingTask;
#[derive(Debug, Clone, Deserialize)]
pub struct ChunkTaskDetail {
pub block_hashes: Vec<CommonHash>,
}
fn get_block_number(block_trace: &BlockTrace) -> Option<u64> {
block_trace.header.number.map(|n| n.as_u64())
}
#[derive(Default)]
pub struct DarwinV2Handler {
chunk_prover: Option<RefCell<ChunkProver<'static>>>,
batch_prover: Option<RefCell<BatchProver<'static>>>,
geth_client: Option<Rc<RefCell<GethClient>>>,
}
impl DarwinV2Handler {
pub fn new_multi(
prover_types: Vec<ProverType>,
params_dir: &str,
assets_dir: &str,
geth_client: Option<Rc<RefCell<GethClient>>>,
) -> Result<Self> {
let class_name = std::intrinsics::type_name::<Self>();
let prover_types_set = prover_types
.into_iter()
.collect::<std::collections::HashSet<ProverType>>();
let mut handler = Self {
batch_prover: None,
chunk_prover: None,
geth_client,
};
let degrees: Vec<u32> = get_degrees(&prover_types_set, |prover_type| match prover_type {
ProverType::Chunk => ZKEVM_DEGREES.clone(),
ProverType::Batch => AGG_DEGREES.clone(),
});
let params_map = get_params_map_instance(|| {
log::info!(
"calling get_params_map from {}, prover_types: {:?}, degrees: {:?}",
class_name,
prover_types_set,
degrees
);
CommonProver::load_params_map(params_dir, &degrees)
});
for prover_type in prover_types_set {
match prover_type {
ProverType::Chunk => {
handler.chunk_prover = Some(RefCell::new(ChunkProver::from_params_and_assets(
params_map, assets_dir,
)));
}
ProverType::Batch => {
handler.batch_prover = Some(RefCell::new(BatchProver::from_params_and_assets(
params_map, assets_dir,
)))
}
}
}
Ok(handler)
}
pub fn new(
prover_type: ProverType,
params_dir: &str,
assets_dir: &str,
geth_client: Option<Rc<RefCell<GethClient>>>,
) -> Result<Self> {
Self::new_multi(vec![prover_type], params_dir, assets_dir, geth_client)
}
fn gen_chunk_proof_raw(&self, chunk_trace: Vec<BlockTrace>) -> Result<ChunkProof> {
if let Some(prover) = self.chunk_prover.as_ref() {
let chunk = ChunkProvingTask::from(chunk_trace);
let chunk_proof =
prover
.borrow_mut()
.gen_chunk_proof(chunk, None, None, self.get_output_dir())?;
return Ok(chunk_proof);
}
unreachable!("please check errors in proof_type logic")
}
fn gen_chunk_proof(&self, task: &crate::types::Task) -> Result<String> {
let chunk_trace = self.gen_chunk_traces(task)?;
let chunk_proof = self.gen_chunk_proof_raw(chunk_trace)?;
Ok(serde_json::to_string(&chunk_proof)?)
}
fn gen_batch_proof_raw(&self, batch_task_detail: BatchTaskDetail) -> Result<BatchProof> {
if let Some(prover) = self.batch_prover.as_ref() {
let chunk_hashes_proofs: Vec<(ChunkInfo, ChunkProof)> = batch_task_detail
.chunk_infos
.clone()
.into_iter()
.zip(batch_task_detail.batch_proving_task.chunk_proofs.clone())
.collect();
let chunk_proofs: Vec<ChunkProof> =
chunk_hashes_proofs.iter().map(|t| t.1.clone()).collect();
let is_valid = prover.borrow_mut().check_protocol_of_chunks(&chunk_proofs);
if !is_valid {
bail!("non-match chunk protocol")
}
check_chunk_hashes("", &chunk_hashes_proofs).context("failed to check chunk info")?;
let batch_proof = prover.borrow_mut().gen_batch_proof(
batch_task_detail.batch_proving_task,
None,
self.get_output_dir(),
)?;
return Ok(batch_proof);
}
unreachable!("please check errors in proof_type logic")
}
fn gen_batch_proof(&self, task: &crate::types::Task) -> Result<String> {
log::info!("[circuit] gen_batch_proof for task {}", task.id);
let batch_task_detail: BatchTaskDetail = serde_json::from_str(&task.task_data)?;
let batch_proof = self.gen_batch_proof_raw(batch_task_detail)?;
Ok(serde_json::to_string(&batch_proof)?)
}
fn gen_bundle_proof_raw(&self, bundle_task_detail: BundleTaskDetail) -> Result<BundleProof> {
if let Some(prover) = self.batch_prover.as_ref() {
let bundle_proof = prover.borrow_mut().gen_bundle_proof(
bundle_task_detail,
None,
self.get_output_dir(),
)?;
return Ok(bundle_proof);
}
unreachable!("please check errors in proof_type logic")
}
fn gen_bundle_proof(&self, task: &crate::types::Task) -> Result<String> {
log::info!("[circuit] gen_bundle_proof for task {}", task.id);
let bundle_task_detail: BundleTaskDetail = serde_json::from_str(&task.task_data)?;
let bundle_proof = self.gen_bundle_proof_raw(bundle_task_detail)?;
Ok(serde_json::to_string(&bundle_proof)?)
}
fn get_output_dir(&self) -> Option<&str> {
OUTPUT_DIR.as_deref()
}
fn gen_chunk_traces(&self, task: &Task) -> Result<Vec<BlockTrace>> {
let chunk_task_detail: ChunkTaskDetail = serde_json::from_str(&task.task_data)?;
self.get_sorted_traces_by_hashes(&chunk_task_detail.block_hashes)
}
fn get_sorted_traces_by_hashes(&self, block_hashes: &[CommonHash]) -> Result<Vec<BlockTrace>> {
if block_hashes.is_empty() {
log::error!("[prover] failed to get sorted traces: block_hashes are empty");
bail!("block_hashes are empty")
}
let mut block_traces = Vec::new();
for hash in block_hashes.iter() {
let trace = self
.geth_client
.as_ref()
.unwrap()
.borrow_mut()
.get_block_trace_by_hash(hash)?;
block_traces.push(trace);
}
block_traces.sort_by(|a, b| {
if get_block_number(a).is_none() {
Ordering::Less
} else if get_block_number(b).is_none() {
Ordering::Greater
} else {
get_block_number(a)
.unwrap()
.cmp(&get_block_number(b).unwrap())
}
});
let block_numbers: Vec<u64> = block_traces
.iter()
.map(|trace| get_block_number(trace).unwrap_or(0))
.collect();
let mut i = 0;
while i < block_numbers.len() - 1 {
if block_numbers[i] + 1 != block_numbers[i + 1] {
log::error!(
"[prover] block numbers are not continuous, got {} and {}",
block_numbers[i],
block_numbers[i + 1]
);
bail!(
"block numbers are not continuous, got {} and {}",
block_numbers[i],
block_numbers[i + 1]
)
}
i += 1;
}
Ok(block_traces)
}
}
impl CircuitsHandler for DarwinV2Handler {
fn get_vk(&self, task_type: TaskType) -> Option<Vec<u8>> {
match task_type {
TaskType::Chunk => self
.chunk_prover
.as_ref()
.and_then(|prover| prover.borrow().get_vk()),
TaskType::Batch => self
.batch_prover
.as_ref()
.and_then(|prover| prover.borrow().get_batch_vk()),
TaskType::Bundle => self
.batch_prover
.as_ref()
.and_then(|prover| prover.borrow().get_bundle_vk()),
_ => unreachable!(),
}
}
fn get_proof_data(&self, task_type: TaskType, task: &crate::types::Task) -> Result<String> {
match task_type {
TaskType::Chunk => self.gen_chunk_proof(task),
TaskType::Batch => self.gen_batch_proof(task),
TaskType::Bundle => self.gen_bundle_proof(task),
_ => unreachable!(),
}
}
}
// =================================== tests module ========================================
#[cfg(test)]
mod tests {
use super::*;
use crate::zk_circuits_handler::utils::encode_vk;
use ethers_core::types::H256;
use prover_darwin_v2::{
aggregator::eip4844, utils::chunk_trace_to_witness_block, BatchData, BatchHeader,
MAX_AGG_SNARKS,
};
use std::{path::PathBuf, sync::LazyLock};
#[ctor::ctor]
fn init() {
crate::utils::log_init(None);
log::info!("logger initialized");
}
static DEFAULT_WORK_DIR: &str = "/assets";
static WORK_DIR: LazyLock<String> = LazyLock::new(|| {
std::env::var("DARWIN_V2_TEST_DIR")
.unwrap_or(String::from(DEFAULT_WORK_DIR))
.trim_end_matches('/')
.to_string()
});
static PARAMS_PATH: LazyLock<String> = LazyLock::new(|| format!("{}/test_params", *WORK_DIR));
static ASSETS_PATH: LazyLock<String> = LazyLock::new(|| format!("{}/test_assets", *WORK_DIR));
static PROOF_DUMP_PATH: LazyLock<String> =
LazyLock::new(|| format!("{}/proof_data", *WORK_DIR));
static BATCH_DIR_PATH: LazyLock<String> =
LazyLock::new(|| format!("{}/traces/batch_24", *WORK_DIR));
static BATCH_VK_PATH: LazyLock<String> =
LazyLock::new(|| format!("{}/test_assets/vk_batch.vkey", *WORK_DIR));
static CHUNK_VK_PATH: LazyLock<String> =
LazyLock::new(|| format!("{}/test_assets/vk_chunk.vkey", *WORK_DIR));
#[test]
fn it_works() {
let result = true;
assert!(result);
}
#[test]
fn test_circuits() -> Result<()> {
let bi_handler = DarwinV2Handler::new_multi(
vec![ProverType::Chunk, ProverType::Batch],
&PARAMS_PATH,
&ASSETS_PATH,
None,
)?;
let chunk_handler = bi_handler;
let chunk_vk = chunk_handler.get_vk(TaskType::Chunk).unwrap();
check_vk(TaskType::Chunk, chunk_vk, "chunk vk must be available");
let chunk_dir_paths = get_chunk_dir_paths()?;
log::info!("chunk_dir_paths, {:?}", chunk_dir_paths);
let mut chunk_traces = vec![];
let mut chunk_infos = vec![];
let mut chunk_proofs = vec![];
for (id, chunk_path) in chunk_dir_paths.into_iter().enumerate() {
let chunk_id = format!("chunk_proof{}", id + 1);
log::info!("start to process {chunk_id}");
let chunk_trace = read_chunk_trace(chunk_path)?;
chunk_traces.push(chunk_trace.clone());
let chunk_info = traces_to_chunk_info(chunk_trace.clone())?;
chunk_infos.push(chunk_info);
log::info!("start to prove {chunk_id}");
let chunk_proof = chunk_handler.gen_chunk_proof_raw(chunk_trace)?;
let proof_data = serde_json::to_string(&chunk_proof)?;
dump_proof(chunk_id, proof_data)?;
chunk_proofs.push(chunk_proof);
}
let batch_handler = chunk_handler;
let batch_vk = batch_handler.get_vk(TaskType::Batch).unwrap();
check_vk(TaskType::Batch, batch_vk, "batch vk must be available");
let batch_task_detail = make_batch_task_detail(chunk_traces, chunk_proofs, None);
log::info!("start to prove batch");
let batch_proof = batch_handler.gen_batch_proof_raw(batch_task_detail)?;
let proof_data = serde_json::to_string(&batch_proof)?;
dump_proof("batch_proof".to_string(), proof_data)?;
Ok(())
}
// copied from https://github.com/scroll-tech/scroll-prover/blob/main/integration/src/prove.rs
fn get_blob_from_chunks(chunks: &[ChunkInfo]) -> Vec<u8> {
let num_chunks = chunks.len();
let padded_chunk =
ChunkInfo::mock_padded_chunk_info_for_testing(chunks.last().as_ref().unwrap());
let chunks_with_padding = [
chunks.to_vec(),
vec![padded_chunk; MAX_AGG_SNARKS - num_chunks],
]
.concat();
let batch_data = BatchData::<{ MAX_AGG_SNARKS }>::new(chunks.len(), &chunks_with_padding);
let batch_bytes = batch_data.get_batch_data_bytes();
let blob_bytes = eip4844::get_blob_bytes(&batch_bytes);
log::info!("blob_bytes len {}", blob_bytes.len());
blob_bytes
}
// TODO: chunk_infos can be extracted from chunk_proofs.
// Still needed?
fn make_batch_task_detail(
chunk_traces: Vec<Vec<BlockTrace>>,
chunk_proofs: Vec<ChunkProof>,
last_batcher_header: Option<BatchHeader<{ MAX_AGG_SNARKS }>>,
) -> BatchTaskDetail {
// dummy parent batch hash
let dummy_parent_batch_hash = H256([
0xab, 0xac, 0xad, 0xae, 0xaf, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
]);
let chunk_infos: Vec<_> = chunk_proofs.iter().map(|p| p.chunk_info.clone()).collect();
let l1_message_popped = chunk_traces
.iter()
.flatten()
.map(|chunk| chunk.num_l1_txs())
.sum();
let last_block_timestamp = chunk_traces.last().map_or(0, |block_traces| {
block_traces
.last()
.map_or(0, |block_trace| block_trace.header.timestamp.as_u64())
});
let blob_bytes = get_blob_from_chunks(&chunk_infos);
let batch_header = BatchHeader::construct_from_chunks(
last_batcher_header.map_or(4, |header| header.version),
last_batcher_header.map_or(123, |header| header.batch_index + 1),
l1_message_popped,
last_batcher_header.map_or(l1_message_popped, |header| {
header.total_l1_message_popped + l1_message_popped
}),
last_batcher_header.map_or(dummy_parent_batch_hash, |header| header.batch_hash()),
last_block_timestamp,
&chunk_infos,
&blob_bytes,
);
BatchTaskDetail {
chunk_infos,
batch_proving_task: BatchProvingTask {
chunk_proofs,
batch_header,
blob_bytes,
},
}
}
fn check_vk(proof_type: TaskType, vk: Vec<u8>, info: &str) {
log::info!("check_vk, {:?}", proof_type);
let vk_from_file = read_vk(proof_type).unwrap();
assert_eq!(vk_from_file, encode_vk(vk), "{info}")
}
fn read_vk(proof_type: TaskType) -> Result<String> {
log::info!("read_vk, {:?}", proof_type);
let vk_file = match proof_type {
TaskType::Chunk => CHUNK_VK_PATH.clone(),
TaskType::Batch => BATCH_VK_PATH.clone(),
TaskType::Bundle => todo!(),
TaskType::Undefined => unreachable!(),
};
let data = std::fs::read(vk_file)?;
Ok(encode_vk(data))
}
fn read_chunk_trace(path: PathBuf) -> Result<Vec<BlockTrace>> {
log::info!("read_chunk_trace, {:?}", path);
let mut chunk_trace: Vec<BlockTrace> = vec![];
fn read_block_trace(file: &PathBuf) -> Result<BlockTrace> {
let f = std::fs::File::open(file)?;
Ok(serde_json::from_reader(&f)?)
}
if path.is_dir() {
let entries = std::fs::read_dir(&path)?;
let mut files: Vec<String> = entries
.into_iter()
.filter_map(|e| {
if e.is_err() {
return None;
}
let entry = e.unwrap();
if entry.path().is_dir() {
return None;
}
if let Result::Ok(file_name) = entry.file_name().into_string() {
Some(file_name)
} else {
None
}
})
.collect();
files.sort();
log::info!("files in chunk {:?} is {:?}", path, files);
for file in files {
let block_trace = read_block_trace(&path.join(file))?;
chunk_trace.push(block_trace);
}
} else {
let block_trace = read_block_trace(&path)?;
chunk_trace.push(block_trace);
}
Ok(chunk_trace)
}
fn get_chunk_dir_paths() -> Result<Vec<PathBuf>> {
let batch_path = PathBuf::from(BATCH_DIR_PATH.clone());
let entries = std::fs::read_dir(&batch_path)?;
let mut files: Vec<String> = entries
.filter_map(|e| {
if e.is_err() {
return None;
}
let entry = e.unwrap();
if entry.path().is_dir() {
if let Result::Ok(file_name) = entry.file_name().into_string() {
Some(file_name)
} else {
None
}
} else {
None
}
})
.collect();
files.sort();
log::info!("files in batch {:?} is {:?}", batch_path, files);
Ok(files.into_iter().map(|f| batch_path.join(f)).collect())
}
fn traces_to_chunk_info(chunk_trace: Vec<BlockTrace>) -> Result<ChunkInfo> {
let witness_block = chunk_trace_to_witness_block(chunk_trace)?;
Ok(ChunkInfo::from_witness_block(&witness_block, false))
}
fn dump_proof(id: String, proof_data: String) -> Result<()> {
let dump_path = PathBuf::from(PROOF_DUMP_PATH.clone());
Ok(std::fs::write(dump_path.join(id), proof_data)?)
}
}