feat: update

common/libzkp/impl/Cargo.lock

@@ -1374,6 +1374,12 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "glob"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
+
 [[package]]
 name = "gloo-timers"
 version = "0.2.6"
@@ -4172,6 +4178,7 @@ version = "0.1.0"
 dependencies = [
  "base64 0.13.1",
  "env_logger 0.9.3",
+ "glob",
  "halo2_proofs",
  "libc",
  "log",

common/libzkp/impl/Cargo.toml

@@ -23,6 +23,7 @@ halo2curves = { git = "https://github.com/scroll-tech/halo2curves.git", branch =
 halo2_proofs = { git = "https://github.com/scroll-tech/halo2.git", branch = "develop" }
 prover = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.9.8", default-features = false, features = ["parallel_syn", "scroll", "shanghai", "strict-ccc"] }
 
+glob = "0.3"
 base64 = "0.13.0"
 env_logger = "0.9.0"
 libc = "0.2"

common/libzkp/impl/src/batch.rs

@@ -66,7 +66,7 @@ pub unsafe extern "C" fn get_batch_vk() -> *const c_char {
 
 /// # Safety
 #[no_mangle]
-pub unsafe extern "C" fn check_chunk_proofs(chunk_proofs: *const c_char) -> *const c_char {
+pub unsafe extern "C" fn check_chunk_proofs(chunk_proofs: *const c_char) -> *mut c_char {
     let check_result: Result<bool, String> = panic_catch(|| {
         let chunk_proofs = c_char_to_vec(chunk_proofs);
         let chunk_proofs = serde_json::from_slice::<Vec<ChunkProof>>(&chunk_proofs)
@@ -94,7 +94,7 @@ pub unsafe extern "C" fn check_chunk_proofs(chunk_proofs: *const c_char) -> *con
         },
     };
 
-    serde_json::to_vec(&r).map_or(null(), vec_to_c_char)
+    serde_json::to_vec(&r).map_or(std::ptr::null_mut(), vec_to_c_char)
 }
 
 /// # Safety
@@ -102,7 +102,7 @@ pub unsafe extern "C" fn check_chunk_proofs(chunk_proofs: *const c_char) -> *con
 pub unsafe extern "C" fn gen_batch_proof(
     chunk_hashes: *const c_char,
     chunk_proofs: *const c_char,
-) -> *const c_char {
+) -> *mut c_char {
     let proof_result: Result<Vec<u8>, String> = panic_catch(|| {
         let chunk_hashes = c_char_to_vec(chunk_hashes);
         let chunk_proofs = c_char_to_vec(chunk_proofs);
@@ -143,7 +143,7 @@ pub unsafe extern "C" fn gen_batch_proof(
         },
     };
 
-    serde_json::to_vec(&r).map_or(null(), vec_to_c_char)
+    serde_json::to_vec(&r).map_or(std::ptr::null_mut(), vec_to_c_char)
 }
 
 /// # Safety

common/libzkp/impl/src/chunk.rs

@@ -5,14 +5,15 @@ use crate::{
         OUTPUT_DIR,
     },
 };
+use glob::glob;
 use libc::c_char;
 use prover::{
     consts::CHUNK_VK_FILENAME,
-    utils::init_env_and_log,
+    utils::{get_block_trace_from_file, init_env_and_log},
     zkevm::{Prover, Verifier},
     BlockTrace, ChunkProof,
 };
-use std::{cell::OnceCell, env, ptr::null};
+use std::{cell::OnceCell, env, ffi::CString, ptr::null};
 
 static mut PROVER: OnceCell<Prover> = OnceCell::new();
 static mut VERIFIER: OnceCell<Verifier> = OnceCell::new();
@@ -66,9 +67,14 @@ pub unsafe extern "C" fn get_chunk_vk() -> *const c_char {
 
 /// # Safety
 #[no_mangle]
-pub unsafe extern "C" fn gen_chunk_proof(block_traces: *const c_char) -> *const c_char {
+pub unsafe extern "C" fn gen_chunk_proof(block_traces1: *const c_char) {
+    let chunk_trace = load_batch_traces().1;
+    let json_str = serde_json::to_string(&chunk_trace).expect("Serialization failed");
+    let c_string = CString::new(json_str).expect("CString conversion failed");
+    let c_str_ptr = c_string.as_ptr();
+
     let proof_result: Result<Vec<u8>, String> = panic_catch(|| {
-        let block_traces = c_char_to_vec(block_traces);
+        let block_traces = c_char_to_vec(c_str_ptr);
         let block_traces = serde_json::from_slice::<Vec<BlockTrace>>(&block_traces)
             .map_err(|e| format!("failed to deserialize block traces: {e:?}"))?;
 
@@ -82,7 +88,7 @@ pub unsafe extern "C" fn gen_chunk_proof(block_traces: *const c_char) -> *const
     })
     .unwrap_or_else(|e| Err(format!("unwind error: {e:?}")));
 
-    let r = match proof_result {
+    let _ = match proof_result {
         Ok(proof_bytes) => ProofResult {
             message: Some(proof_bytes),
             error: None,
@@ -93,7 +99,7 @@ pub unsafe extern "C" fn gen_chunk_proof(block_traces: *const c_char) -> *const
         },
     };
 
-    serde_json::to_vec(&r).map_or(null(), vec_to_c_char)
+    // serde_json::to_vec(&r).map_or(std::ptr::null_mut(), vec_to_c_char)
 }
 
 /// # Safety
@@ -105,3 +111,31 @@ pub unsafe extern "C" fn verify_chunk_proof(proof: *const c_char) -> c_char {
     let verified = panic_catch(|| VERIFIER.get().unwrap().verify_chunk_proof(proof));
     verified.unwrap_or(false) as c_char
 }
+
+fn load_batch_traces() -> (Vec<String>, Vec<BlockTrace>) {
+    let file_names: Vec<String> = glob(&"/assets/traces/1_transfer.json".to_string())
+        .unwrap()
+        .map(|p| p.unwrap().to_str().unwrap().to_string())
+        .collect();
+    log::info!("test batch with {:?}", file_names);
+    let mut names_and_traces = file_names
+        .into_iter()
+        .map(|trace_path| {
+            let trace: BlockTrace = get_block_trace_from_file(trace_path.clone());
+            (
+                trace_path,
+                trace.clone(),
+                trace.header.number.unwrap().as_u64(),
+            )
+        })
+        .collect::<Vec<_>>();
+    names_and_traces.sort_by(|a, b| a.2.cmp(&b.2));
+    log::info!(
+        "sorted: {:?}",
+        names_and_traces
+            .iter()
+            .map(|(f, _, _)| f.clone())
+            .collect::<Vec<String>>()
+    );
+    names_and_traces.into_iter().map(|(f, t, _)| (f, t)).unzip()
+}

common/libzkp/impl/src/lib.rs

@@ -1,6 +1,6 @@
 #![feature(once_cell)]
 
 mod batch;
-mod chunk;
+pub mod chunk;
 mod types;
 mod utils;

common/libzkp/impl/src/utils.rs

@@ -38,7 +38,7 @@ pub(crate) fn string_to_c_char(string: String) -> *const c_char {
     CString::new(string).unwrap().into_raw()
 }
 
-pub(crate) fn vec_to_c_char(bytes: Vec<u8>) -> *const c_char {
+pub(crate) fn vec_to_c_char(bytes: Vec<u8>) -> *mut c_char {
     CString::new(bytes).unwrap().into_raw()
 }
 

common/libzkp/impl/tests/chunk_test.rs

@@ -0,0 +1,66 @@
+use glob::glob;
+use prover::{utils::get_block_trace_from_file, BlockTrace};
+use std::ffi::{CStr, CString};
+use zkp::chunk;
+
+#[test]
+fn chunk_test() {
+    println!("start chunk_test.");
+    unsafe {
+        let params = CString::new("/assets/test_params").expect("test_params conversion failed");
+        let assets = CString::new("/assets/test_assets").expect("test_assets conversion failed");
+
+        chunk::init_chunk_prover(params.as_ptr(), assets.as_ptr());
+
+        let chunk_trace = load_batch_traces().1;
+        let json_str = serde_json::to_string(&chunk_trace).expect("Serialization failed");
+        let c_string = CString::new(json_str).expect("CString conversion failed");
+        let c_str_ptr = c_string.as_ptr();
+
+        let ptr_cstr = CStr::from_ptr(c_str_ptr)
+            .to_str()
+            .expect("Failed to convert C string to Rust string");
+
+        println!("c_str_ptr len: {:?}", ptr_cstr.len());
+
+        let mut count = 1;
+        loop {
+            count += 1;
+            println!("count {:?}", count);
+
+            let _ = chunk::gen_chunk_proof(c_str_ptr);
+            // let ret_cstr = CStr::from_ptr(ret)
+            //     .to_str()
+            //     .expect("Failed to convert C string to Rust string");
+            // println!("ret: {:?}", ret_cstr)
+        }
+    }
+}
+
+fn load_batch_traces() -> (Vec<String>, Vec<BlockTrace>) {
+    let file_names: Vec<String> = glob(&"/assets/traces/1_transfer.json".to_string())
+        .unwrap()
+        .map(|p| p.unwrap().to_str().unwrap().to_string())
+        .collect();
+    log::info!("test batch with {:?}", file_names);
+    let mut names_and_traces = file_names
+        .into_iter()
+        .map(|trace_path| {
+            let trace: BlockTrace = get_block_trace_from_file(trace_path.clone());
+            (
+                trace_path,
+                trace.clone(),
+                trace.header.number.unwrap().as_u64(),
+            )
+        })
+        .collect::<Vec<_>>();
+    names_and_traces.sort_by(|a, b| a.2.cmp(&b.2));
+    log::info!(
+        "sorted: {:?}",
+        names_and_traces
+            .iter()
+            .map(|(f, _, _)| f.clone())
+            .collect::<Vec<String>>()
+    );
+    names_and_traces.into_iter().map(|(f, t, _)| (f, t)).unzip()
+}

common/libzkp/interface/libzkp.h

@@ -8,7 +8,7 @@ char verify_batch_proof(char* proof);
 void init_chunk_prover(char* params_dir, char* assets_dir);
 void init_chunk_verifier(char* params_dir, char* assets_dir);
 char* get_chunk_vk();
-char* gen_chunk_proof(char* block_traces);
+void gen_chunk_proof(char* block_traces);
 char verify_chunk_proof(char* proof);
 
 char* block_traces_to_chunk_info(char* block_traces);

prover/core/prover.go

@@ -206,25 +206,11 @@ func (p *ProverCore) proveBatch(chunkInfosByt []byte, chunkProofsByt []byte) ([]
 }
 
 func (p *ProverCore) proveChunk(tracesByt []byte) ([]byte, error) {
-	tracesStr := C.CString(string(tracesByt))
-	defer C.free(unsafe.Pointer(tracesStr))
-
 	log.Info("Start to create chunk proof ...")
-	cProof := C.gen_chunk_proof(tracesStr)
-	defer C.free_c_chars(cProof)
+	C.gen_chunk_proof("")
+	//defer C.free_c_chars(cProof)
 	log.Info("Finish creating chunk proof!")
-
-	var result ProofResult
-	err := json.Unmarshal([]byte(C.GoString(cProof)), &result)
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse chunk proof result: %v", err)
-	}
-
-	if result.Error != "" {
-		return nil, fmt.Errorf("failed to generate chunk proof: %s", result.Error)
-	}
-
-	return result.Message, nil
+	return nil, nil
 }
 
 func (p *ProverCore) mayDumpProof(id string, proofByt []byte) error {

prover/core/prover_oom_test.go

@@ -0,0 +1,61 @@
+//go:build ffi
+
+// go test -v -race -gcflags="-l" -ldflags="-s=false" -tags ffi ./...
+package core
+
+import (
+	"encoding/json"
+	"flag"
+	"github.com/stretchr/testify/assert"
+	"io/ioutil"
+	"net/http"
+	_ "net/http/pprof"
+	"testing"
+
+	"github.com/scroll-tech/go-ethereum/core/types"
+	"scroll-tech/common/types/message"
+
+	"scroll-tech/prover/config"
+)
+
+var (
+	paramsPath = flag.String("params", "/assets/test_params", "params dir")
+	assetsPath = flag.String("assets", "/assets/test_assets", "assets dir")
+	tracePath1 = flag.String("trace1", "/assets/traces/1_transfer.json", "chunk trace 1")
+)
+
+func initPyroscopse() {
+	go func() {
+		if runServerErr := http.ListenAndServe(":8089", nil); runServerErr != nil {
+			panic(runServerErr)
+		}
+	}()
+}
+
+func TestFFI(t *testing.T) {
+	ballast := make([]byte, 100*1024*1024*1024) // 10G
+	initPyroscopse()
+
+	chunkProverConfig := &config.ProverCoreConfig{
+		ParamsPath: *paramsPath,
+		AssetsPath: *assetsPath,
+		ProofType:  message.ProofTypeChunk,
+	}
+
+	chunkProverCore, _ := NewProverCore(chunkProverConfig)
+
+	for {
+		chunkProverCore.proveChunk()
+	}
+	runtime.KeepAlive(ballast)
+}
+
+func readChunkTrace(t *testing.T, filePat string) []*types.BlockTrace {
+	byt, err := ioutil.ReadFile(filePat)
+	assert.NoError(t, err)
+
+	trace := &types.BlockTrace{}
+	assert.NoError(t, json.Unmarshal(byt, trace))
+
+	return []*types.BlockTrace{trace}
+}

prover/go.mod

@@ -21,6 +21,8 @@ require (
 	github.com/go-stack/stack v1.8.1 // indirect
 	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/grafana/pyroscope-go v1.0.4 // indirect
+	github.com/grafana/pyroscope-go/godeltaprof v0.1.4 // indirect
 	github.com/holiman/uint256 v1.2.4 // indirect
 	github.com/huin/goupnp v1.3.0 // indirect
 	github.com/iden3/go-iden3-crypto v0.0.15 // indirect

prover/go.sum

@@ -40,6 +40,10 @@ github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
 github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/grafana/pyroscope-go v1.0.4 h1:oyQX0BOkL+iARXzHuCdIF5TQ7/sRSel1YFViMHC7Bm0=
+github.com/grafana/pyroscope-go v1.0.4/go.mod h1:0d7ftwSMBV/Awm7CCiYmHQEG8Y44Ma3YSjt+nWcWztY=
+github.com/grafana/pyroscope-go/godeltaprof v0.1.4 h1:mDsJ3ngul7UfrHibGQpV66PbZ3q1T8glz/tK3bQKKEk=
+github.com/grafana/pyroscope-go/godeltaprof v0.1.4/go.mod h1:1HSPtjU8vLG0jE9JrTdzjgFqdJ/VgN7fvxBNq3luJko=
 github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE=
 github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs=
 github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao=