mirror of
https://github.com/scroll-tech/scroll.git
synced 2026-01-11 23:18:07 -05:00
Compare commits
3 Commits
develop
...
steven/mem
| Author | SHA1 | Date | |
|---|---|---|---|
|
9221fc25b6 | ||
|
|
91289a9b5d | ||
|
|
bdc09f7f3d |
57
common/libzkp/impl/Cargo.lock
generated
57
common/libzkp/impl/Cargo.lock
generated
@@ -31,7 +31,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "aggregator"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.7#2055cc0bb970aa28d597c945b6078e2469af8862"
|
||||
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=mem-leak-test#3502843bee8fd4e77350b85b6f1ad53ece6e6299"
|
||||
dependencies = [
|
||||
"ark-std",
|
||||
"env_logger 0.10.0",
|
||||
@@ -333,7 +333,7 @@ checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1"
|
||||
[[package]]
|
||||
name = "bus-mapping"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.7#2055cc0bb970aa28d597c945b6078e2469af8862"
|
||||
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=mem-leak-test#3502843bee8fd4e77350b85b6f1ad53ece6e6299"
|
||||
dependencies = [
|
||||
"eth-types",
|
||||
"ethers-core",
|
||||
@@ -959,7 +959,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "eth-types"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.7#2055cc0bb970aa28d597c945b6078e2469af8862"
|
||||
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=mem-leak-test#3502843bee8fd4e77350b85b6f1ad53ece6e6299"
|
||||
dependencies = [
|
||||
"ethers-core",
|
||||
"ethers-signers",
|
||||
@@ -968,6 +968,7 @@ dependencies = [
|
||||
"itertools",
|
||||
"lazy_static",
|
||||
"libsecp256k1",
|
||||
"log",
|
||||
"num",
|
||||
"num-bigint",
|
||||
"once_cell",
|
||||
@@ -1116,7 +1117,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "external-tracer"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.7#2055cc0bb970aa28d597c945b6078e2469af8862"
|
||||
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=mem-leak-test#3502843bee8fd4e77350b85b6f1ad53ece6e6299"
|
||||
dependencies = [
|
||||
"eth-types",
|
||||
"geth-utils",
|
||||
@@ -1179,6 +1180,12 @@ dependencies = [
|
||||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fs_extra"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "funty"
|
||||
version = "2.0.0"
|
||||
@@ -1296,7 +1303,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "gadgets"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.7#2055cc0bb970aa28d597c945b6078e2469af8862"
|
||||
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=mem-leak-test#3502843bee8fd4e77350b85b6f1ad53ece6e6299"
|
||||
dependencies = [
|
||||
"digest 0.7.6",
|
||||
"eth-types",
|
||||
@@ -1328,7 +1335,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "geth-utils"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.7#2055cc0bb970aa28d597c945b6078e2469af8862"
|
||||
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=mem-leak-test#3502843bee8fd4e77350b85b6f1ad53ece6e6299"
|
||||
dependencies = [
|
||||
"env_logger 0.9.3",
|
||||
"gobuild 0.1.0-alpha.2 (git+https://github.com/scroll-tech/gobuild.git)",
|
||||
@@ -1519,7 +1526,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "halo2_proofs"
|
||||
version = "0.2.0"
|
||||
source = "git+https://github.com/scroll-tech/halo2.git?branch=develop#e3fe25eadd714fd991f35190d17ff0b8fb031188"
|
||||
source = "git+https://github.com/scroll-tech/halo2.git?branch=develop#92fe9b3e2420ee51336b39513e397df7c8175cf9"
|
||||
dependencies = [
|
||||
"ark-std",
|
||||
"blake2b_simd",
|
||||
@@ -1889,6 +1896,27 @@ version = "1.0.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
|
||||
|
||||
[[package]]
|
||||
name = "jemalloc-sys"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d3b9f3f5c9b31aa0f5ed3260385ac205db665baa41d49bb8338008ae94ede45"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"fs_extra",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jemallocator"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "43ae63fcfc45e99ab3d1b29a46782ad679e98436c3169d15a167a1108a724b69"
|
||||
dependencies = [
|
||||
"jemalloc-sys",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.64"
|
||||
@@ -1937,7 +1965,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "keccak256"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.7#2055cc0bb970aa28d597c945b6078e2469af8862"
|
||||
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=mem-leak-test#3502843bee8fd4e77350b85b6f1ad53ece6e6299"
|
||||
dependencies = [
|
||||
"env_logger 0.9.3",
|
||||
"eth-types",
|
||||
@@ -2135,7 +2163,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "mock"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.7#2055cc0bb970aa28d597c945b6078e2469af8862"
|
||||
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=mem-leak-test#3502843bee8fd4e77350b85b6f1ad53ece6e6299"
|
||||
dependencies = [
|
||||
"eth-types",
|
||||
"ethers-core",
|
||||
@@ -2151,7 +2179,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "mpt-zktrie"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.7#2055cc0bb970aa28d597c945b6078e2469af8862"
|
||||
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=mem-leak-test#3502843bee8fd4e77350b85b6f1ad53ece6e6299"
|
||||
dependencies = [
|
||||
"eth-types",
|
||||
"halo2-mpt-circuits",
|
||||
@@ -2582,7 +2610,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "prover"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.7#2055cc0bb970aa28d597c945b6078e2469af8862"
|
||||
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=mem-leak-test#3502843bee8fd4e77350b85b6f1ad53ece6e6299"
|
||||
dependencies = [
|
||||
"aggregator",
|
||||
"anyhow",
|
||||
@@ -3196,9 +3224,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.104"
|
||||
version = "1.0.107"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c"
|
||||
checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"ryu",
|
||||
@@ -4125,7 +4153,7 @@ checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9"
|
||||
[[package]]
|
||||
name = "zkevm-circuits"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.7#2055cc0bb970aa28d597c945b6078e2469af8862"
|
||||
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=mem-leak-test#3502843bee8fd4e77350b85b6f1ad53ece6e6299"
|
||||
dependencies = [
|
||||
"array-init",
|
||||
"bus-mapping",
|
||||
@@ -4173,6 +4201,7 @@ dependencies = [
|
||||
"base64 0.13.1",
|
||||
"env_logger 0.9.3",
|
||||
"halo2_proofs",
|
||||
"jemallocator",
|
||||
"libc",
|
||||
"log",
|
||||
"once_cell",
|
||||
|
||||
@@ -7,10 +7,29 @@ edition = "2021"
|
||||
[lib]
|
||||
crate-type = ["cdylib"]
|
||||
|
||||
[dependencies]
|
||||
base64 = "0.13.0"
|
||||
env_logger = "0.9.0"
|
||||
libc = "0.2"
|
||||
log = "0.4"
|
||||
once_cell = "1.8.0"
|
||||
serde = "1.0"
|
||||
serde_derive = "1.0"
|
||||
serde_json = "1.0.107"
|
||||
|
||||
halo2_proofs = { git = "https://github.com/scroll-tech/halo2.git", branch = "develop" }
|
||||
prover = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "mem-leak-test", default-features = false, features = ["parallel_syn", "scroll", "shanghai"] }
|
||||
|
||||
[target.'cfg(not(target_env = "msvc"))'.dependencies]
|
||||
jemallocator = {version = "0.3", features = ["disable_initial_exec_tls"]}
|
||||
|
||||
[patch.crates-io]
|
||||
ethers-core = { git = "https://github.com/scroll-tech/ethers-rs.git", branch = "v2.0.7" }
|
||||
[patch."https://github.com/privacy-scaling-explorations/halo2.git"]
|
||||
halo2_proofs = { git = "https://github.com/scroll-tech/halo2.git", branch = "develop" }
|
||||
# halo2_proofs = { path = "/home/scroll/steven/halo2-gpu/halo2_proofs" }
|
||||
# [patch."https://github.com/scroll-tech/halo2.git"]
|
||||
# halo2_proofs = { path = "/home/scroll/steven/halo2-gpu/halo2_proofs" }
|
||||
[patch."https://github.com/privacy-scaling-explorations/poseidon.git"]
|
||||
poseidon = { git = "https://github.com/scroll-tech/poseidon.git", branch = "scroll-dev-0220" }
|
||||
[patch."https://github.com/privacy-scaling-explorations/halo2wrong.git"]
|
||||
@@ -19,21 +38,10 @@ maingate = { git = "https://github.com/scroll-tech/halo2wrong", branch = "halo2-
|
||||
[patch."https://github.com/privacy-scaling-explorations/halo2curves.git"]
|
||||
halo2curves = { git = "https://github.com/scroll-tech/halo2curves.git", branch = "0.3.1-derive-serde" }
|
||||
|
||||
[dependencies]
|
||||
halo2_proofs = { git = "https://github.com/scroll-tech/halo2.git", branch = "develop" }
|
||||
prover = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.9.7", default-features = false, features = ["parallel_syn", "scroll", "shanghai"] }
|
||||
|
||||
base64 = "0.13.0"
|
||||
env_logger = "0.9.0"
|
||||
libc = "0.2"
|
||||
log = "0.4"
|
||||
once_cell = "1.8.0"
|
||||
serde = "1.0"
|
||||
serde_derive = "1.0"
|
||||
serde_json = "1.0.66"
|
||||
|
||||
[profile.test]
|
||||
opt-level = 3
|
||||
debug-assertions = true
|
||||
|
||||
[profile.release]
|
||||
opt-level = 3
|
||||
debug-assertions = true
|
||||
|
||||
@@ -8,14 +8,47 @@ use crate::{
|
||||
use libc::c_char;
|
||||
use prover::{
|
||||
consts::CHUNK_VK_FILENAME,
|
||||
utils::init_env_and_log,
|
||||
utils::{get_block_trace_from_file, init_env_and_log},
|
||||
zkevm::{Prover, Verifier},
|
||||
BlockTrace, ChunkProof,
|
||||
};
|
||||
use std::{cell::OnceCell, env, ptr::null};
|
||||
use std::{
|
||||
env,
|
||||
process::{Command, Stdio},
|
||||
ptr::null,
|
||||
};
|
||||
|
||||
static mut PROVER: OnceCell<Prover> = OnceCell::new();
|
||||
static mut VERIFIER: OnceCell<Verifier> = OnceCell::new();
|
||||
#[no_mangle]
|
||||
static mut PROVER: Option<Prover> = None;
|
||||
#[no_mangle]
|
||||
static mut VERIFIER: Option<Verifier> = None;
|
||||
|
||||
/// # Safety
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn test_chunk(params_dir: *const c_char, assets_dir: *const c_char) {
|
||||
let params_dir = c_char_to_str(params_dir);
|
||||
let assets_dir = c_char_to_str(assets_dir);
|
||||
|
||||
init_env_and_log("chunk_tests");
|
||||
|
||||
env::set_var("SCROLL_PROVER_ASSETS_DIR", assets_dir);
|
||||
|
||||
let prover = Prover::from_dirs(params_dir, assets_dir);
|
||||
PROVER = Some(prover);
|
||||
log::info!("Constructed chunk prover");
|
||||
|
||||
let chunk_trace = vec![get_block_trace_from_file("/assets/traces/1_transfer.json")];
|
||||
log::info!("Loaded chunk trace");
|
||||
|
||||
for i in 0..50 {
|
||||
log::info!("Proof-{i} BEGIN mem: {}", mem_usage());
|
||||
PROVER
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.gen_chunk_proof(chunk_trace.clone(), None, None, None);
|
||||
log::info!("Proof-{i} END mem: {}", mem_usage());
|
||||
}
|
||||
}
|
||||
|
||||
/// # Safety
|
||||
#[no_mangle]
|
||||
@@ -35,7 +68,7 @@ pub unsafe extern "C" fn init_chunk_prover(params_dir: *const c_char, assets_dir
|
||||
|
||||
let prover = Prover::from_dirs(params_dir, assets_dir);
|
||||
|
||||
PROVER.set(prover).unwrap();
|
||||
PROVER = Some(prover);
|
||||
}
|
||||
|
||||
/// # Safety
|
||||
@@ -50,13 +83,13 @@ pub unsafe extern "C" fn init_chunk_verifier(params_dir: *const c_char, assets_d
|
||||
env::set_var("SCROLL_PROVER_ASSETS_DIR", assets_dir);
|
||||
let verifier = Verifier::from_dirs(params_dir, assets_dir);
|
||||
|
||||
VERIFIER.set(verifier).unwrap();
|
||||
VERIFIER = Some(verifier);
|
||||
}
|
||||
|
||||
/// # Safety
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn get_chunk_vk() -> *const c_char {
|
||||
let vk_result = panic_catch(|| PROVER.get_mut().unwrap().get_vk());
|
||||
let vk_result = panic_catch(|| PROVER.as_mut().unwrap().get_vk());
|
||||
|
||||
vk_result
|
||||
.ok()
|
||||
@@ -66,34 +99,32 @@ pub unsafe extern "C" fn get_chunk_vk() -> *const c_char {
|
||||
|
||||
/// # Safety
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn gen_chunk_proof(block_traces: *const c_char) -> *const c_char {
|
||||
let proof_result: Result<Vec<u8>, String> = panic_catch(|| {
|
||||
let block_traces = c_char_to_vec(block_traces);
|
||||
let block_traces = serde_json::from_slice::<Vec<BlockTrace>>(&block_traces)
|
||||
.map_err(|e| format!("failed to deserialize block traces: {e:?}"))?;
|
||||
pub unsafe extern "C" fn gen_chunk_proof(
|
||||
block_traces: *mut u8,
|
||||
len: libc::c_uint,
|
||||
) -> *const c_char {
|
||||
log::warn!("gupeng - aaaaa");
|
||||
// return null();
|
||||
|
||||
let proof = PROVER
|
||||
.get_mut()
|
||||
.expect("failed to get mutable reference to PROVER.")
|
||||
.gen_chunk_proof(block_traces, None, None, OUTPUT_DIR.as_deref())
|
||||
.map_err(|e| format!("failed to generate proof: {e:?}"))?;
|
||||
// let block_traces1 = c_char_to_vec(block_traces);
|
||||
// let block_traces1 = std::slice::from_raw_parts(block_traces, len as usize);
|
||||
|
||||
serde_json::to_vec(&proof).map_err(|e| format!("failed to serialize the proof: {e:?}"))
|
||||
})
|
||||
.unwrap_or_else(|e| Err(format!("unwind error: {e:?}")));
|
||||
// let block_traces = serde_json::from_slice::<Vec<BlockTrace>>(block_traces1).unwrap();
|
||||
|
||||
let r = match proof_result {
|
||||
Ok(proof_bytes) => ProofResult {
|
||||
message: Some(proof_bytes),
|
||||
error: None,
|
||||
},
|
||||
Err(err) => ProofResult {
|
||||
message: None,
|
||||
error: Some(err),
|
||||
},
|
||||
};
|
||||
let block_traces = vec![get_block_trace_from_file("/assets/traces/1_transfer.json")];
|
||||
log::info!("Loaded chunk trace");
|
||||
|
||||
serde_json::to_vec(&r).map_or(null(), vec_to_c_char)
|
||||
// let block_traces = vec![get_block_trace_from_file("/assets/traces/1_transfer.json")];
|
||||
// log::info!("Loaded chunk trace");
|
||||
|
||||
let prover = PROVER
|
||||
.as_mut()
|
||||
.expect("failed to get mutable reference to PROVER.");
|
||||
|
||||
prover.gen_chunk_proof(block_traces, None, None, None); // OUTPUT_DIR.as_deref());
|
||||
log::warn!("gupeng - kkkkk");
|
||||
|
||||
null()
|
||||
}
|
||||
|
||||
/// # Safety
|
||||
@@ -102,6 +133,12 @@ pub unsafe extern "C" fn verify_chunk_proof(proof: *const c_char) -> c_char {
|
||||
let proof = c_char_to_vec(proof);
|
||||
let proof = serde_json::from_slice::<ChunkProof>(proof.as_slice()).unwrap();
|
||||
|
||||
let verified = panic_catch(|| VERIFIER.get().unwrap().verify_chunk_proof(proof));
|
||||
let verified = panic_catch(|| VERIFIER.as_mut().unwrap().verify_chunk_proof(proof));
|
||||
verified.unwrap_or(false) as c_char
|
||||
}
|
||||
|
||||
fn mem_usage() -> String {
|
||||
let cmd = "echo \"$(date '+%Y-%m-%d %H:%M:%S') $(free -g | grep Mem: | sed 's/Mem://g')\"";
|
||||
let output = Command::new("bash").arg("-c").arg(cmd).output().unwrap();
|
||||
String::from_utf8_lossy(&output.stdout).to_string()
|
||||
}
|
||||
|
||||
@@ -1,5 +1,12 @@
|
||||
#![feature(once_cell)]
|
||||
|
||||
#[cfg(not(target_env = "msvc"))]
|
||||
use jemallocator::Jemalloc;
|
||||
|
||||
#[cfg(not(target_env = "msvc"))]
|
||||
#[global_allocator]
|
||||
static GLOBAL: Jemalloc = Jemalloc;
|
||||
|
||||
mod batch;
|
||||
mod chunk;
|
||||
mod types;
|
||||
|
||||
@@ -5,10 +5,11 @@ char* check_chunk_proofs(char* chunk_proofs);
|
||||
char* gen_batch_proof(char* chunk_hashes, char* chunk_proofs);
|
||||
char verify_batch_proof(char* proof);
|
||||
|
||||
void test_chunk(char* params_dir, char* assets_dir);
|
||||
void init_chunk_prover(char* params_dir, char* assets_dir);
|
||||
void init_chunk_verifier(char* params_dir, char* assets_dir);
|
||||
char* get_chunk_vk();
|
||||
char* gen_chunk_proof(char* block_traces);
|
||||
char* gen_chunk_proof(char* block_traces, unsigned int len);
|
||||
char verify_chunk_proof(char* proof);
|
||||
|
||||
char* block_traces_to_chunk_info(char* block_traces);
|
||||
|
||||
@@ -22,7 +22,7 @@ var (
|
||||
assetsPath = flag.String("assets", "/assets/test_assets", "assets dir")
|
||||
batchProofPath = flag.String("batch_proof", "/assets/proof_data/batch_proof", "batch proof file path")
|
||||
chunkProofPath1 = flag.String("chunk_proof1", "/assets/proof_data/chunk_proof1", "chunk proof file path 1")
|
||||
chunkProofPath2 = flag.String("chunk_proof2", "/assets/proof_data/chunk_proof2", "chunk proof file path 2")
|
||||
// chunkProofPath2 = flag.String("chunk_proof2", "/assets/proof_data/chunk_proof2", "chunk proof file path 2")
|
||||
)
|
||||
|
||||
func TestFFI(t *testing.T) {
|
||||
@@ -43,11 +43,13 @@ func TestFFI(t *testing.T) {
|
||||
as.True(chunkOk1)
|
||||
t.Log("Verified chunk proof 1")
|
||||
|
||||
chunkProof2 := readChunkProof(*chunkProofPath2, as)
|
||||
chunkOk2, err := v.VerifyChunkProof(chunkProof2)
|
||||
as.NoError(err)
|
||||
as.True(chunkOk2)
|
||||
t.Log("Verified chunk proof 2")
|
||||
/*
|
||||
chunkProof2 := readChunkProof(*chunkProofPath2, as)
|
||||
chunkOk2, err := v.VerifyChunkProof(chunkProof2)
|
||||
as.NoError(err)
|
||||
as.True(chunkOk2)
|
||||
t.Log("Verified chunk proof 2")
|
||||
*/
|
||||
|
||||
batchProof := readBatchProof(*batchProofPath, as)
|
||||
batchOk, err := v.VerifyBatchProof(batchProof)
|
||||
|
||||
@@ -19,7 +19,7 @@ else
|
||||
endif
|
||||
|
||||
libzkp:
|
||||
cd ../common/libzkp/impl && cargo clean && cargo build --release && cp ./target/release/libzkp.so ../interface/
|
||||
cd ../common/libzkp/impl && rm -f ./target/release/libzkp.so && cargo build --release && cp ./target/release/libzkp.so ../interface/
|
||||
rm -rf ./core/lib && cp -r ../common/libzkp/interface ./core/lib
|
||||
find ../common | grep libzktrie.so | xargs -I{} cp {} ./core/lib/
|
||||
|
||||
|
||||
@@ -3,35 +3,32 @@
|
||||
package core
|
||||
|
||||
/*
|
||||
#cgo LDFLAGS: -lzkp -lm -ldl -lzktrie -L${SRCDIR}/lib/ -Wl,-rpath=${SRCDIR}/lib
|
||||
#cgo gpu LDFLAGS: -lzkp -lm -ldl -lgmp -lstdc++ -lprocps -lzktrie -L/usr/local/cuda/lib64/ -lcudart -L${SRCDIR}/lib/ -Wl,-rpath=${SRCDIR}/lib
|
||||
#include <stdlib.h>
|
||||
#include "./lib/libzkp.h"
|
||||
#cgo LDFLAGS: -lpthread -lzkp -lm -ldl -lzktrie -L${SRCDIR}/lib/ -Wl,-rpath=${SRCDIR}/lib
|
||||
#cgo gpu LDFLAGS: -lpthread -lzkp -lm -ldl -lgmp -lstdc++ -lprocps -lzktrie -L/usr/local/cuda/lib64/ -lcudart -L${SRCDIR}/lib/ -Wl,-rpath=${SRCDIR}/lib
|
||||
# include <stdio.h>
|
||||
# include <stdlib.h>
|
||||
# include "./lib/libzkp.h"
|
||||
*/
|
||||
import "C" //nolint:typecheck
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"unsafe"
|
||||
|
||||
"github.com/scroll-tech/go-ethereum/core/types"
|
||||
"github.com/scroll-tech/go-ethereum/log"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
|
||||
"scroll-tech/common/types/message"
|
||||
|
||||
"scroll-tech/prover/config"
|
||||
)
|
||||
|
||||
// ProverCore sends block-traces to rust-prover through ffi and get back the zk-proof.
|
||||
type ProverCore struct {
|
||||
cfg *config.ProverCoreConfig
|
||||
VK string
|
||||
}
|
||||
|
||||
// NewProverCore inits a ProverCore object.
|
||||
func NewProverCore(cfg *config.ProverCoreConfig) (*ProverCore, error) {
|
||||
paramsPathStr := C.CString(cfg.ParamsPath)
|
||||
assetsPathStr := C.CString(cfg.AssetsPath)
|
||||
@@ -41,218 +38,35 @@ func NewProverCore(cfg *config.ProverCoreConfig) (*ProverCore, error) {
|
||||
}()
|
||||
|
||||
var vk string
|
||||
var rawVK *C.char
|
||||
if cfg.ProofType == message.ProofTypeBatch {
|
||||
C.init_batch_prover(paramsPathStr, assetsPathStr)
|
||||
rawVK = C.get_batch_vk()
|
||||
} else if cfg.ProofType == message.ProofTypeChunk {
|
||||
C.init_chunk_prover(paramsPathStr, assetsPathStr)
|
||||
rawVK = C.get_chunk_vk()
|
||||
}
|
||||
defer C.free_c_chars(rawVK)
|
||||
|
||||
if rawVK != nil {
|
||||
vk = C.GoString(rawVK)
|
||||
}
|
||||
|
||||
if cfg.DumpDir != "" {
|
||||
err := os.MkdirAll(cfg.DumpDir, os.ModePerm)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.Info("Enabled dump_proof", "dir", cfg.DumpDir)
|
||||
}
|
||||
C.init_chunk_prover(paramsPathStr, assetsPathStr)
|
||||
|
||||
return &ProverCore{cfg: cfg, VK: vk}, nil
|
||||
}
|
||||
|
||||
// ProveBatch call rust ffi to generate batch proof.
|
||||
func (p *ProverCore) ProveBatch(taskID string, chunkInfos []*message.ChunkInfo, chunkProofs []*message.ChunkProof) (*message.BatchProof, error) {
|
||||
if p.cfg.ProofType != message.ProofTypeBatch {
|
||||
return nil, fmt.Errorf("prover is not a batch-prover (type: %v), but is trying to prove a batch", p.cfg.ProofType)
|
||||
}
|
||||
|
||||
chunkInfosByt, err := json.Marshal(chunkInfos)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
chunkProofsByt, err := json.Marshal(chunkProofs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
isValid, err := p.checkChunkProofs(chunkProofsByt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !isValid {
|
||||
return nil, fmt.Errorf("non-match chunk protocol, task-id: %s", taskID)
|
||||
}
|
||||
|
||||
proofByt, err := p.proveBatch(chunkInfosByt, chunkProofsByt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to generate batch proof: %v", err)
|
||||
}
|
||||
|
||||
err = p.mayDumpProof(taskID, proofByt)
|
||||
if err != nil {
|
||||
log.Error("Dump batch proof failed", "task-id", taskID, "error", err)
|
||||
}
|
||||
|
||||
zkProof := &message.BatchProof{}
|
||||
return zkProof, json.Unmarshal(proofByt, zkProof)
|
||||
}
|
||||
|
||||
// ProveChunk call rust ffi to generate chunk proof.
|
||||
func (p *ProverCore) ProveChunk(taskID string, traces []*types.BlockTrace) (*message.ChunkProof, error) {
|
||||
if p.cfg.ProofType != message.ProofTypeChunk {
|
||||
return nil, fmt.Errorf("prover is not a chunk-prover (type: %v), but is trying to prove a chunk", p.cfg.ProofType)
|
||||
}
|
||||
var json = jsoniter.ConfigCompatibleWithStandardLibrary
|
||||
tracesByt, _ := json.Marshal(traces)
|
||||
|
||||
tracesByt, err := json.Marshal(traces)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
proofByt, err := p.proveChunk(tracesByt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.proveChunk(tracesByt)
|
||||
|
||||
err = p.mayDumpProof(taskID, proofByt)
|
||||
if err != nil {
|
||||
log.Error("Dump chunk proof failed", "task-id", taskID, "error", err)
|
||||
}
|
||||
|
||||
zkProof := &message.ChunkProof{}
|
||||
return zkProof, json.Unmarshal(proofByt, zkProof)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// TracesToChunkInfo convert traces to chunk info
|
||||
func (p *ProverCore) TracesToChunkInfo(traces []*types.BlockTrace) (*message.ChunkInfo, error) {
|
||||
tracesByt, err := json.Marshal(traces)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
chunkInfoByt := p.tracesToChunkInfo(tracesByt)
|
||||
|
||||
chunkInfo := &message.ChunkInfo{}
|
||||
return chunkInfo, json.Unmarshal(chunkInfoByt, chunkInfo)
|
||||
}
|
||||
|
||||
// CheckChunkProofsResponse represents the result of a chunk proof checking operation.
|
||||
// Ok indicates whether the proof checking was successful.
|
||||
// Error provides additional details in case the check failed.
|
||||
type CheckChunkProofsResponse struct {
|
||||
Ok bool `json:"ok"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// ProofResult encapsulates the result from generating a proof.
|
||||
// Message holds the generated proof in byte slice format.
|
||||
// Error provides additional details in case the proof generation failed.
|
||||
type ProofResult struct {
|
||||
Message []byte `json:"message,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func (p *ProverCore) checkChunkProofs(chunkProofsByt []byte) (bool, error) {
|
||||
chunkProofsStr := C.CString(string(chunkProofsByt))
|
||||
defer C.free(unsafe.Pointer(chunkProofsStr))
|
||||
|
||||
log.Info("Start to check chunk proofs ...")
|
||||
cResult := C.check_chunk_proofs(chunkProofsStr)
|
||||
defer C.free_c_chars(cResult)
|
||||
log.Info("Finish checking chunk proofs!")
|
||||
|
||||
var result CheckChunkProofsResponse
|
||||
err := json.Unmarshal([]byte(C.GoString(cResult)), &result)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to parse check chunk proofs result: %v", err)
|
||||
}
|
||||
|
||||
if result.Error != "" {
|
||||
return false, fmt.Errorf("failed to check chunk proofs: %s", result.Error)
|
||||
}
|
||||
|
||||
return result.Ok, nil
|
||||
}
|
||||
|
||||
func (p *ProverCore) proveBatch(chunkInfosByt []byte, chunkProofsByt []byte) ([]byte, error) {
|
||||
chunkInfosStr := C.CString(string(chunkInfosByt))
|
||||
chunkProofsStr := C.CString(string(chunkProofsByt))
|
||||
|
||||
defer func() {
|
||||
C.free(unsafe.Pointer(chunkInfosStr))
|
||||
C.free(unsafe.Pointer(chunkProofsStr))
|
||||
}()
|
||||
|
||||
log.Info("Start to create batch proof ...")
|
||||
bResult := C.gen_batch_proof(chunkInfosStr, chunkProofsStr)
|
||||
defer C.free_c_chars(bResult)
|
||||
log.Info("Finish creating batch proof!")
|
||||
|
||||
var result ProofResult
|
||||
err := json.Unmarshal([]byte(C.GoString(bResult)), &result)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse batch proof result: %v", err)
|
||||
}
|
||||
|
||||
if result.Error != "" {
|
||||
return nil, fmt.Errorf("failed to generate batch proof: %s", result.Error)
|
||||
}
|
||||
|
||||
return result.Message, nil
|
||||
type MyString struct {
|
||||
Str *C.char
|
||||
Len int
|
||||
}
|
||||
|
||||
func (p *ProverCore) proveChunk(tracesByt []byte) ([]byte, error) {
|
||||
tracesStr := C.CString(string(tracesByt))
|
||||
defer C.free(unsafe.Pointer(tracesStr))
|
||||
|
||||
log.Info("Start to create chunk proof ...")
|
||||
cProof := C.gen_chunk_proof(tracesStr)
|
||||
|
||||
len := len(tracesByt)
|
||||
ptr := C.CBytes(tracesByt)
|
||||
|
||||
cProof := C.gen_chunk_proof((*C.char)(ptr), C.uint(len))
|
||||
C.free(ptr)
|
||||
defer C.free_c_chars(cProof)
|
||||
log.Info("Finish creating chunk proof!")
|
||||
|
||||
var result ProofResult
|
||||
err := json.Unmarshal([]byte(C.GoString(cProof)), &result)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse chunk proof result: %v", err)
|
||||
}
|
||||
|
||||
if result.Error != "" {
|
||||
return nil, fmt.Errorf("failed to generate chunk proof: %s", result.Error)
|
||||
}
|
||||
|
||||
return result.Message, nil
|
||||
}
|
||||
|
||||
func (p *ProverCore) mayDumpProof(id string, proofByt []byte) error {
|
||||
if p.cfg.DumpDir == "" {
|
||||
return nil
|
||||
}
|
||||
path := filepath.Join(p.cfg.DumpDir, id)
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if err = f.Close(); err != nil {
|
||||
log.Error("failed to close proof dump file", "id", id, "error", err)
|
||||
}
|
||||
}()
|
||||
log.Info("Saving proof", "task-id", id)
|
||||
_, err = f.Write(proofByt)
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *ProverCore) tracesToChunkInfo(tracesByt []byte) []byte {
|
||||
tracesStr := C.CString(string(tracesByt))
|
||||
defer C.free(unsafe.Pointer(tracesStr))
|
||||
|
||||
cChunkInfo := C.block_traces_to_chunk_info(tracesStr)
|
||||
defer C.free_c_chars(cChunkInfo)
|
||||
|
||||
chunkInfo := C.GoString(cChunkInfo)
|
||||
return []byte(chunkInfo)
|
||||
return make([]byte, 1), nil
|
||||
}
|
||||
|
||||
@@ -4,16 +4,19 @@
|
||||
package core_test
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"io"
|
||||
"os"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/scroll-tech/go-ethereum/core/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
|
||||
"scroll-tech/common/types/message"
|
||||
|
||||
"scroll-tech/prover/config"
|
||||
@@ -21,93 +24,80 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
paramsPath = flag.String("params", "/assets/test_params", "params dir")
|
||||
assetsPath = flag.String("assets", "/assets/test_assets", "assets dir")
|
||||
proofDumpPath = flag.String("dump", "/assets/proof_data", "the path proofs dump to")
|
||||
tracePath1 = flag.String("trace1", "/assets/traces/1_transfer.json", "chunk trace 1")
|
||||
tracePath2 = flag.String("trace2", "/assets/traces/10_transfer.json", "chunk trace 2")
|
||||
batchVkPath = flag.String("batch-vk", "/assets/test_assets/agg_vk.vkey", "batch vk")
|
||||
chunkVkPath = flag.String("chunk-vk", "/assets/test_assets/chunk_vk.vkey", "chunk vk")
|
||||
paramsPath = flag.String("params", "/assets/test_params", "params dir")
|
||||
assetsPath = flag.String("assets", "/assets/test_assets", "assets dir")
|
||||
tracePath1 = flag.String("trace1", "/assets/traces/1_transfer.json", "chunk trace 1")
|
||||
)
|
||||
|
||||
func TestFFI(t *testing.T) {
|
||||
as := assert.New(t)
|
||||
|
||||
chunkProverConfig := &config.ProverCoreConfig{
|
||||
DumpDir: *proofDumpPath,
|
||||
ParamsPath: *paramsPath,
|
||||
AssetsPath: *assetsPath,
|
||||
ProofType: message.ProofTypeChunk,
|
||||
}
|
||||
chunkProverCore, err := core.NewProverCore(chunkProverConfig)
|
||||
as.NoError(err)
|
||||
t.Log("Constructed chunk prover")
|
||||
|
||||
as.Equal(chunkProverCore.VK, readVk(*chunkVkPath, as))
|
||||
t.Log("Chunk VK must be available when init")
|
||||
|
||||
chunkProverCore, _ := core.NewProverCore(chunkProverConfig)
|
||||
chunkTrace1 := readChunkTrace(*tracePath1, as)
|
||||
chunkTrace2 := readChunkTrace(*tracePath2, as)
|
||||
t.Log("Loaded chunk traces")
|
||||
|
||||
chunkInfo1, err := chunkProverCore.TracesToChunkInfo(chunkTrace1)
|
||||
as.NoError(err)
|
||||
chunkInfo2, err := chunkProverCore.TracesToChunkInfo(chunkTrace2)
|
||||
as.NoError(err)
|
||||
t.Log("Converted to chunk infos")
|
||||
|
||||
chunkProof1, err := chunkProverCore.ProveChunk("chunk_proof1", chunkTrace1)
|
||||
as.NoError(err)
|
||||
t.Log("Generated and dumped chunk proof 1")
|
||||
|
||||
chunkProof2, err := chunkProverCore.ProveChunk("chunk_proof2", chunkTrace2)
|
||||
as.NoError(err)
|
||||
t.Log("Generated and dumped chunk proof 2")
|
||||
|
||||
as.Equal(chunkProverCore.VK, readVk(*chunkVkPath, as))
|
||||
t.Log("Chunk VKs must be equal after proving")
|
||||
|
||||
batchProverConfig := &config.ProverCoreConfig{
|
||||
DumpDir: *proofDumpPath,
|
||||
ParamsPath: *paramsPath,
|
||||
AssetsPath: *assetsPath,
|
||||
ProofType: message.ProofTypeBatch,
|
||||
for i := 1; i <= 2000; i++ {
|
||||
t.Log("Proof-", i, " BEGIN mem: ", memUsage(as))
|
||||
chunkProverCore.ProveChunk("chunk_proof1", chunkTrace1)
|
||||
t.Log("Proof-", i, " END mem: ", memUsage(as))
|
||||
}
|
||||
batchProverCore, err := core.NewProverCore(batchProverConfig)
|
||||
as.NoError(err)
|
||||
}
|
||||
|
||||
as.Equal(batchProverCore.VK, readVk(*batchVkPath, as))
|
||||
t.Log("Batch VK must be available when init")
|
||||
|
||||
chunkInfos := []*message.ChunkInfo{chunkInfo1, chunkInfo2}
|
||||
chunkProofs := []*message.ChunkProof{chunkProof1, chunkProof2}
|
||||
_, err = batchProverCore.ProveBatch("batch_proof", chunkInfos, chunkProofs)
|
||||
as.NoError(err)
|
||||
t.Log("Generated and dumped batch proof")
|
||||
|
||||
as.Equal(batchProverCore.VK, readVk(*batchVkPath, as))
|
||||
t.Log("Batch VKs must be equal after proving")
|
||||
var blockTracePool = sync.Pool{
|
||||
New: func() interface{} {
|
||||
return make([]byte, 0, 4000000000) // Adjust the capacity based on your needs
|
||||
},
|
||||
}
|
||||
|
||||
func readChunkTrace(filePat string, as *assert.Assertions) []*types.BlockTrace {
|
||||
f, err := os.Open(filePat)
|
||||
as.NoError(err)
|
||||
defer as.NoError(f.Close())
|
||||
byt, err := io.ReadAll(f)
|
||||
as.NoError(err)
|
||||
/* gupeng
|
||||
|
||||
buf := blockTracePool.Get().([]byte)
|
||||
defer blockTracePool.Put(buf)
|
||||
|
||||
byt, _ := ioutil.ReadFile(filePat)
|
||||
buf = append(buf[:0], byt...)
|
||||
|
||||
trace := &types.BlockTrace{}
|
||||
as.NoError(json.Unmarshal(byt, trace))
|
||||
var json = jsoniter.ConfigCompatibleWithStandardLibrary
|
||||
as.NoError(json.Unmarshal(buf, trace))
|
||||
|
||||
return []*types.BlockTrace{trace}
|
||||
|
||||
*/
|
||||
return []*types.BlockTrace{}
|
||||
}
|
||||
|
||||
func readVk(filePat string, as *assert.Assertions) string {
|
||||
f, err := os.Open(filePat)
|
||||
as.NoError(err)
|
||||
defer as.NoError(f.Close())
|
||||
byt, err := io.ReadAll(f)
|
||||
func memUsage(as *assert.Assertions) string {
|
||||
mem := "echo \"$(date '+%Y-%m-%d %H:%M:%S') $(free -g | grep Mem: | sed 's/Mem://g')\""
|
||||
cmd := exec.Command("bash", "-c", mem)
|
||||
|
||||
output, err := cmd.Output()
|
||||
as.NoError(err)
|
||||
|
||||
return base64.StdEncoding.EncodeToString(byt)
|
||||
return string(output)
|
||||
}
|
||||
|
||||
func printGC(msg string) {
|
||||
var stats runtime.MemStats
|
||||
runtime.ReadMemStats(&stats)
|
||||
|
||||
fmt.Printf("%s:\n", msg)
|
||||
fmt.Printf(" Alloc: %v MB\n", bToMb(stats.Alloc))
|
||||
fmt.Printf(" TotalAlloc: %v MB\n", bToMb(stats.TotalAlloc))
|
||||
fmt.Printf(" Sys: %v MB\n", bToMb(stats.Sys))
|
||||
fmt.Printf(" NumGC: %v\n", stats.NumGC)
|
||||
fmt.Printf(" NumGoroutine: %v\n", runtime.NumGoroutine())
|
||||
fmt.Printf(" HeapAlloc: %v MB\n", bToMb(stats.HeapAlloc))
|
||||
fmt.Printf(" HeapSys: %v MB\n", bToMb(stats.HeapSys))
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
func bToMb(b uint64) uint64 {
|
||||
return b / 1024 / 1024
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user