more logs

common/libzkp/impl/Cargo.lock

@@ -3573,6 +3573,26 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "tikv-jemalloc-sys"
+version = "0.5.4+5.3.0-patched"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9402443cb8fd499b6f327e40565234ff34dbda27460c5b47db0db77443dd85d1"
+dependencies = [
+ "cc",
+ "libc",
+]
+
+[[package]]
+name = "tikv-jemallocator"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "965fe0c26be5c56c94e38ba547249074803efd52adfb66de62107d95aab3eaca"
+dependencies = [
+ "libc",
+ "tikv-jemalloc-sys",
+]
+
 [[package]]
 name = "time"
 version = "0.1.45"
@@ -4180,6 +4200,7 @@ dependencies = [
  "serde",
  "serde_derive",
  "serde_json",
+ "tikv-jemallocator",
 ]
 
 [[package]]

common/libzkp/impl/Cargo.toml

@@ -22,6 +22,7 @@ halo2curves = { git = "https://github.com/scroll-tech/halo2curves.git", branch =
 [dependencies]
 halo2_proofs = { git = "https://github.com/scroll-tech/halo2.git", branch = "develop" }
 prover = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.9.7", default-features = false, features = ["parallel_syn", "scroll", "shanghai"] }
+tikv-jemallocator = "0.5.0"
 
 base64 = "0.13.0"
 env_logger = "0.9.0"

common/libzkp/impl/src/lib.rs

@@ -1,5 +1,12 @@
 #![feature(once_cell)]
 
+#[cfg(not(target_env = "msvc"))]
+use tikv_jemallocator::Jemalloc;
+
+#[cfg(not(target_env = "msvc"))]
+#[global_allocator]
+static ALLOCATOR: Jemalloc = Jemalloc;
+
 mod batch;
 mod chunk;
 mod types;

coordinator/internal/orm/batch.go

@@ -74,10 +74,6 @@ func (*Batch) TableName() string {
 // GetUnassignedBatch retrieves unassigned batch based on the specified limit.
 // The returned batch are sorted in ascending order by their index.
 func (o *Batch) GetUnassignedBatch(ctx context.Context, maxActiveAttempts, maxTotalAttempts uint8) (*Batch, error) {
-	t := time.Now()
-	defer func() {
-		log.Info("GetUnassignedBatch cost", "duration(ms)", time.Since(t).Milliseconds())
-	}()
 	db := o.db.WithContext(ctx)
 	db = db.Where("proving_status = ?", int(types.ProvingTaskUnassigned))
 	db = db.Where("total_attempts < ?", maxTotalAttempts)
@@ -99,10 +95,6 @@ func (o *Batch) GetUnassignedBatch(ctx context.Context, maxActiveAttempts, maxTo
 // GetAssignedBatch retrieves assigned batch based on the specified limit.
 // The returned batch are sorted in ascending order by their index.
 func (o *Batch) GetAssignedBatch(ctx context.Context, maxActiveAttempts, maxTotalAttempts uint8) (*Batch, error) {
-	t := time.Now()
-	defer func() {
-		log.Info("GetAssignedBatch cost", "duration(ms)", time.Since(t).Milliseconds())
-	}()
 	db := o.db.WithContext(ctx)
 	db = db.Where("proving_status = ?", int(types.ProvingTaskAssigned))
 	db = db.Where("total_attempts < ?", maxTotalAttempts)

coordinator/internal/orm/chunk.go

@@ -69,10 +69,6 @@ func (*Chunk) TableName() string {
 // GetUnassignedChunk retrieves unassigned chunk based on the specified limit.
 // The returned chunks are sorted in ascending order by their index.
 func (o *Chunk) GetUnassignedChunk(ctx context.Context, height int, maxActiveAttempts, maxTotalAttempts uint8) (*Chunk, error) {
-	t := time.Now()
-	defer func() {
-		log.Info("GetUnassignedChunk cost", "duration(ms)", time.Since(t).Milliseconds())
-	}()
 	db := o.db.WithContext(ctx)
 	db = db.Model(&Chunk{})
 	db = db.Where("proving_status = ?", int(types.ProvingTaskUnassigned))
@@ -95,10 +91,6 @@ func (o *Chunk) GetUnassignedChunk(ctx context.Context, height int, maxActiveAtt
 // GetAssignedChunk retrieves assigned chunk based on the specified limit.
 // The returned chunks are sorted in ascending order by their index.
 func (o *Chunk) GetAssignedChunk(ctx context.Context, height int, maxActiveAttempts, maxTotalAttempts uint8) (*Chunk, error) {
-	t := time.Now()
-	defer func() {
-		log.Info("GetAssignedChunk cost", "duration(ms)", time.Since(t).Milliseconds())
-	}()
 	db := o.db.WithContext(ctx)
 	db = db.Model(&Chunk{})
 	db = db.Where("proving_status = ?", int(types.ProvingTaskAssigned))

prover/cmd/main.go

@@ -1,7 +1,16 @@
 package main
 
-import "scroll-tech/prover/cmd/app"
+import (
+	_ "expvar"
+	"net/http"
+	_ "net/http/pprof"
+	"scroll-tech/prover/cmd/app"
+)
 
 func main() {
+	go func() {
+		http.ListenAndServe("0.0.0.0:6060", nil)
+	}()
+
 	app.Run()
 }

prover/core/prover.go

@@ -118,11 +118,13 @@ func (p *ProverCore) ProveChunk(taskID string, traces []*types.BlockTrace) (*mes
 		return nil, err
 	}
 
+	log.Info("dump proof")
 	err = p.mayDumpProof(taskID, proofByt)
 	if err != nil {
 		log.Error("Dump chunk proof failed", "task-id", taskID, "error", err)
 	}
 
+	log.Info("send proof back")
 	zkProof := &message.ChunkProof{}
 	return zkProof, json.Unmarshal(proofByt, zkProof)
 }
@@ -219,6 +221,7 @@ func (p *ProverCore) proveChunk(tracesByt []byte) ([]byte, error) {
 		return nil, fmt.Errorf("failed to parse chunk proof result: %v", err)
 	}
 
+	log.Info("unmarshal proof")
 	if result.Error != "" {
 		return nil, fmt.Errorf("failed to generate chunk proof: %s", result.Error)
 	}

prover/prover.go

@@ -166,6 +166,7 @@ func (r *Prover) proveAndSubmit() error {
 			log.Error("failed to prove task", "task_type", task.Task.Type, "task-id", task.Task.ID, "err", err)
 			return r.submitErr(task, message.ProofFailureNoPanic, err)
 		}
+		log.Info("submit proof")
 		return r.submitProof(proofMsg, task.Task.UUID)
 	}
 

run.sh

@@ -0,0 +1,5 @@
+#!/bin/bash -e 
+cp ./build/bin/prover ./gpu_prover
+
+MALLOC_CONF=prof_leak:true,lg_prof_sample:0,prof_final:true \
+LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2 ./gpu_prover