refactor(coordinator & prover): RESTful API (#696)

Co-authored-by: georgehao <haohongfan@gmail.com>
Co-authored-by: HAOYUatHZ <37070449+HAOYUatHZ@users.noreply.github.com>

common/libzkp/impl/src/batch.rs

@@ -8,7 +8,7 @@ use prover::{
 use std::{cell::OnceCell, panic, ptr::null};
 use types::eth::BlockTrace;
 
-static mut BATCH_PROVER: OnceCell<Prover> = OnceCell::new();
+static mut PROVER: OnceCell<Prover> = OnceCell::new();
 static mut VERIFIER: OnceCell<Verifier> = OnceCell::new();
 
 /// # Safety
@@ -19,7 +19,7 @@ pub unsafe extern "C" fn init_batch_prover(params_dir: *const c_char) {
     let params_dir = c_char_to_str(params_dir);
     let prover = Prover::from_params_dir(params_dir);
 
-    BATCH_PROVER.set(prover).unwrap();
+    PROVER.set(prover).unwrap();
 }
 
 /// # Safety
@@ -54,7 +54,7 @@ pub unsafe extern "C" fn gen_batch_proof(
         .collect();
 
     let proof_result = panic::catch_unwind(|| {
-        let proof = BATCH_PROVER
+        let proof = PROVER
             .get_mut()
             .unwrap()
             .gen_agg_evm_proof(chunk_hashes_proofs, None, OUTPUT_DIR.as_deref())

common/libzkp/impl/src/batch_verifier.rs

@@ -1,90 +0,0 @@
-use crate::utils::{c_char_to_str, c_char_to_vec, vec_to_c_char, OUTPUT_DIR};
-use libc::c_char;
-use prover::{
-    aggregator::{Prover, Verifier},
-    utils::{chunk_trace_to_witness_block, init_env_and_log},
-    BatchProof, ChunkHash, ChunkProof,
-};
-use std::{cell::OnceCell, panic, ptr::null};
-use types::eth::BlockTrace;
-
-static mut BATCH_PROVER: OnceCell<Prover> = OnceCell::new();
-static mut VERIFIER: OnceCell<Verifier> = OnceCell::new();
-
-/// # Safety
-#[no_mangle]
-pub unsafe extern "C" fn init_batch_prover(params_dir: *const c_char) {
-    init_env_and_log("ffi_batch_prove");
-
-    let params_dir = c_char_to_str(params_dir);
-    let prover = Prover::from_params_dir(params_dir);
-
-    BATCH_PROVER.set(prover).unwrap();
-}
-
-/// # Safety
-#[no_mangle]
-pub unsafe extern "C" fn init_batch_verifier(params_dir: *const c_char, assets_dir: *const c_char) {
-    init_env_and_log("ffi_batch_verify");
-
-    let params_dir = c_char_to_str(params_dir);
-    let assets_dir = c_char_to_str(assets_dir);
-
-    let verifier = Verifier::from_dirs(params_dir, assets_dir);
-
-    VERIFIER.set(verifier).unwrap();
-}
-
-/// # Safety
-#[no_mangle]
-pub unsafe extern "C" fn gen_batch_proof(
-    chunk_hashes: *const c_char,
-    chunk_proofs: *const c_char,
-) -> *const c_char {
-    let chunk_hashes = c_char_to_vec(chunk_hashes);
-    let chunk_proofs = c_char_to_vec(chunk_proofs);
-
-    let chunk_hashes = serde_json::from_slice::<Vec<ChunkHash>>(&chunk_hashes).unwrap();
-    let chunk_proofs = serde_json::from_slice::<Vec<ChunkProof>>(&chunk_proofs).unwrap();
-    assert_eq!(chunk_hashes.len(), chunk_proofs.len());
-
-    let chunk_hashes_proofs = chunk_hashes
-        .into_iter()
-        .zip(chunk_proofs.into_iter())
-        .collect();
-
-    let proof_result = panic::catch_unwind(|| {
-        let proof = BATCH_PROVER
-            .get_mut()
-            .unwrap()
-            .gen_agg_evm_proof(chunk_hashes_proofs, None, OUTPUT_DIR.as_deref())
-            .unwrap();
-
-        serde_json::to_vec(&proof).unwrap()
-    });
-    proof_result.map_or(null(), vec_to_c_char)
-}
-
-/// # Safety
-#[no_mangle]
-pub unsafe extern "C" fn verify_batch_proof(proof: *const c_char) -> c_char {
-    let proof = c_char_to_vec(proof);
-    let proof = serde_json::from_slice::<BatchProof>(proof.as_slice()).unwrap();
-
-    let verified = panic::catch_unwind(|| VERIFIER.get().unwrap().verify_agg_evm_proof(proof));
-    verified.unwrap_or(false) as c_char
-}
-
-// This function is only used for debugging on Go side.
-/// # Safety
-#[no_mangle]
-pub unsafe extern "C" fn block_traces_to_chunk_info(block_traces: *const c_char) -> *const c_char {
-    let block_traces = c_char_to_vec(block_traces);
-    let block_traces = serde_json::from_slice::<Vec<BlockTrace>>(&block_traces).unwrap();
-
-    let witness_block = chunk_trace_to_witness_block(block_traces).unwrap();
-    let chunk_info = ChunkHash::from_witness_block(&witness_block, false);
-
-    let chunk_info_bytes = serde_json::to_vec(&chunk_info).unwrap();
-    vec_to_c_char(chunk_info_bytes)
-}

common/libzkp/impl/src/chunk.rs

@@ -8,7 +8,7 @@ use prover::{
 use std::{cell::OnceCell, panic, ptr::null};
 use types::eth::BlockTrace;
 
-static mut CHUNK_PROVER: OnceCell<Prover> = OnceCell::new();
+static mut PROVER: OnceCell<Prover> = OnceCell::new();
 static mut VERIFIER: OnceCell<Verifier> = OnceCell::new();
 
 /// # Safety
@@ -19,7 +19,7 @@ pub unsafe extern "C" fn init_chunk_prover(params_dir: *const c_char) {
     let params_dir = c_char_to_str(params_dir);
     let prover = Prover::from_params_dir(params_dir);
 
-    CHUNK_PROVER.set(prover).unwrap();
+    PROVER.set(prover).unwrap();
 }
 
 /// # Safety
@@ -42,7 +42,7 @@ pub unsafe extern "C" fn gen_chunk_proof(block_traces: *const c_char) -> *const
     let block_traces = serde_json::from_slice::<Vec<BlockTrace>>(&block_traces).unwrap();
 
     let proof_result = panic::catch_unwind(|| {
-        let proof = CHUNK_PROVER
+        let proof = PROVER
             .get_mut()
             .unwrap()
             .gen_chunk_proof(block_traces, None, OUTPUT_DIR.as_deref())

common/libzkp/impl/src/chunk_verifier.rs

@@ -1,65 +0,0 @@
-use crate::utils::{c_char_to_str, c_char_to_vec, vec_to_c_char, OUTPUT_DIR};
-use libc::c_char;
-use prover::{
-    utils::init_env_and_log,
-    zkevm::{Prover, Verifier},
-    ChunkProof,
-};
-use std::{cell::OnceCell, panic, ptr::null};
-use types::eth::BlockTrace;
-
-static mut CHUNK_PROVER: OnceCell<Prover> = OnceCell::new();
-static mut VERIFIER: OnceCell<Verifier> = OnceCell::new();
-
-/// # Safety
-#[no_mangle]
-pub unsafe extern "C" fn init_chunk_prover(params_dir: *const c_char) {
-    init_env_and_log("ffi_chunk_prove");
-
-    let params_dir = c_char_to_str(params_dir);
-    let prover = Prover::from_params_dir(params_dir);
-
-    CHUNK_PROVER.set(prover).unwrap();
-}
-
-/// # Safety
-#[no_mangle]
-pub unsafe extern "C" fn init_chunk_verifier(params_dir: *const c_char, assets_dir: *const c_char) {
-    init_env_and_log("ffi_chunk_verify");
-
-    let params_dir = c_char_to_str(params_dir);
-    let assets_dir = c_char_to_str(assets_dir);
-
-    let verifier = Verifier::from_dirs(params_dir, assets_dir);
-
-    VERIFIER.set(verifier).unwrap();
-}
-
-/// # Safety
-#[no_mangle]
-pub unsafe extern "C" fn gen_chunk_proof(block_traces: *const c_char) -> *const c_char {
-    let block_traces = c_char_to_vec(block_traces);
-    let block_traces = serde_json::from_slice::<Vec<BlockTrace>>(&block_traces).unwrap();
-
-    let proof_result = panic::catch_unwind(|| {
-        let proof = CHUNK_PROVER
-            .get_mut()
-            .unwrap()
-            .gen_chunk_proof(block_traces, None, OUTPUT_DIR.as_deref())
-            .unwrap();
-
-        serde_json::to_vec(&proof).unwrap()
-    });
-
-    proof_result.map_or(null(), vec_to_c_char)
-}
-
-/// # Safety
-#[no_mangle]
-pub unsafe extern "C" fn verify_chunk_proof(proof: *const c_char) -> c_char {
-    let proof = c_char_to_vec(proof);
-    let proof = serde_json::from_slice::<ChunkProof>(proof.as_slice()).unwrap();
-
-    let verified = panic::catch_unwind(|| VERIFIER.get().unwrap().verify_chunk_proof(proof));
-    verified.unwrap_or(false) as c_char
-}

common/libzkp/impl/src/lib.rs

@@ -1,5 +1,5 @@
 #![feature(once_cell)]
 
-mod prove_batch;
-mod prove_chunk;
+mod batch;
+mod chunk;
 mod utils;

common/types/message/message.go

@@ -60,6 +60,8 @@ type AuthMsg struct {
 type Identity struct {
 	// ProverName the prover name
 	ProverName string `json:"prover_name"`
+	// ProverVersion the prover version
+	ProverVersion string `json:"prover_version"`
 	// Challenge unique challenge generated by manager
 	Challenge string `json:"challenge"`
 }

common/types/message/message_test.go

@@ -15,8 +15,9 @@ func TestAuthMessageSignAndVerify(t *testing.T) {
 
 	authMsg := &AuthMsg{
 		Identity: &Identity{
-			Challenge:  "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2OTEwMzgxNzUsIm9yaWdfaWF0IjoxNjkxMDM0NTc1fQ.HybBMsEJFhyZqtIa2iVcHUP7CEFttf708jmTMAImAWA",
-			ProverName: "test",
+			Challenge:     "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2OTEwMzgxNzUsIm9yaWdfaWF0IjoxNjkxMDM0NTc1fQ.HybBMsEJFhyZqtIa2iVcHUP7CEFttf708jmTMAImAWA",
+			ProverName:    "test",
+			ProverVersion: "v1.0.0",
 		},
 	}
 	assert.NoError(t, authMsg.SignWithKey(privkey))
@@ -45,14 +46,15 @@ func TestGenerateToken(t *testing.T) {
 
 func TestIdentityHash(t *testing.T) {
 	identity := &Identity{
-		Challenge:  "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2OTEwMzM0MTksIm9yaWdfaWF0IjoxNjkxMDI5ODE5fQ.EhkLZsj__rNPVC3ZDYBtvdh0nB8mmM_Hl82hObaIWOs",
-		ProverName: "test",
+		Challenge:     "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2OTEwMzM0MTksIm9yaWdfaWF0IjoxNjkxMDI5ODE5fQ.EhkLZsj__rNPVC3ZDYBtvdh0nB8mmM_Hl82hObaIWOs",
+		ProverName:    "test",
+		ProverVersion: "v1.0.0",
 	}
 
 	hash, err := identity.Hash()
 	assert.NoError(t, err)
 
-	expectedHash := "7373bb57ab7c01307d86fec55e088e00c526d82d9d1303fa4a189ff6ea95fbe2"
+	expectedHash := "83f5e0ad023e9c1de639ab07b9b4cb972ec9dbbd2524794c533a420a5b137721"
 	assert.Equal(t, expectedHash, hex.EncodeToString(hash))
 }
 

coordinator/internal/controller/api/auth.go

@@ -48,8 +48,9 @@ func (a *AuthController) PayloadFunc(data interface{}) jwt.MapClaims {
 	// recover the public key
 	authMsg := message.AuthMsg{
 		Identity: &message.Identity{
-			Challenge:  v.Message.Challenge,
-			ProverName: v.Message.ProverName,
+			Challenge:     v.Message.Challenge,
+			ProverName:    v.Message.ProverName,
+			ProverVersion: v.Message.ProverVersion,
 		},
 		Signature: v.Signature,
 	}
@@ -60,8 +61,9 @@ func (a *AuthController) PayloadFunc(data interface{}) jwt.MapClaims {
 	}
 
 	return jwt.MapClaims{
-		types.PublicKey:  publicKey,
-		types.ProverName: v.Message.ProverName,
+		types.PublicKey:     publicKey,
+		types.ProverName:    v.Message.ProverName,
+		types.ProverVersion: v.Message.ProverVersion,
 	}
 }
 
@@ -75,5 +77,9 @@ func (a *AuthController) IdentityHandler(c *gin.Context) interface{} {
 	if publicKey, ok := claims[types.PublicKey]; ok {
 		c.Set(types.PublicKey, publicKey)
 	}
+
+	if proverVersion, ok := claims[types.ProverVersion]; ok {
+		c.Set(types.ProverVersion, proverVersion)
+	}
 	return nil
 }

coordinator/internal/types/auth.go

@@ -7,12 +7,15 @@ const (
 	PublicKey = "public_key"
 	// ProverName the prover name key for context
 	ProverName = "prover_name"
+	// ProverVersion the prover version for context
+	ProverVersion = "prover_version"
 )
 
 // Message the login message struct
 type Message struct {
-	Challenge  string `form:"challenge" json:"challenge" binding:"required"`
-	ProverName string `form:"prover_name" json:"prover_name" binding:"required"`
+	Challenge     string `form:"challenge" json:"challenge" binding:"required"`
+	ProverVersion string `form:"prover_version" json:"prover_version" binding:"required"`
+	ProverName    string `form:"prover_name" json:"prover_name" binding:"required"`
 }
 
 // LoginParameter for /login api

coordinator/internal/types/get_task.go

@@ -2,9 +2,8 @@ package types
 
 // GetTaskParameter for ProverTasks request parameter
 type GetTaskParameter struct {
-	ProverVersion string `form:"prover_version" json:"prover_version" binding:"required"`
-	ProverHeight  int    `form:"prover_height" json:"prover_height" binding:"required"`
-	TaskType      int    `form:"task_type" json:"task_type"`
+	ProverHeight int `form:"prover_height" json:"prover_height" binding:"required"`
+	TaskType     int `form:"task_type" json:"task_type"`
 }
 
 // GetTaskSchema the schema data return to prover for get prover task

coordinator/test/mock_prover.go

@@ -76,14 +76,15 @@ func (r *mockProver) challenge(t *testing.T) string {
 func (r *mockProver) login(t *testing.T, challengeString string) string {
 	authMsg := message.AuthMsg{
 		Identity: &message.Identity{
-			Challenge:  challengeString,
-			ProverName: "test",
+			Challenge:     challengeString,
+			ProverName:    "test",
+			ProverVersion: "v1.0.0",
 		},
 	}
 	assert.NoError(t, authMsg.SignWithKey(r.privKey))
 
-	body := fmt.Sprintf("{\"message\":{\"challenge\":\"%s\",\"prover_name\":\"%s\"},\"signature\":\"%s\"}",
-		authMsg.Identity.Challenge, authMsg.Identity.ProverName, authMsg.Signature)
+	body := fmt.Sprintf("{\"message\":{\"challenge\":\"%s\",\"prover_name\":\"%s\", \"prover_version\":\"%s\"},\"signature\":\"%s\"}",
+		authMsg.Identity.Challenge, authMsg.Identity.ProverName, authMsg.Identity.ProverVersion, authMsg.Signature)
 
 	var result types.Response
 	client := resty.New()
@@ -137,7 +138,7 @@ func (r *mockProver) getProverTask(t *testing.T, proofType message.ProofType) *t
 	resp, err := client.R().
 		SetHeader("Content-Type", "application/json").
 		SetHeader("Authorization", fmt.Sprintf("Bearer %s", token)).
-		SetBody(map[string]interface{}{"prover_version": "v1.0.0", "prover_height": 100, "task_type": int(proofType)}).
+		SetBody(map[string]interface{}{"prover_height": 100, "task_type": int(proofType)}).
 		SetResult(&result).
 		Post("http://" + r.coordinatorURL + "/coordinator/v1/get_task")
 	assert.NoError(t, err)

prover/client/client.go

@@ -14,6 +14,7 @@ import (
 
 	"scroll-tech/common/types"
 	"scroll-tech/common/types/message"
+	"scroll-tech/common/version"
 )
 
 // CoordinatorClient is a client used for interacting with the Coordinator service.
@@ -65,8 +66,9 @@ func (c *CoordinatorClient) Login(ctx context.Context) error {
 	// Prepare and sign the login request
 	authMsg := &message.AuthMsg{
 		Identity: &message.Identity{
-			ProverName: c.proverName,
-			Challenge:  challengeResult.Data.Token,
+			ProverVersion: version.Version,
+			ProverName:    c.proverName,
+			Challenge:     challengeResult.Data.Token,
 		},
 	}
 
@@ -78,11 +80,13 @@ func (c *CoordinatorClient) Login(ctx context.Context) error {
 	// Login to coordinator
 	loginReq := &LoginRequest{
 		Message: struct {
-			Challenge  string `json:"challenge"`
-			ProverName string `json:"prover_name"`
+			Challenge     string `json:"challenge"`
+			ProverName    string `json:"prover_name"`
+			ProverVersion string `json:"prover_version"`
 		}{
-			Challenge:  authMsg.Identity.Challenge,
-			ProverName: authMsg.Identity.ProverName,
+			Challenge:     authMsg.Identity.Challenge,
+			ProverName:    authMsg.Identity.ProverName,
+			ProverVersion: authMsg.Identity.ProverVersion,
 		},
 		Signature: authMsg.Signature,
 	}

prover/client/types.go

@@ -17,8 +17,9 @@ type ChallengeResponse struct {
 // LoginRequest defines the request structure for login API
 type LoginRequest struct {
 	Message struct {
-		Challenge  string `json:"challenge"`
-		ProverName string `json:"prover_name"`
+		Challenge     string `json:"challenge"`
+		ProverName    string `json:"prover_name"`
+		ProverVersion string `json:"prover_version"`
 	} `json:"message"`
 	Signature string `json:"signature"`
 }
@@ -35,9 +36,8 @@ type LoginResponse struct {
 
 // GetTaskRequest defines the request structure for GetTask API
 type GetTaskRequest struct {
-	ProverVersion string            `json:"prover_version"`
-	ProverHeight  uint64            `json:"prover_height"`
-	TaskType      message.ProofType `json:"task_type"`
+	ProverHeight uint64            `json:"prover_height"`
+	TaskType     message.ProofType `json:"task_type"`
 }
 
 // GetTaskResponse defines the response structure for GetTask API

prover/core/prover_test.go

@@ -1,5 +1,6 @@
 //go:build ffi
 
+// go test -v -race -gcflags="-l" -ldflags="-s=false" -tags ffi ./...
 package core_test
 
 import (
@@ -12,6 +13,7 @@ import (
 	"github.com/scroll-tech/go-ethereum/core/types"
 	"github.com/stretchr/testify/assert"
 
+	scrollTypes "scroll-tech/common/types"
 	"scroll-tech/common/types/message"
 
 	"scroll-tech/prover/config"
@@ -47,6 +49,32 @@ func TestFFI(t *testing.T) {
 	as.NoError(err)
 	t.Log("Converted to chunk infos")
 
+	wrappedBlock1 := &scrollTypes.WrappedBlock{
+		Header:       chunkTrace1[0].Header,
+		Transactions: chunkTrace1[0].Transactions,
+		WithdrawRoot: chunkTrace1[0].WithdrawTrieRoot,
+	}
+	chunk1 := &scrollTypes.Chunk{Blocks: []*scrollTypes.WrappedBlock{wrappedBlock1}}
+	chunkHash1, err := chunk1.Hash(0)
+	as.NoError(err)
+	as.Equal(chunkInfo1.PostStateRoot, wrappedBlock1.Header.Root)
+	as.Equal(chunkInfo1.WithdrawRoot, wrappedBlock1.WithdrawRoot)
+	as.Equal(chunkInfo1.DataHash, chunkHash1)
+	t.Log("Successful to check chunk info 1")
+
+	wrappedBlock2 := &scrollTypes.WrappedBlock{
+		Header:       chunkTrace2[0].Header,
+		Transactions: chunkTrace2[0].Transactions,
+		WithdrawRoot: chunkTrace2[0].WithdrawTrieRoot,
+	}
+	chunk2 := &scrollTypes.Chunk{Blocks: []*scrollTypes.WrappedBlock{wrappedBlock2}}
+	chunkHash2, err := chunk2.Hash(chunk1.NumL1Messages(0))
+	as.NoError(err)
+	as.Equal(chunkInfo2.PostStateRoot, wrappedBlock2.Header.Root)
+	as.Equal(chunkInfo2.WithdrawRoot, wrappedBlock2.WithdrawRoot)
+	as.Equal(chunkInfo2.DataHash, chunkHash2)
+	t.Log("Successful to check chunk info 2")
+
 	chunkProof1, err := chunkProverCore.ProveChunk("chunk_proof1", chunkTrace1)
 	as.NoError(err)
 	t.Log("Generated and dumped chunk proof 1")

prover/prover.go

@@ -24,7 +24,6 @@ import (
 
 	"scroll-tech/common/types/message"
 	"scroll-tech/common/utils"
-	"scroll-tech/common/version"
 )
 
 var (
@@ -190,9 +189,8 @@ func (r *Prover) fetchTaskFromCoordinator() (*store.ProvingTask, error) {
 
 	// prepare the request
 	req := &client.GetTaskRequest{
-		ProverVersion: version.Version,
-		ProverHeight:  latestBlockNumber,
-		TaskType:      r.Type(),
+		ProverHeight: latestBlockNumber,
+		TaskType:     r.Type(),
 	}
 
 	// send the request