Compare commits

..

39 Commits

Author SHA1 Message Date
colinlyguo
106f409a02 tweak and bump version 2023-08-05 16:00:33 +08:00
colinlyguo
b42a17580b go mod tidy and bump version 2023-08-05 03:41:04 +08:00
colinlyguo
47bf98eb30 address comments 2023-08-05 03:32:29 +08:00
colinlyguo
e3a49fc041 happy path integration test 2023-08-04 22:31:56 +08:00
colinlyguo
72355067b8 fix get task and proof 2023-08-04 20:56:52 +08:00
georgehao
fcf34edbfb feat(coordinator): add prover rest api (#716)
Co-authored-by: colinlyguo <colinlyguo@scroll.io>
2023-08-04 20:48:17 +08:00
georgehao
d11de12637 Merge branch 'develop' of github.com:scroll-tech/scroll into pull-model-prover-client 2023-08-04 17:01:31 +08:00
colin
3be3f9926a Merge branch 'develop' into pull-model-prover-client 2023-08-04 15:22:39 +08:00
colin
ba51b5794d Merge branch 'develop' into pull-model-prover-client 2023-08-04 14:37:08 +08:00
colinlyguo
f22efbff22 tweak integration-test 2023-08-03 20:02:26 +08:00
colin
8f92ca9010 Merge branch 'develop' into pull-model-prover-client 2023-08-03 19:33:16 +08:00
colinlyguo
2c69352948 fix 2023-08-03 19:17:22 +08:00
colinlyguo
0652852c16 address comments 2023-08-03 19:01:46 +08:00
colinlyguo
bbb4ca6d76 revert flag change 2023-08-03 17:56:26 +08:00
colinlyguo
1b542830af add mock_proof_submission config 2023-08-03 17:38:25 +08:00
colinlyguo
975e61afe4 tweak 2023-08-03 17:12:45 +08:00
colinlyguo
7bf538f287 refactor error code 2023-08-03 17:08:26 +08:00
colinlyguo
58fc8a3add remove signature 2023-08-03 16:30:27 +08:00
colinlyguo
bf9dabd9b7 tweak login request 2023-08-03 15:54:50 +08:00
colinlyguo
beea5754cb add retry mechanism for JWT expiration 2023-08-03 15:46:14 +08:00
colin
d0c82f78ad Merge branch 'develop' into pull-model-prover-client 2023-08-03 15:03:02 +08:00
colinlyguo
3913a7397c run goimport 2023-08-03 15:01:51 +08:00
colinlyguo
c63e25a9bf add login auth 2023-08-03 14:57:24 +08:00
colinlyguo
8960849677 address comments 2023-08-03 13:39:31 +08:00
colinlyguo
24d8658ddb address comments 2023-08-03 13:11:12 +08:00
colinlyguo
f88f883b6a simplify http body result marshal 2023-08-03 11:46:57 +08:00
colinlyguo
c8fd76dd02 trigger ci 2023-08-02 16:48:04 +08:00
colin
b82bbedaca Merge branch 'develop' into pull-model-prover-client 2023-08-02 16:36:14 +08:00
colinlyguo
1ea0dcd0ae bump version 2023-08-02 16:35:42 +08:00
colinlyguo
e87bdbabe2 tweak code 2023-08-02 16:32:41 +08:00
colinlyguo
4db1565ba3 comment out prover start process test (coordinator not support login yet) 2023-08-02 03:13:36 +08:00
colinlyguo
2425374362 fix 2023-08-02 03:05:06 +08:00
colinlyguo
89b2e3133a scrollfmt 2023-08-02 02:44:42 +08:00
colinlyguo
5b7a2bb40c bump version 2023-08-02 02:36:18 +08:00
colinlyguo
0466e2d201 add retry configs 2023-08-02 02:32:05 +08:00
colinlyguo
f86cbb2bcb fix 2023-08-02 02:14:30 +08:00
colin
712c2c5ad3 Merge branch 'develop' into pull-model-prover-client 2023-08-02 01:54:47 +08:00
colinlyguo
a8a5db9a18 fix 2023-08-01 22:27:45 +08:00
colinlyguo
61d8b48b73 a commit 2023-08-01 21:05:13 +08:00
15 changed files with 80 additions and 100 deletions

View File

@@ -60,8 +60,6 @@ type AuthMsg struct {
type Identity struct {
// ProverName the prover name
ProverName string `json:"prover_name"`
// ProverVersion the prover version
ProverVersion string `json:"prover_version"`
// Challenge unique challenge generated by manager
Challenge string `json:"challenge"`
}

View File

@@ -15,9 +15,8 @@ func TestAuthMessageSignAndVerify(t *testing.T) {
authMsg := &AuthMsg{
Identity: &Identity{
Challenge: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2OTEwMzgxNzUsIm9yaWdfaWF0IjoxNjkxMDM0NTc1fQ.HybBMsEJFhyZqtIa2iVcHUP7CEFttf708jmTMAImAWA",
ProverName: "test",
ProverVersion: "v1.0.0",
Challenge: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2OTEwMzgxNzUsIm9yaWdfaWF0IjoxNjkxMDM0NTc1fQ.HybBMsEJFhyZqtIa2iVcHUP7CEFttf708jmTMAImAWA",
ProverName: "test",
},
}
assert.NoError(t, authMsg.SignWithKey(privkey))
@@ -46,15 +45,14 @@ func TestGenerateToken(t *testing.T) {
func TestIdentityHash(t *testing.T) {
identity := &Identity{
Challenge: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2OTEwMzM0MTksIm9yaWdfaWF0IjoxNjkxMDI5ODE5fQ.EhkLZsj__rNPVC3ZDYBtvdh0nB8mmM_Hl82hObaIWOs",
ProverName: "test",
ProverVersion: "v1.0.0",
Challenge: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2OTEwMzM0MTksIm9yaWdfaWF0IjoxNjkxMDI5ODE5fQ.EhkLZsj__rNPVC3ZDYBtvdh0nB8mmM_Hl82hObaIWOs",
ProverName: "test",
}
hash, err := identity.Hash()
assert.NoError(t, err)
expectedHash := "83f5e0ad023e9c1de639ab07b9b4cb972ec9dbbd2524794c533a420a5b137721"
expectedHash := "7373bb57ab7c01307d86fec55e088e00c526d82d9d1303fa4a189ff6ea95fbe2"
assert.Equal(t, expectedHash, hex.EncodeToString(hash))
}

View File

@@ -81,11 +81,9 @@ func (c *CoordinatorApp) MockConfig(store bool) error {
}
// Reset prover manager config for manager test cases.
cfg.ProverManager = &coordinatorConfig.ProverManager{
ProversPerSession: 1,
Verifier: &coordinatorConfig.VerifierConfig{MockMode: true},
CollectionTimeSec: 60,
SessionAttempts: 10,
MaxVerifierWorkers: 4,
ProversPerSession: 1,
Verifier: &coordinatorConfig.VerifierConfig{MockMode: true},
CollectionTimeSec: 60,
}
cfg.DB.DSN = base.DBImg.Endpoint()
cfg.L2.ChainID = 111

View File

@@ -1,14 +1,14 @@
{
"prover_manager": {
"provers_per_session": 1,
"session_attempts": 5,
"session_attempts": 2,
"collection_time_sec": 180,
"verifier": {
"mock_mode": true,
"params_path": "",
"assets_path": ""
},
"max_verifier_workers": 4
"max_verifier_workers": 10
},
"db": {
"driver_name": "postgres",
@@ -22,6 +22,6 @@
"auth": {
"secret": "prover secret key",
"challenge_expire_duration_sec": 3600,
"login_expire_duration_sec": 10
"login_expire_duration_sec": 3600
}
}

View File

@@ -16,7 +16,7 @@ type ProverManager struct {
// Currently we only consider proving timeout as failure here.
SessionAttempts uint8 `json:"session_attempts"`
// Zk verifier config.
Verifier *VerifierConfig `json:"verifier"`
Verifier *VerifierConfig `json:"verifier,omitempty"`
// Proof collection time (in seconds).
CollectionTimeSec int `json:"collection_time_sec"`
// Max number of workers in verifier worker pool

View File

@@ -14,14 +14,14 @@ func TestConfig(t *testing.T) {
configTemplate := `{
"prover_manager": {
"provers_per_session": 1,
"session_attempts": 5,
"session_attempts": %d,
"collection_time_sec": 180,
"verifier": {
"mock_mode": true,
"params_path": "",
"agg_vk_path": ""
},
"max_verifier_workers": 4
"max_verifier_workers": %d
},
"db": {
"driver_name": "postgres",
@@ -46,7 +46,8 @@ func TestConfig(t *testing.T) {
assert.NoError(t, tmpFile.Close())
assert.NoError(t, os.Remove(tmpFile.Name()))
}()
_, err = tmpFile.WriteString(configTemplate)
config := fmt.Sprintf(configTemplate, defaultNumberOfSessionRetryAttempts, defaultNumberOfVerifierWorkers)
_, err = tmpFile.WriteString(config)
assert.NoError(t, err)
cfg, err := NewConfig(tmpFile.Name())
@@ -87,4 +88,36 @@ func TestConfig(t *testing.T) {
_, err = NewConfig(tmpFile.Name())
assert.Error(t, err)
})
t.Run("Default MaxVerifierWorkers", func(t *testing.T) {
tmpFile, err := os.CreateTemp("", "example")
assert.NoError(t, err)
defer func() {
assert.NoError(t, tmpFile.Close())
assert.NoError(t, os.Remove(tmpFile.Name()))
}()
config := fmt.Sprintf(configTemplate, defaultNumberOfSessionRetryAttempts, 0)
_, err = tmpFile.WriteString(config)
assert.NoError(t, err)
cfg, err := NewConfig(tmpFile.Name())
assert.NoError(t, err)
assert.Equal(t, defaultNumberOfVerifierWorkers, cfg.ProverManager.MaxVerifierWorkers)
})
t.Run("Default SessionAttempts", func(t *testing.T) {
tmpFile, err := os.CreateTemp("", "example")
assert.NoError(t, err)
defer func() {
assert.NoError(t, tmpFile.Close())
assert.NoError(t, os.Remove(tmpFile.Name()))
}()
config := fmt.Sprintf(configTemplate, 0, defaultNumberOfVerifierWorkers)
_, err = tmpFile.WriteString(config)
assert.NoError(t, err)
cfg, err := NewConfig(tmpFile.Name())
assert.NoError(t, err)
assert.Equal(t, uint8(defaultNumberOfSessionRetryAttempts), cfg.ProverManager.SessionAttempts)
})
}

View File

@@ -31,16 +31,8 @@ func (a *AuthController) Login(c *gin.Context) (interface{}, error) {
if err := c.ShouldBind(&login); err != nil {
return "", fmt.Errorf("missing the public_key, err:%w", err)
}
// check login parameter's token is equal to bearer token, the Authorization must be existed
// if not exist, the jwt token will intercept it
brearToken := c.GetHeader("Authorization")
if brearToken != "Bearer "+login.Message.Challenge {
return "", fmt.Errorf("check challenge failure for the not equal challenge string")
}
// check the challenge is used, if used, return failure
if err := a.loginLogic.InsertChallengeString(c, login.Message.Challenge); err != nil {
if err := a.loginLogic.InsertChallengeString(c, login.Signature); err != nil {
return "", fmt.Errorf("login insert challenge string failure:%w", err)
}
return login, nil
@@ -56,9 +48,8 @@ func (a *AuthController) PayloadFunc(data interface{}) jwt.MapClaims {
// recover the public key
authMsg := message.AuthMsg{
Identity: &message.Identity{
Challenge: v.Message.Challenge,
ProverName: v.Message.ProverName,
ProverVersion: v.Message.ProverVersion,
Challenge: v.Message.Challenge,
ProverName: v.Message.ProverName,
},
Signature: v.Signature,
}
@@ -69,9 +60,8 @@ func (a *AuthController) PayloadFunc(data interface{}) jwt.MapClaims {
}
return jwt.MapClaims{
types.PublicKey: publicKey,
types.ProverName: v.Message.ProverName,
types.ProverVersion: v.Message.ProverVersion,
types.PublicKey: publicKey,
types.ProverName: v.Message.ProverName,
}
}
@@ -85,9 +75,5 @@ func (a *AuthController) IdentityHandler(c *gin.Context) interface{} {
if publicKey, ok := claims[types.PublicKey]; ok {
c.Set(types.PublicKey, publicKey)
}
if proverVersion, ok := claims[types.ProverVersion]; ok {
c.Set(types.ProverVersion, proverVersion)
}
return nil
}

View File

@@ -20,6 +20,6 @@ func NewLoginLogic(db *gorm.DB) *LoginLogic {
}
// InsertChallengeString insert and check the challenge string is existed
func (l *LoginLogic) InsertChallengeString(ctx *gin.Context, challenge string) error {
return l.challengeOrm.InsertChallenge(ctx, challenge)
func (l *LoginLogic) InsertChallengeString(ctx *gin.Context, signature string) error {
return l.challengeOrm.InsertChallenge(ctx, signature)
}

View File

@@ -7,15 +7,12 @@ const (
PublicKey = "public_key"
// ProverName the prover name key for context
ProverName = "prover_name"
// ProverVersion the prover version for context
ProverVersion = "prover_version"
)
// Message the login message struct
type Message struct {
Challenge string `form:"challenge" json:"challenge" binding:"required"`
ProverVersion string `form:"prover_version" json:"prover_version" binding:"required"`
ProverName string `form:"prover_name" json:"prover_name" binding:"required"`
Challenge string `form:"challenge" json:"challenge" binding:"required"`
ProverName string `form:"prover_name" json:"prover_name" binding:"required"`
}
// LoginParameter for /login api

View File

@@ -2,8 +2,9 @@ package types
// GetTaskParameter for ProverTasks request parameter
type GetTaskParameter struct {
ProverHeight int `form:"prover_height" json:"prover_height" binding:"required"`
TaskType int `form:"task_type" json:"task_type"`
ProverVersion string `form:"prover_version" json:"prover_version" binding:"required"`
ProverHeight int `form:"prover_height" json:"prover_height" binding:"required"`
TaskType int `form:"task_type" json:"task_type"`
}
// GetTaskSchema the schema data return to prover for get prover task

View File

@@ -76,15 +76,14 @@ func (r *mockProver) challenge(t *testing.T) string {
func (r *mockProver) login(t *testing.T, challengeString string) string {
authMsg := message.AuthMsg{
Identity: &message.Identity{
Challenge: challengeString,
ProverName: "test",
ProverVersion: "v1.0.0",
Challenge: challengeString,
ProverName: "test",
},
}
assert.NoError(t, authMsg.SignWithKey(r.privKey))
body := fmt.Sprintf("{\"message\":{\"challenge\":\"%s\",\"prover_name\":\"%s\", \"prover_version\":\"%s\"},\"signature\":\"%s\"}",
authMsg.Identity.Challenge, authMsg.Identity.ProverName, authMsg.Identity.ProverVersion, authMsg.Signature)
body := fmt.Sprintf("{\"message\":{\"challenge\":\"%s\",\"prover_name\":\"%s\"},\"signature\":\"%s\"}",
authMsg.Identity.Challenge, authMsg.Identity.ProverName, authMsg.Signature)
var result types.Response
client := resty.New()
@@ -138,7 +137,7 @@ func (r *mockProver) getProverTask(t *testing.T, proofType message.ProofType) *t
resp, err := client.R().
SetHeader("Content-Type", "application/json").
SetHeader("Authorization", fmt.Sprintf("Bearer %s", token)).
SetBody(map[string]interface{}{"prover_height": 100, "task_type": int(proofType)}).
SetBody(map[string]interface{}{"prover_version": "v1.0.0", "prover_height": 100, "task_type": int(proofType)}).
SetResult(&result).
Post("http://" + r.coordinatorURL + "/coordinator/v1/get_task")
assert.NoError(t, err)

View File

@@ -14,7 +14,6 @@ import (
"scroll-tech/common/types"
"scroll-tech/common/types/message"
"scroll-tech/common/version"
)
// CoordinatorClient is a client used for interacting with the Coordinator service.
@@ -66,9 +65,8 @@ func (c *CoordinatorClient) Login(ctx context.Context) error {
// Prepare and sign the login request
authMsg := &message.AuthMsg{
Identity: &message.Identity{
ProverVersion: version.Version,
ProverName: c.proverName,
Challenge: challengeResult.Data.Token,
ProverName: c.proverName,
Challenge: challengeResult.Data.Token,
},
}
@@ -80,13 +78,11 @@ func (c *CoordinatorClient) Login(ctx context.Context) error {
// Login to coordinator
loginReq := &LoginRequest{
Message: struct {
Challenge string `json:"challenge"`
ProverName string `json:"prover_name"`
ProverVersion string `json:"prover_version"`
Challenge string `json:"challenge"`
ProverName string `json:"prover_name"`
}{
Challenge: authMsg.Identity.Challenge,
ProverName: authMsg.Identity.ProverName,
ProverVersion: authMsg.Identity.ProverVersion,
Challenge: authMsg.Identity.Challenge,
ProverName: authMsg.Identity.ProverName,
},
Signature: authMsg.Signature,
}

View File

@@ -17,9 +17,8 @@ type ChallengeResponse struct {
// LoginRequest defines the request structure for login API
type LoginRequest struct {
Message struct {
Challenge string `json:"challenge"`
ProverName string `json:"prover_name"`
ProverVersion string `json:"prover_version"`
Challenge string `json:"challenge"`
ProverName string `json:"prover_name"`
} `json:"message"`
Signature string `json:"signature"`
}
@@ -36,8 +35,9 @@ type LoginResponse struct {
// GetTaskRequest defines the request structure for GetTask API
type GetTaskRequest struct {
ProverHeight uint64 `json:"prover_height"`
TaskType message.ProofType `json:"task_type"`
ProverVersion string `json:"prover_version"`
ProverHeight uint64 `json:"prover_height"`
TaskType message.ProofType `json:"task_type"`
}
// GetTaskResponse defines the response structure for GetTask API

View File

@@ -1,6 +1,5 @@
//go:build ffi
// go test -v -race -gcflags="-l" -ldflags="-s=false" -tags ffi ./...
package core_test
import (
@@ -13,7 +12,6 @@ import (
"github.com/scroll-tech/go-ethereum/core/types"
"github.com/stretchr/testify/assert"
scrollTypes "scroll-tech/common/types"
"scroll-tech/common/types/message"
"scroll-tech/prover/config"
@@ -49,32 +47,6 @@ func TestFFI(t *testing.T) {
as.NoError(err)
t.Log("Converted to chunk infos")
wrappedBlock1 := &scrollTypes.WrappedBlock{
Header: chunkTrace1[0].Header,
Transactions: chunkTrace1[0].Transactions,
WithdrawRoot: chunkTrace1[0].WithdrawTrieRoot,
}
chunk1 := &scrollTypes.Chunk{Blocks: []*scrollTypes.WrappedBlock{wrappedBlock1}}
chunkHash1, err := chunk1.Hash(0)
as.NoError(err)
as.Equal(chunkInfo1.PostStateRoot, wrappedBlock1.Header.Root)
as.Equal(chunkInfo1.WithdrawRoot, wrappedBlock1.WithdrawRoot)
as.Equal(chunkInfo1.DataHash, chunkHash1)
t.Log("Successful to check chunk info 1")
wrappedBlock2 := &scrollTypes.WrappedBlock{
Header: chunkTrace2[0].Header,
Transactions: chunkTrace2[0].Transactions,
WithdrawRoot: chunkTrace2[0].WithdrawTrieRoot,
}
chunk2 := &scrollTypes.Chunk{Blocks: []*scrollTypes.WrappedBlock{wrappedBlock2}}
chunkHash2, err := chunk2.Hash(chunk1.NumL1Messages(0))
as.NoError(err)
as.Equal(chunkInfo2.PostStateRoot, wrappedBlock2.Header.Root)
as.Equal(chunkInfo2.WithdrawRoot, wrappedBlock2.WithdrawRoot)
as.Equal(chunkInfo2.DataHash, chunkHash2)
t.Log("Successful to check chunk info 2")
chunkProof1, err := chunkProverCore.ProveChunk("chunk_proof1", chunkTrace1)
as.NoError(err)
t.Log("Generated and dumped chunk proof 1")

View File

@@ -24,6 +24,7 @@ import (
"scroll-tech/common/types/message"
"scroll-tech/common/utils"
"scroll-tech/common/version"
)
var (
@@ -189,8 +190,9 @@ func (r *Prover) fetchTaskFromCoordinator() (*store.ProvingTask, error) {
// prepare the request
req := &client.GetTaskRequest{
ProverHeight: latestBlockNumber,
TaskType: r.Type(),
ProverVersion: version.Version,
ProverHeight: latestBlockNumber,
TaskType: r.Type(),
}
// send the request