Compare commits

..

39 Commits

Author SHA1 Message Date
colinlyguo
106f409a02 tweak and bump version 2023-08-05 16:00:33 +08:00
colinlyguo
b42a17580b go mod tidy and bump version 2023-08-05 03:41:04 +08:00
colinlyguo
47bf98eb30 address comments 2023-08-05 03:32:29 +08:00
colinlyguo
e3a49fc041 happy path integration test 2023-08-04 22:31:56 +08:00
colinlyguo
72355067b8 fix get task and proof 2023-08-04 20:56:52 +08:00
georgehao
fcf34edbfb feat(coordinator): add prover rest api (#716)
Co-authored-by: colinlyguo <colinlyguo@scroll.io>
2023-08-04 20:48:17 +08:00
georgehao
d11de12637 Merge branch 'develop' of github.com:scroll-tech/scroll into pull-model-prover-client 2023-08-04 17:01:31 +08:00
colin
3be3f9926a Merge branch 'develop' into pull-model-prover-client 2023-08-04 15:22:39 +08:00
colin
ba51b5794d Merge branch 'develop' into pull-model-prover-client 2023-08-04 14:37:08 +08:00
colinlyguo
f22efbff22 tweak integration-test 2023-08-03 20:02:26 +08:00
colin
8f92ca9010 Merge branch 'develop' into pull-model-prover-client 2023-08-03 19:33:16 +08:00
colinlyguo
2c69352948 fix 2023-08-03 19:17:22 +08:00
colinlyguo
0652852c16 address comments 2023-08-03 19:01:46 +08:00
colinlyguo
bbb4ca6d76 revert flag change 2023-08-03 17:56:26 +08:00
colinlyguo
1b542830af add mock_proof_submission config 2023-08-03 17:38:25 +08:00
colinlyguo
975e61afe4 tweak 2023-08-03 17:12:45 +08:00
colinlyguo
7bf538f287 refactor error code 2023-08-03 17:08:26 +08:00
colinlyguo
58fc8a3add remove signature 2023-08-03 16:30:27 +08:00
colinlyguo
bf9dabd9b7 tweak login request 2023-08-03 15:54:50 +08:00
colinlyguo
beea5754cb add retry mechanism for JWT expiration 2023-08-03 15:46:14 +08:00
colin
d0c82f78ad Merge branch 'develop' into pull-model-prover-client 2023-08-03 15:03:02 +08:00
colinlyguo
3913a7397c run goimport 2023-08-03 15:01:51 +08:00
colinlyguo
c63e25a9bf add login auth 2023-08-03 14:57:24 +08:00
colinlyguo
8960849677 address comments 2023-08-03 13:39:31 +08:00
colinlyguo
24d8658ddb address comments 2023-08-03 13:11:12 +08:00
colinlyguo
f88f883b6a simplify http body result marshal 2023-08-03 11:46:57 +08:00
colinlyguo
c8fd76dd02 trigger ci 2023-08-02 16:48:04 +08:00
colin
b82bbedaca Merge branch 'develop' into pull-model-prover-client 2023-08-02 16:36:14 +08:00
colinlyguo
1ea0dcd0ae bump version 2023-08-02 16:35:42 +08:00
colinlyguo
e87bdbabe2 tweak code 2023-08-02 16:32:41 +08:00
colinlyguo
4db1565ba3 comment out prover start process test (coordinator not support login yet) 2023-08-02 03:13:36 +08:00
colinlyguo
2425374362 fix 2023-08-02 03:05:06 +08:00
colinlyguo
89b2e3133a scrollfmt 2023-08-02 02:44:42 +08:00
colinlyguo
5b7a2bb40c bump version 2023-08-02 02:36:18 +08:00
colinlyguo
0466e2d201 add retry configs 2023-08-02 02:32:05 +08:00
colinlyguo
f86cbb2bcb fix 2023-08-02 02:14:30 +08:00
colin
712c2c5ad3 Merge branch 'develop' into pull-model-prover-client 2023-08-02 01:54:47 +08:00
colinlyguo
a8a5db9a18 fix 2023-08-01 22:27:45 +08:00
colinlyguo
61d8b48b73 a commit 2023-08-01 21:05:13 +08:00
25 changed files with 192 additions and 191 deletions

View File

@@ -15,7 +15,6 @@ func Route(router *gin.Engine, conf *config.Config) {
router.Use(cors.New(cors.Config{
AllowOrigins: []string{"*"},
AllowMethods: []string{"GET", "POST", "PUT", "DELETE"},
AllowHeaders: []string{"Origin", "Content-Type", "Authorization"},
AllowCredentials: true,
MaxAge: 12 * time.Hour,
}))

View File

@@ -32,7 +32,7 @@ dependencies = [
[[package]]
name = "aggregator"
version = "0.1.0"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.5.3#2c8c749b3e4a61e89028289f4ff93157c5671d7b"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#2855c13b5d3e6ec4056f823f56a33bf25d0080bb"
dependencies = [
"ark-std",
"env_logger 0.10.0",
@@ -432,7 +432,7 @@ checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1"
[[package]]
name = "bus-mapping"
version = "0.1.0"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.5.3#2c8c749b3e4a61e89028289f4ff93157c5671d7b"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#2855c13b5d3e6ec4056f823f56a33bf25d0080bb"
dependencies = [
"eth-types",
"ethers-core",
@@ -1045,7 +1045,7 @@ dependencies = [
[[package]]
name = "eth-types"
version = "0.1.0"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.5.3#2c8c749b3e4a61e89028289f4ff93157c5671d7b"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#2855c13b5d3e6ec4056f823f56a33bf25d0080bb"
dependencies = [
"ethers-core",
"ethers-signers",
@@ -1223,7 +1223,7 @@ dependencies = [
[[package]]
name = "external-tracer"
version = "0.1.0"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.5.3#2c8c749b3e4a61e89028289f4ff93157c5671d7b"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#2855c13b5d3e6ec4056f823f56a33bf25d0080bb"
dependencies = [
"eth-types",
"geth-utils",
@@ -1436,7 +1436,7 @@ dependencies = [
[[package]]
name = "gadgets"
version = "0.1.0"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.5.3#2c8c749b3e4a61e89028289f4ff93157c5671d7b"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#2855c13b5d3e6ec4056f823f56a33bf25d0080bb"
dependencies = [
"digest 0.7.6",
"eth-types",
@@ -1476,7 +1476,7 @@ dependencies = [
[[package]]
name = "geth-utils"
version = "0.1.0"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.5.3#2c8c749b3e4a61e89028289f4ff93157c5671d7b"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#2855c13b5d3e6ec4056f823f56a33bf25d0080bb"
dependencies = [
"env_logger 0.9.3",
"gobuild 0.1.0-alpha.2 (git+https://github.com/scroll-tech/gobuild.git)",
@@ -2074,7 +2074,7 @@ dependencies = [
[[package]]
name = "keccak256"
version = "0.1.0"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.5.3#2c8c749b3e4a61e89028289f4ff93157c5671d7b"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#2855c13b5d3e6ec4056f823f56a33bf25d0080bb"
dependencies = [
"env_logger 0.9.3",
"eth-types",
@@ -2261,7 +2261,7 @@ dependencies = [
[[package]]
name = "mock"
version = "0.1.0"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.5.3#2c8c749b3e4a61e89028289f4ff93157c5671d7b"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#2855c13b5d3e6ec4056f823f56a33bf25d0080bb"
dependencies = [
"eth-types",
"ethers-core",
@@ -2276,7 +2276,7 @@ dependencies = [
[[package]]
name = "mpt-zktrie"
version = "0.1.0"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.5.3#2c8c749b3e4a61e89028289f4ff93157c5671d7b"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#2855c13b5d3e6ec4056f823f56a33bf25d0080bb"
dependencies = [
"bus-mapping",
"eth-types",
@@ -2752,7 +2752,7 @@ dependencies = [
[[package]]
name = "prover"
version = "0.4.0"
source = "git+https://github.com/scroll-tech/scroll-prover?tag=v0.5.3#337089ac40bac756d88b9ae30a3be1f82538b216"
source = "git+https://github.com/scroll-tech/scroll-prover?tag=v0.5.1#15aac6e1484a42f723098fbc9d8783f374e7e90a"
dependencies = [
"aggregator",
"anyhow",
@@ -3621,7 +3621,7 @@ checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9"
[[package]]
name = "snark-verifier"
version = "0.1.0"
source = "git+https://github.com/scroll-tech/snark-verifier?branch=develop#12c306ec57849921e690221b10b8a08189868d4a"
source = "git+https://github.com/scroll-tech/snark-verifier?branch=develop#f8bdcbee60348e5c996c04f19ff30522e6b276b0"
dependencies = [
"bytes",
"ethereum-types 0.14.1",
@@ -3645,7 +3645,7 @@ dependencies = [
[[package]]
name = "snark-verifier-sdk"
version = "0.0.1"
source = "git+https://github.com/scroll-tech/snark-verifier?branch=develop#12c306ec57849921e690221b10b8a08189868d4a"
source = "git+https://github.com/scroll-tech/snark-verifier?branch=develop#f8bdcbee60348e5c996c04f19ff30522e6b276b0"
dependencies = [
"bincode",
"env_logger 0.10.0",
@@ -4037,7 +4037,7 @@ checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
[[package]]
name = "types"
version = "0.4.0"
source = "git+https://github.com/scroll-tech/scroll-prover?tag=v0.5.3#337089ac40bac756d88b9ae30a3be1f82538b216"
source = "git+https://github.com/scroll-tech/scroll-prover?tag=v0.5.1#15aac6e1484a42f723098fbc9d8783f374e7e90a"
dependencies = [
"base64 0.13.1",
"blake2",
@@ -4482,7 +4482,7 @@ checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9"
[[package]]
name = "zkevm-circuits"
version = "0.1.0"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.5.3#2c8c749b3e4a61e89028289f4ff93157c5671d7b"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#2855c13b5d3e6ec4056f823f56a33bf25d0080bb"
dependencies = [
"array-init",
"bus-mapping",

View File

@@ -18,8 +18,8 @@ maingate = { git = "https://github.com/scroll-tech/halo2wrong", branch = "halo2-
halo2curves = { git = "https://github.com/scroll-tech/halo2curves.git", branch = "0.3.1-derive-serde" }
[dependencies]
prover = { git = "https://github.com/scroll-tech/scroll-prover", tag = "v0.5.3" }
types = { git = "https://github.com/scroll-tech/scroll-prover", tag = "v0.5.3" }
prover = { git = "https://github.com/scroll-tech/scroll-prover", tag = "v0.5.1" }
types = { git = "https://github.com/scroll-tech/scroll-prover", tag = "v0.5.1" }
halo2_proofs = { git = "https://github.com/scroll-tech/halo2.git", branch = "develop" }
log = "0.4"
@@ -33,6 +33,8 @@ once_cell = "1.8.0"
[profile.test]
opt-level = 3
# debug-assertions = true
[profile.release]
opt-level = 3
# debug-assertions = true

View File

@@ -33,7 +33,7 @@ func (r ProofType) String() string {
case ProofTypeBatch:
return "proof type batch"
default:
return fmt.Sprintf("illegal proof type: %d", r)
return "illegal proof type"
}
}
@@ -60,8 +60,6 @@ type AuthMsg struct {
type Identity struct {
// ProverName the prover name
ProverName string `json:"prover_name"`
// ProverVersion the prover version
ProverVersion string `json:"prover_version"`
// Challenge unique challenge generated by manager
Challenge string `json:"challenge"`
}

View File

@@ -15,9 +15,8 @@ func TestAuthMessageSignAndVerify(t *testing.T) {
authMsg := &AuthMsg{
Identity: &Identity{
Challenge: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2OTEwMzgxNzUsIm9yaWdfaWF0IjoxNjkxMDM0NTc1fQ.HybBMsEJFhyZqtIa2iVcHUP7CEFttf708jmTMAImAWA",
ProverName: "test",
ProverVersion: "v1.0.0",
Challenge: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2OTEwMzgxNzUsIm9yaWdfaWF0IjoxNjkxMDM0NTc1fQ.HybBMsEJFhyZqtIa2iVcHUP7CEFttf708jmTMAImAWA",
ProverName: "test",
},
}
assert.NoError(t, authMsg.SignWithKey(privkey))
@@ -46,15 +45,14 @@ func TestGenerateToken(t *testing.T) {
func TestIdentityHash(t *testing.T) {
identity := &Identity{
Challenge: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2OTEwMzM0MTksIm9yaWdfaWF0IjoxNjkxMDI5ODE5fQ.EhkLZsj__rNPVC3ZDYBtvdh0nB8mmM_Hl82hObaIWOs",
ProverName: "test",
ProverVersion: "v1.0.0",
Challenge: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2OTEwMzM0MTksIm9yaWdfaWF0IjoxNjkxMDI5ODE5fQ.EhkLZsj__rNPVC3ZDYBtvdh0nB8mmM_Hl82hObaIWOs",
ProverName: "test",
}
hash, err := identity.Hash()
assert.NoError(t, err)
expectedHash := "83f5e0ad023e9c1de639ab07b9b4cb972ec9dbbd2524794c533a420a5b137721"
expectedHash := "7373bb57ab7c01307d86fec55e088e00c526d82d9d1303fa4a189ff6ea95fbe2"
assert.Equal(t, expectedHash, hex.EncodeToString(hash))
}
@@ -118,7 +116,7 @@ func TestProveTypeString(t *testing.T) {
assert.Equal(t, "proof type batch", proofTypeBatch.String())
illegalProof := ProofType(3)
assert.Equal(t, "illegal proof type: 3", illegalProof.String())
assert.Equal(t, "illegal proof type", illegalProof.String())
}
func TestProofMsgPublicKey(t *testing.T) {

View File

@@ -5,7 +5,7 @@ import (
"runtime/debug"
)
var tag = "v4.1.5"
var tag = "v4.1.1"
var commit = func() string {
if info, ok := debug.ReadBuildInfo(); ok {

View File

@@ -81,11 +81,9 @@ func (c *CoordinatorApp) MockConfig(store bool) error {
}
// Reset prover manager config for manager test cases.
cfg.ProverManager = &coordinatorConfig.ProverManager{
ProversPerSession: 1,
Verifier: &coordinatorConfig.VerifierConfig{MockMode: true},
CollectionTimeSec: 60,
SessionAttempts: 10,
MaxVerifierWorkers: 4,
ProversPerSession: 1,
Verifier: &coordinatorConfig.VerifierConfig{MockMode: true},
CollectionTimeSec: 60,
}
cfg.DB.DSN = base.DBImg.Endpoint()
cfg.L2.ChainID = 111

View File

@@ -1,14 +1,14 @@
{
"prover_manager": {
"provers_per_session": 1,
"session_attempts": 5,
"session_attempts": 2,
"collection_time_sec": 180,
"verifier": {
"mock_mode": true,
"params_path": "",
"assets_path": ""
},
"max_verifier_workers": 4
"max_verifier_workers": 10
},
"db": {
"driver_name": "postgres",
@@ -22,6 +22,6 @@
"auth": {
"secret": "prover secret key",
"challenge_expire_duration_sec": 3600,
"login_expire_duration_sec": 10
"login_expire_duration_sec": 3600
}
}

View File

@@ -16,7 +16,7 @@ type ProverManager struct {
// Currently we only consider proving timeout as failure here.
SessionAttempts uint8 `json:"session_attempts"`
// Zk verifier config.
Verifier *VerifierConfig `json:"verifier"`
Verifier *VerifierConfig `json:"verifier,omitempty"`
// Proof collection time (in seconds).
CollectionTimeSec int `json:"collection_time_sec"`
// Max number of workers in verifier worker pool

View File

@@ -14,14 +14,14 @@ func TestConfig(t *testing.T) {
configTemplate := `{
"prover_manager": {
"provers_per_session": 1,
"session_attempts": 5,
"session_attempts": %d,
"collection_time_sec": 180,
"verifier": {
"mock_mode": true,
"params_path": "",
"agg_vk_path": ""
},
"max_verifier_workers": 4
"max_verifier_workers": %d
},
"db": {
"driver_name": "postgres",
@@ -46,7 +46,8 @@ func TestConfig(t *testing.T) {
assert.NoError(t, tmpFile.Close())
assert.NoError(t, os.Remove(tmpFile.Name()))
}()
_, err = tmpFile.WriteString(configTemplate)
config := fmt.Sprintf(configTemplate, defaultNumberOfSessionRetryAttempts, defaultNumberOfVerifierWorkers)
_, err = tmpFile.WriteString(config)
assert.NoError(t, err)
cfg, err := NewConfig(tmpFile.Name())
@@ -87,4 +88,36 @@ func TestConfig(t *testing.T) {
_, err = NewConfig(tmpFile.Name())
assert.Error(t, err)
})
t.Run("Default MaxVerifierWorkers", func(t *testing.T) {
tmpFile, err := os.CreateTemp("", "example")
assert.NoError(t, err)
defer func() {
assert.NoError(t, tmpFile.Close())
assert.NoError(t, os.Remove(tmpFile.Name()))
}()
config := fmt.Sprintf(configTemplate, defaultNumberOfSessionRetryAttempts, 0)
_, err = tmpFile.WriteString(config)
assert.NoError(t, err)
cfg, err := NewConfig(tmpFile.Name())
assert.NoError(t, err)
assert.Equal(t, defaultNumberOfVerifierWorkers, cfg.ProverManager.MaxVerifierWorkers)
})
t.Run("Default SessionAttempts", func(t *testing.T) {
tmpFile, err := os.CreateTemp("", "example")
assert.NoError(t, err)
defer func() {
assert.NoError(t, tmpFile.Close())
assert.NoError(t, os.Remove(tmpFile.Name()))
}()
config := fmt.Sprintf(configTemplate, 0, defaultNumberOfVerifierWorkers)
_, err = tmpFile.WriteString(config)
assert.NoError(t, err)
cfg, err := NewConfig(tmpFile.Name())
assert.NoError(t, err)
assert.Equal(t, uint8(defaultNumberOfSessionRetryAttempts), cfg.ProverManager.SessionAttempts)
})
}

View File

@@ -31,16 +31,8 @@ func (a *AuthController) Login(c *gin.Context) (interface{}, error) {
if err := c.ShouldBind(&login); err != nil {
return "", fmt.Errorf("missing the public_key, err:%w", err)
}
// check login parameter's token is equal to bearer token, the Authorization must be existed
// if not exist, the jwt token will intercept it
brearToken := c.GetHeader("Authorization")
if brearToken != "Bearer "+login.Message.Challenge {
return "", fmt.Errorf("check challenge failure for the not equal challenge string")
}
// check the challenge is used, if used, return failure
if err := a.loginLogic.InsertChallengeString(c, login.Message.Challenge); err != nil {
if err := a.loginLogic.InsertChallengeString(c, login.Signature); err != nil {
return "", fmt.Errorf("login insert challenge string failure:%w", err)
}
return login, nil
@@ -56,9 +48,8 @@ func (a *AuthController) PayloadFunc(data interface{}) jwt.MapClaims {
// recover the public key
authMsg := message.AuthMsg{
Identity: &message.Identity{
Challenge: v.Message.Challenge,
ProverName: v.Message.ProverName,
ProverVersion: v.Message.ProverVersion,
Challenge: v.Message.Challenge,
ProverName: v.Message.ProverName,
},
Signature: v.Signature,
}
@@ -69,9 +60,8 @@ func (a *AuthController) PayloadFunc(data interface{}) jwt.MapClaims {
}
return jwt.MapClaims{
types.PublicKey: publicKey,
types.ProverName: v.Message.ProverName,
types.ProverVersion: v.Message.ProverVersion,
types.PublicKey: publicKey,
types.ProverName: v.Message.ProverName,
}
}
@@ -85,9 +75,5 @@ func (a *AuthController) IdentityHandler(c *gin.Context) interface{} {
if publicKey, ok := claims[types.PublicKey]; ok {
c.Set(types.PublicKey, publicKey)
}
if proverVersion, ok := claims[types.ProverVersion]; ok {
c.Set(types.ProverVersion, proverVersion)
}
return nil
}

View File

@@ -20,6 +20,6 @@ func NewLoginLogic(db *gorm.DB) *LoginLogic {
}
// InsertChallengeString insert and check the challenge string is existed
func (l *LoginLogic) InsertChallengeString(ctx *gin.Context, challenge string) error {
return l.challengeOrm.InsertChallenge(ctx, challenge)
func (l *LoginLogic) InsertChallengeString(ctx *gin.Context, signature string) error {
return l.challengeOrm.InsertChallenge(ctx, signature)
}

View File

@@ -87,7 +87,7 @@ func (m *ProofReceiverLogic) HandleZkProof(ctx *gin.Context, proofMsg *message.P
if errors.Is(err, ErrValidatorFailureProofMsgStatusNotOk) {
m.proofFailure(ctx, proofMsg.ID, pk, proofMsg.Type)
}
return err
return nil
}
proofTime := time.Since(proverTask.CreatedAt)
@@ -134,21 +134,23 @@ func (m *ProofReceiverLogic) HandleZkProof(ctx *gin.Context, proofMsg *message.P
}
if verifyErr != nil || !success {
if verifyErr != nil {
// TODO: this is only a temp workaround for testnet, we should return err in real cases
log.Error("failed to verify zk proof", "proof id", proofMsg.ID, "prover pk", pk, "prove type",
proofMsg.Type, "proof time", proofTimeSec, "error", verifyErr)
}
m.proofFailure(ctx, proofMsg.ID, pk, proofMsg.Type)
// TODO: Prover needs to be slashed if proof is invalid.
coordinatorProofsVerifiedFailedTimeTimer.Update(proofTime)
log.Info("proof verified by coordinator failed", "proof id", proofMsg.ID, "prover name", proverTask.ProverName,
"prover pk", pk, "prove type", proofMsg.Type, "proof time", proofTimeSec, "error", verifyErr)
if verifyErr == nil {
verifyErr = fmt.Errorf("verification succeeded and it's an invalid proof")
}
return verifyErr
return nil
}
if err := m.closeProofTask(ctx, proofMsg.ID, pk, proofMsg); err != nil {
m.proofRecover(ctx, proofMsg.ID, pk, proofMsg.Type)
return err
}
coordinatorProofsVerifiedSuccessTimeTimer.Update(proofTime)

View File

@@ -17,8 +17,6 @@ func Route(router *gin.Engine, cfg *config.Config) {
func v1(router *gin.RouterGroup, conf *config.Config) {
r := router.Group("/v1")
r.GET("/health", api.HealthCheck.HealthCheck)
challengeMiddleware := middleware.ChallengeMiddleware(conf)
r.GET("/challenge", challengeMiddleware.LoginHandler)
@@ -28,6 +26,7 @@ func v1(router *gin.RouterGroup, conf *config.Config) {
// need jwt token api
r.Use(loginMiddleware.MiddlewareFunc())
{
r.GET("/healthz", api.HealthCheck.HealthCheck)
r.POST("/get_task", api.GetTask.GetTasks)
r.POST("/submit_proof", api.SubmitProof.SubmitProof)
}

View File

@@ -7,15 +7,12 @@ const (
PublicKey = "public_key"
// ProverName the prover name key for context
ProverName = "prover_name"
// ProverVersion the prover version for context
ProverVersion = "prover_version"
)
// Message the login message struct
type Message struct {
Challenge string `form:"challenge" json:"challenge" binding:"required"`
ProverVersion string `form:"prover_version" json:"prover_version" binding:"required"`
ProverName string `form:"prover_name" json:"prover_name" binding:"required"`
Challenge string `form:"challenge" json:"challenge" binding:"required"`
ProverName string `form:"prover_name" json:"prover_name" binding:"required"`
}
// LoginParameter for /login api

View File

@@ -2,8 +2,9 @@ package types
// GetTaskParameter for ProverTasks request parameter
type GetTaskParameter struct {
ProverHeight int `form:"prover_height" json:"prover_height" binding:"required"`
TaskType int `form:"task_type" json:"task_type"`
ProverVersion string `form:"prover_version" json:"prover_version" binding:"required"`
ProverHeight int `form:"prover_height" json:"prover_height" binding:"required"`
TaskType int `form:"task_type" json:"task_type"`
}
// GetTaskSchema the schema data return to prover for get prover task

View File

@@ -0,0 +1,21 @@
package types
import (
"scroll-tech/common/types"
"scroll-tech/common/types/message"
)
// ProversInfo is assigned provers info of a task (session)
type ProversInfo struct {
ID string `json:"id"`
ProverStatusList []*ProverStatus `json:"provers"`
StartTimestamp int64 `json:"start_timestamp"`
ProveType message.ProofType `json:"prove_type,omitempty"`
}
// ProverStatus is the prover name and prover prove status
type ProverStatus struct {
PublicKey string `json:"public_key"`
Name string `json:"name"`
Status types.ProverProveStatus `json:"status"`
}

View File

@@ -170,7 +170,9 @@ func testHandshake(t *testing.T) {
}()
chunkProver := newMockProver(t, "prover_chunk_test", coordinatorURL, message.ProofTypeChunk)
assert.True(t, chunkProver.healthCheckSuccess(t))
token := chunkProver.connectToCoordinator(t)
assert.NotEmpty(t, token)
assert.True(t, chunkProver.healthCheck(t, token, types.Success))
}
func testFailedHandshake(t *testing.T) {
@@ -179,17 +181,21 @@ func testFailedHandshake(t *testing.T) {
proofCollector, httpHandler := setupCoordinator(t, 1, coordinatorURL)
defer func() {
proofCollector.Stop()
assert.NoError(t, httpHandler.Shutdown(context.Background()))
}()
// Try to perform handshake without token
chunkProver := newMockProver(t, "prover_chunk_test", coordinatorURL, message.ProofTypeChunk)
assert.True(t, chunkProver.healthCheckSuccess(t))
token := chunkProver.connectToCoordinator(t)
assert.NotEmpty(t, token)
assert.True(t, chunkProver.healthCheck(t, token, types.Success))
// Try to perform handshake with server shutdown
assert.NoError(t, httpHandler.Shutdown(context.Background()))
time.Sleep(time.Second)
// Try to perform handshake with timeouted token
batchProver := newMockProver(t, "prover_batch_test", coordinatorURL, message.ProofTypeBatch)
assert.True(t, batchProver.healthCheckFailure(t))
token = chunkProver.connectToCoordinator(t)
assert.NotEmpty(t, token)
<-time.After(time.Duration(tokenTimeout+1) * time.Second)
assert.True(t, batchProver.healthCheck(t, token, types.ErrJWTTokenExpired))
}
func testValidProof(t *testing.T) {
@@ -229,7 +235,7 @@ func testValidProof(t *testing.T) {
}
proverTask := provers[i].getProverTask(t, proofType)
assert.NotNil(t, proverTask)
provers[i].submitProof(t, proverTask, proofStatus, types.Success)
provers[i].submitProof(t, proverTask, proofStatus)
}
// verify proof status
@@ -290,7 +296,7 @@ func testInvalidProof(t *testing.T) {
provers[i] = newMockProver(t, "prover_test"+strconv.Itoa(i), coordinatorURL, proofType)
proverTask := provers[i].getProverTask(t, proofType)
assert.NotNil(t, proverTask)
provers[i].submitProof(t, proverTask, verifiedFailed, types.ErrCoordinatorHandleZkProofFailure)
provers[i].submitProof(t, proverTask, verifiedFailed)
}
// verify proof status
@@ -351,7 +357,7 @@ func testProofGeneratedFailed(t *testing.T) {
provers[i] = newMockProver(t, "prover_test"+strconv.Itoa(i), coordinatorURL, proofType)
proverTask := provers[i].getProverTask(t, proofType)
assert.NotNil(t, proverTask)
provers[i].submitProof(t, proverTask, generatedFailed, types.ErrCoordinatorHandleZkProofFailure)
provers[i].submitProof(t, proverTask, generatedFailed)
}
// verify proof status
@@ -425,12 +431,12 @@ func testTimeoutProof(t *testing.T) {
chunkProver2 := newMockProver(t, "prover_test"+strconv.Itoa(2), coordinatorURL, message.ProofTypeChunk)
proverChunkTask2 := chunkProver2.getProverTask(t, message.ProofTypeChunk)
assert.NotNil(t, proverChunkTask2)
chunkProver2.submitProof(t, proverChunkTask2, verifiedSuccess, types.Success)
chunkProver2.submitProof(t, proverChunkTask2, verifiedSuccess)
batchProver2 := newMockProver(t, "prover_test"+strconv.Itoa(3), coordinatorURL, message.ProofTypeBatch)
proverBatchTask2 := batchProver2.getProverTask(t, message.ProofTypeBatch)
assert.NotNil(t, proverBatchTask2)
batchProver2.submitProof(t, proverBatchTask2, verifiedSuccess, types.Success)
batchProver2.submitProof(t, proverBatchTask2, verifiedSuccess)
// verify proof status, it should be verified now, because second prover sent valid proof
chunkProofStatus2, err := chunkOrm.GetProvingStatusByHash(context.Background(), dbChunk.Hash)

View File

@@ -76,15 +76,14 @@ func (r *mockProver) challenge(t *testing.T) string {
func (r *mockProver) login(t *testing.T, challengeString string) string {
authMsg := message.AuthMsg{
Identity: &message.Identity{
Challenge: challengeString,
ProverName: "test",
ProverVersion: "v1.0.0",
Challenge: challengeString,
ProverName: "test",
},
}
assert.NoError(t, authMsg.SignWithKey(r.privKey))
body := fmt.Sprintf("{\"message\":{\"challenge\":\"%s\",\"prover_name\":\"%s\", \"prover_version\":\"%s\"},\"signature\":\"%s\"}",
authMsg.Identity.Challenge, authMsg.Identity.ProverName, authMsg.Identity.ProverVersion, authMsg.Signature)
body := fmt.Sprintf("{\"message\":{\"challenge\":\"%s\",\"prover_name\":\"%s\"},\"signature\":\"%s\"}",
authMsg.Identity.Challenge, authMsg.Identity.ProverName, authMsg.Signature)
var result types.Response
client := resty.New()
@@ -108,27 +107,17 @@ func (r *mockProver) login(t *testing.T, challengeString string) string {
return loginData.Token
}
func (r *mockProver) healthCheckSuccess(t *testing.T) bool {
func (r *mockProver) healthCheck(t *testing.T, token string, errCode int) bool {
var result types.Response
client := resty.New()
resp, err := client.R().
SetHeader("Content-Type", "application/json").
SetHeader("Authorization", fmt.Sprintf("Bearer %s", token)).
SetResult(&result).
Get("http://" + r.coordinatorURL + "/coordinator/v1/health")
Get("http://" + r.coordinatorURL + "/coordinator/v1/healthz")
assert.NoError(t, err)
assert.Equal(t, http.StatusOK, resp.StatusCode())
assert.Equal(t, ctypes.Success, result.ErrCode)
return true
}
func (r *mockProver) healthCheckFailure(t *testing.T) bool {
var result types.Response
client := resty.New()
resp, err := client.R().
SetResult(&result).
Get("http://" + r.coordinatorURL + "/coordinator/v1/health")
assert.Error(t, err)
assert.Equal(t, 0, resp.StatusCode())
assert.Equal(t, 0, result.ErrCode)
assert.Equal(t, errCode, result.ErrCode)
return true
}
@@ -148,7 +137,7 @@ func (r *mockProver) getProverTask(t *testing.T, proofType message.ProofType) *t
resp, err := client.R().
SetHeader("Content-Type", "application/json").
SetHeader("Authorization", fmt.Sprintf("Bearer %s", token)).
SetBody(map[string]interface{}{"prover_height": 100, "task_type": int(proofType)}).
SetBody(map[string]interface{}{"prover_version": "v1.0.0", "prover_height": 100, "task_type": int(proofType)}).
SetResult(&result).
Post("http://" + r.coordinatorURL + "/coordinator/v1/get_task")
assert.NoError(t, err)
@@ -161,7 +150,7 @@ func (r *mockProver) getProverTask(t *testing.T, proofType message.ProofType) *t
return &result.Data
}
func (r *mockProver) submitProof(t *testing.T, proverTaskSchema *types.GetTaskSchema, proofStatus proofStatus, errCode int) {
func (r *mockProver) submitProof(t *testing.T, proverTaskSchema *types.GetTaskSchema, proofStatus proofStatus) {
proof := &message.ProofMsg{
ProofDetail: &message.ProofDetail{
ID: proverTaskSchema.TaskID,
@@ -216,5 +205,5 @@ func (r *mockProver) submitProof(t *testing.T, proverTaskSchema *types.GetTaskSc
Post("http://" + r.coordinatorURL + "/coordinator/v1/submit_proof")
assert.NoError(t, err)
assert.Equal(t, http.StatusOK, resp.StatusCode())
assert.Equal(t, errCode, result.ErrCode)
assert.Equal(t, ctypes.Success, result.ErrCode)
}

View File

@@ -14,7 +14,6 @@ import (
"scroll-tech/common/types"
"scroll-tech/common/types/message"
"scroll-tech/common/version"
)
// CoordinatorClient is a client used for interacting with the Coordinator service.
@@ -66,9 +65,8 @@ func (c *CoordinatorClient) Login(ctx context.Context) error {
// Prepare and sign the login request
authMsg := &message.AuthMsg{
Identity: &message.Identity{
ProverVersion: version.Version,
ProverName: c.proverName,
Challenge: challengeResult.Data.Token,
ProverName: c.proverName,
Challenge: challengeResult.Data.Token,
},
}
@@ -80,13 +78,11 @@ func (c *CoordinatorClient) Login(ctx context.Context) error {
// Login to coordinator
loginReq := &LoginRequest{
Message: struct {
Challenge string `json:"challenge"`
ProverName string `json:"prover_name"`
ProverVersion string `json:"prover_version"`
Challenge string `json:"challenge"`
ProverName string `json:"prover_name"`
}{
Challenge: authMsg.Identity.Challenge,
ProverName: authMsg.Identity.ProverName,
ProverVersion: authMsg.Identity.ProverVersion,
Challenge: authMsg.Identity.Challenge,
ProverName: authMsg.Identity.ProverName,
},
Signature: authMsg.Signature,
}

View File

@@ -17,9 +17,8 @@ type ChallengeResponse struct {
// LoginRequest defines the request structure for login API
type LoginRequest struct {
Message struct {
Challenge string `json:"challenge"`
ProverName string `json:"prover_name"`
ProverVersion string `json:"prover_version"`
Challenge string `json:"challenge"`
ProverName string `json:"prover_name"`
} `json:"message"`
Signature string `json:"signature"`
}
@@ -36,8 +35,9 @@ type LoginResponse struct {
// GetTaskRequest defines the request structure for GetTask API
type GetTaskRequest struct {
ProverHeight uint64 `json:"prover_height"`
TaskType message.ProofType `json:"task_type"`
ProverVersion string `json:"prover_version"`
ProverHeight uint64 `json:"prover_height"`
TaskType message.ProofType `json:"task_type"`
}
// GetTaskResponse defines the response structure for GetTask API

View File

@@ -1,20 +1,19 @@
{
"prover_name": "prover-1",
"prover_name": "my_prover",
"keystore_path": "keystore.json",
"keystore_password": "prover-pwd",
"db_path": "unique-db-path-for-prover-1",
"db_path": "bbolt_db",
"core": {
"params_path": "params",
"proof_type": 2
"params_path": "params"
},
"coordinator": {
"base_url": "http://localhost:8555",
"base_url": "https://coordinator/v1",
"retry_count": 10,
"retry_wait_time_sec": 10,
"connection_timeout_sec": 30
},
"l2geth": {
"endpoint": "http://localhost:9999",
"endpoint": "/var/lib/jenkins/workspace/SequencerPipeline/MyPrivateNetwork/geth.ipc",
"confirmations": "0x1"
}
}

View File

@@ -25,7 +25,7 @@ type Config struct {
// ProverCoreConfig load zk prover config.
type ProverCoreConfig struct {
ParamsPath string `json:"params_path"`
ProofType message.ProofType `json:"proof_type,omitempty"` // 1: chunk prover (default type), 2: batch prover
ProofType message.ProofType `json:"prove_type,omitempty"` // 0: chunk prover (default type), 1: batch prover
DumpDir string `json:"dump_dir,omitempty"`
}

View File

@@ -1,6 +1,5 @@
//go:build ffi
// go test -v -race -gcflags="-l" -ldflags="-s=false" -tags ffi ./...
package core_test
import (
@@ -13,7 +12,6 @@ import (
"github.com/scroll-tech/go-ethereum/core/types"
"github.com/stretchr/testify/assert"
scrollTypes "scroll-tech/common/types"
"scroll-tech/common/types/message"
"scroll-tech/prover/config"
@@ -49,32 +47,6 @@ func TestFFI(t *testing.T) {
as.NoError(err)
t.Log("Converted to chunk infos")
wrappedBlock1 := &scrollTypes.WrappedBlock{
Header: chunkTrace1[0].Header,
Transactions: chunkTrace1[0].Transactions,
WithdrawRoot: chunkTrace1[0].WithdrawTrieRoot,
}
chunk1 := &scrollTypes.Chunk{Blocks: []*scrollTypes.WrappedBlock{wrappedBlock1}}
chunkHash1, err := chunk1.Hash(0)
as.NoError(err)
as.Equal(chunkInfo1.PostStateRoot, wrappedBlock1.Header.Root)
as.Equal(chunkInfo1.WithdrawRoot, wrappedBlock1.WithdrawRoot)
as.Equal(chunkInfo1.DataHash, chunkHash1)
t.Log("Successful to check chunk info 1")
wrappedBlock2 := &scrollTypes.WrappedBlock{
Header: chunkTrace2[0].Header,
Transactions: chunkTrace2[0].Transactions,
WithdrawRoot: chunkTrace2[0].WithdrawTrieRoot,
}
chunk2 := &scrollTypes.Chunk{Blocks: []*scrollTypes.WrappedBlock{wrappedBlock2}}
chunkHash2, err := chunk2.Hash(chunk1.NumL1Messages(0))
as.NoError(err)
as.Equal(chunkInfo2.PostStateRoot, wrappedBlock2.Header.Root)
as.Equal(chunkInfo2.WithdrawRoot, wrappedBlock2.WithdrawRoot)
as.Equal(chunkInfo2.DataHash, chunkHash2)
t.Log("Successful to check chunk info 2")
chunkProof1, err := chunkProverCore.ProveChunk("chunk_proof1", chunkTrace1)
as.NoError(err)
t.Log("Generated and dumped chunk proof 1")

View File

@@ -24,6 +24,7 @@ import (
"scroll-tech/common/types/message"
"scroll-tech/common/utils"
"scroll-tech/common/version"
)
var (
@@ -145,13 +146,6 @@ func (r *Prover) proveAndSubmit() error {
}
}
defer func() {
err = r.stack.Delete(task.Task.ID)
if err != nil {
log.Error("prover stack pop failed!", "err", err)
}
}()
var proofMsg *message.ProofDetail
if task.Times <= 2 {
// If panic times <= 2, try to proof the task.
@@ -160,17 +154,26 @@ func (r *Prover) proveAndSubmit() error {
}
log.Info("start to prove task", "task-type", task.Task.Type, "task-id", task.Task.ID)
proofMsg, err = r.prove(task)
if err != nil { // handling error from prove
return fmt.Errorf("failed to prove task, task-type: %v, err: %v", task.Task.Type, err)
proofMsg = r.prove(task)
} else {
// when the prover has more than 3 times panic,
// it will omit to prove the task, submit StatusProofError and then Delete the task.
proofMsg = &message.ProofDetail{
Status: message.StatusProofError,
Error: "zk proving panic",
ID: task.Task.ID,
Type: task.Task.Type,
}
return r.submitProof(proofMsg)
}
// when the prover has more than 3 times panic,
// it will omit to prove the task, submit StatusProofError and then Delete the task.
return fmt.Errorf("zk proving panic for task, task-type: %v, task-id: %v", task.Task.Type, task.Task.ID)
defer func() {
err = r.stack.Delete(task.Task.ID)
if err != nil {
log.Error("prover stack pop failed!", "err", err)
}
}()
return r.submitProof(proofMsg)
}
// fetchTaskFromCoordinator fetches a new task from the server
@@ -187,8 +190,9 @@ func (r *Prover) fetchTaskFromCoordinator() (*store.ProvingTask, error) {
// prepare the request
req := &client.GetTaskRequest{
ProverHeight: latestBlockNumber,
TaskType: r.Type(),
ProverVersion: version.Version,
ProverHeight: latestBlockNumber,
TaskType: r.Type(),
}
// send the request
@@ -236,9 +240,8 @@ func (r *Prover) fetchTaskFromCoordinator() (*store.ProvingTask, error) {
return provingTask, nil
}
// prove function tries to prove a task. It returns an error if the proof fails.
func (r *Prover) prove(task *store.ProvingTask) (*message.ProofDetail, error) {
detail := &message.ProofDetail{
func (r *Prover) prove(task *store.ProvingTask) (detail *message.ProofDetail) {
detail = &message.ProofDetail{
ID: task.Task.ID,
Type: task.Task.Type,
Status: message.StatusOk,
@@ -248,28 +251,30 @@ func (r *Prover) prove(task *store.ProvingTask) (*message.ProofDetail, error) {
case message.ProofTypeChunk:
proof, err := r.proveChunk(task)
if err != nil {
log.Error("prove chunk failed!", "task-id", task.Task.ID, "err", err)
detail.Status = message.StatusProofError
detail.Error = err.Error()
return detail, err
return
}
detail.ChunkProof = proof
log.Info("prove chunk success", "task-id", task.Task.ID)
return detail, nil
log.Info("prove chunk successfully!", "task-id", task.Task.ID)
return
case message.ProofTypeBatch:
proof, err := r.proveBatch(task)
if err != nil {
log.Error("prove batch failed!", "task-id", task.Task.ID, "err", err)
detail.Status = message.StatusProofError
detail.Error = err.Error()
return detail, err
return
}
detail.BatchProof = proof
log.Info("prove batch success", "task-id", task.Task.ID)
return detail, nil
log.Info("prove batch successfully!", "task-id", task.Task.ID)
return
default:
err := fmt.Errorf("invalid task type: %v", task.Task.Type)
return detail, err
log.Error("invalid task type", "task-id", task.Task.ID, "task-type", task.Task.Type)
return
}
}
@@ -279,7 +284,7 @@ func (r *Prover) proveChunk(task *store.ProvingTask) (*message.ChunkProof, error
}
traces, err := r.getSortedTracesByHashes(task.Task.ChunkTaskDetail.BlockHashes)
if err != nil {
return nil, fmt.Errorf("get traces from eth node failed, block hashes: %v, err: %v", task.Task.ChunkTaskDetail.BlockHashes, err)
return nil, fmt.Errorf("get traces from eth node failed, block hashes: %v", task.Task.ChunkTaskDetail.BlockHashes)
}
return r.proverCore.ProveChunk(task.Task.ID, traces)
}