feat(coordinator): when prover failure, recover status (#814)

Co-authored-by: georgehao <georgehao@users.noreply.github.com>

common/version/version.go

@@ -6,7 +6,7 @@ import (
 	"strings"
 )
 
-var tag = "v4.1.61"
+var tag = "v4.1.62"
 
 var commit = func() string {
 	if info, ok := debug.ReadBuildInfo(); ok {

coordinator/internal/controller/api/submit_proof.go

@@ -64,7 +64,7 @@ func (spc *SubmitProofController) SubmitProof(ctx *gin.Context) {
 		proofMsg.BatchProof = &tmpBatchProof
 	}
 
-	if err := spc.submitProofReceiverLogic.HandleZkProof(ctx, &proofMsg); err != nil {
+	if err := spc.submitProofReceiverLogic.HandleZkProof(ctx, &proofMsg, spp); err != nil {
 		nerr := fmt.Errorf("handle zk proof failure, err:%w", err)
 		coodinatorType.RenderJSON(ctx, types.ErrCoordinatorHandleZkProofFailure, nerr, nil)
 		return

coordinator/internal/logic/submitproof/proof_receiver.go

@@ -121,7 +121,7 @@ func NewSubmitProofReceiverLogic(cfg *config.ProverManager, db *gorm.DB, reg pro
 // HandleZkProof handle a ZkProof submitted from a prover.
 // For now only proving/verifying error will lead to setting status as skipped.
 // db/unmarshal errors will not because they are errors on the business logic side.
-func (m *ProofReceiverLogic) HandleZkProof(ctx *gin.Context, proofMsg *message.ProofMsg) error {
+func (m *ProofReceiverLogic) HandleZkProof(ctx *gin.Context, proofMsg *message.ProofMsg, proofParameter coordinatorType.SubmitProofParameter) error {
 	m.proofReceivedTotal.Inc()
 	pk := ctx.GetString(coordinatorType.PublicKey)
 	if len(pk) == 0 {
@@ -144,7 +144,7 @@ func (m *ProofReceiverLogic) HandleZkProof(ctx *gin.Context, proofMsg *message.P
 	log.Info("handling zk proof", "proof id", proofMsg.ID, "prover name", proverTask.ProverName,
 		"prover pk", pk, "prove type", proverTask.TaskType, "proof time", proofTimeSec)
 
-	if err = m.validator(ctx, proverTask, pk, proofMsg); err != nil {
+	if err = m.validator(ctx, proverTask, pk, proofMsg, proofParameter); err != nil {
 		return err
 	}
 
@@ -161,7 +161,7 @@ func (m *ProofReceiverLogic) HandleZkProof(ctx *gin.Context, proofMsg *message.P
 
 	if verifyErr != nil || !success {
 		m.verifierFailureTotal.WithLabelValues(proverVersion).Inc()
-		m.proofFailure(ctx, proofMsg.ID, pk, proofMsg)
+		m.proofRecover(ctx, proofMsg.ID, pk, proofMsg)
 
 		log.Info("proof verified by coordinator failed", "proof id", proofMsg.ID, "prover name", proverTask.ProverName,
 			"prover pk", pk, "prove type", proofMsg.Type, "proof time", proofTimeSec, "error", verifyErr)
@@ -205,7 +205,7 @@ func (m *ProofReceiverLogic) checkAreAllChunkProofsReady(ctx context.Context, ch
 	return nil
 }
 
-func (m *ProofReceiverLogic) validator(ctx context.Context, proverTask *orm.ProverTask, pk string, proofMsg *message.ProofMsg) (err error) {
+func (m *ProofReceiverLogic) validator(ctx context.Context, proverTask *orm.ProverTask, pk string, proofMsg *message.ProofMsg, proofParameter coordinatorType.SubmitProofParameter) (err error) {
 	defer func() {
 		if err != nil {
 			m.validateFailureTotal.Inc()
@@ -232,14 +232,12 @@ func (m *ProofReceiverLogic) validator(ctx context.Context, proverTask *orm.Prov
 	proofTimeSec := uint64(proofTime.Seconds())
 
 	if proofMsg.Status != message.StatusOk {
+		m.proofRecover(ctx, proofMsg.ID, pk, proofMsg)
 		m.validateFailureProverTaskStatusNotOk.Inc()
-		log.Info("proof generated by prover failed", "proof id", proofMsg.ID, "prover name", proverTask.ProverName,
-			"prover pk", pk, "prove type", proofMsg.Type, "error", proofMsg.Error)
-
-		if updateErr := m.proverTaskOrm.UpdateProverTaskProvingStatus(ctx, proofMsg.Type, proofMsg.ID, pk, types.ProverProofInvalid); updateErr != nil {
-			log.Error("proof generated by prover failed update prover task proving status failure", "proof id", proofMsg.ID,
-				"prover name", proverTask.ProverName, "prover pk", pk, "prove type", proofMsg.Type, "error", proofMsg.Error)
-		}
+		log.Info("proof generated by prover failed",
+			"prove type", proofMsg.Type, "proof id", proofMsg.ID,
+			"prover name", proverTask.ProverName, "prover version", proverTask.ProverVersion,
+			"prover pk", "failure type", proofParameter.FailureType, "failure message", proofParameter.FailureMsg)
 		return ErrValidatorFailureProofMsgStatusNotOk
 	}
 
@@ -267,14 +265,14 @@ func (m *ProofReceiverLogic) validator(ctx context.Context, proverTask *orm.Prov
 	return nil
 }
 
-func (m *ProofReceiverLogic) proofFailure(ctx context.Context, hash string, pubKey string, proofMsg *message.ProofMsg) {
-	log.Info("proof failure update proof status", "hash", hash, "public key", pubKey,
-		"proof type", proofMsg.Type.String(), "status", types.ProvingTaskFailed.String())
-
-	if err := m.updateProofStatus(ctx, hash, pubKey, proofMsg, types.ProvingTaskFailed, 0); err != nil {
-		log.Error("failed to updated proof status ProvingTaskFailed", "hash", hash, "pubKey", pubKey, "error", err)
-	}
-}
+//func (m *ProofReceiverLogic) proofFailure(ctx context.Context, hash string, pubKey string, proofMsg *message.ProofMsg) {
+//	log.Info("proof failure update proof status", "hash", hash, "public key", pubKey,
+//		"proof type", proofMsg.Type.String(), "status", types.ProvingTaskFailed.String())
+//
+//	if err := m.updateProofStatus(ctx, hash, pubKey, proofMsg, types.ProvingTaskFailed, 0); err != nil {
+//		log.Error("failed to updated proof status ProvingTaskFailed", "hash", hash, "pubKey", pubKey, "error", err)
+//	}
+//}
 
 func (m *ProofReceiverLogic) proofRecover(ctx context.Context, hash string, pubKey string, proofMsg *message.ProofMsg) {
 	log.Info("proof recover update proof status", "hash", hash, "public key", pubKey,

coordinator/internal/types/submit_proof.go

@@ -2,8 +2,10 @@ package types
 
 // SubmitProofParameter the SubmitProof api request parameter
 type SubmitProofParameter struct {
-	TaskID   string `form:"task_id" json:"task_id" binding:"required"`
-	TaskType int    `form:"task_type" json:"task_type" binding:"required"`
-	Status   int    `form:"status" json:"status"`
-	Proof    string `form:"proof" json:"proof"`
+	TaskID      string `form:"task_id" json:"task_id" binding:"required"`
+	TaskType    int    `form:"task_type" json:"task_type" binding:"required"`
+	Status      int    `form:"status" json:"status"`
+	Proof       string `form:"proof" json:"proof"`
+	FailureType int    `form:"failure_type" json:"failure_type"`
+	FailureMsg  string `form:"failure_msg" json:"failure_msg"`
 }

coordinator/test/api_test.go

@@ -317,7 +317,7 @@ func testInvalidProof(t *testing.T) {
 			assert.NoError(t, err)
 			batchProofStatus, err = batchOrm.GetProvingStatusByHash(context.Background(), batch.Hash)
 			assert.NoError(t, err)
-			if chunkProofStatus == types.ProvingTaskFailed && batchProofStatus == types.ProvingTaskFailed {
+			if chunkProofStatus == types.ProvingTaskUnassigned && batchProofStatus == types.ProvingTaskUnassigned {
 				return
 			}
 		case <-tickStop: