feat: allow overwriting bridge-history batch indexing start height (#816)

Co-authored-by: HAOYUatHZ <HAOYUatHZ@users.noreply.github.com>
Co-authored-by: georgehao <haohongfan@gmail.com>

bridge-history-api/crossmsg/batch_info_fetcher.go

@@ -95,6 +95,9 @@ func (b *BatchInfoFetcher) fetchBatchInfo() error {
 	} else {
 		startHeight = latestBatchHeight + 1
 	}
+	if startHeight < b.batchInfoStartNumber {
+		startHeight = b.batchInfoStartNumber
+	}
 	for from := startHeight; number >= from; from += fetchLimit {
 		to := from + fetchLimit - 1
 		// number - confirmation can never less than 0 since the for loop condition

bridge/cmd/rollup_relayer/app/app.go

@@ -110,7 +110,7 @@ func action(ctx *cli.Context) error {
 
 	go utils.Loop(subCtx, 2*time.Second, l2relayer.ProcessPendingBatches)
 
-	go utils.Loop(subCtx, 10*time.Second, l2relayer.ProcessCommittedBatches)
+	go utils.Loop(subCtx, 15*time.Second, l2relayer.ProcessCommittedBatches)
 
 	// Finish start all rollup relayer functions.
 	log.Info("Start rollup-relayer successfully")

bridge/internal/controller/relayer/l2_relayer.go

@@ -320,7 +320,7 @@ func (r *Layer2Relayer) ProcessGasPriceOracle() {
 // ProcessPendingBatches processes the pending batches by sending commitBatch transactions to layer 1.
 func (r *Layer2Relayer) ProcessPendingBatches() {
 	// get pending batches from database in ascending order by their index.
-	pendingBatches, err := r.batchOrm.GetPendingBatches(r.ctx, 10)
+	pendingBatches, err := r.batchOrm.GetPendingBatches(r.ctx, 1)
 	if err != nil {
 		log.Error("Failed to fetch pending L2 batches", "err", err)
 		return

bridge/tests/rollup_test.go

@@ -5,7 +5,6 @@ import (
 	"math/big"
 	"testing"
 
-	"github.com/scroll-tech/go-ethereum/accounts/abi/bind"
 	"github.com/scroll-tech/go-ethereum/common"
 	gethTypes "github.com/scroll-tech/go-ethereum/core/types"
 	"github.com/stretchr/testify/assert"
@@ -13,6 +12,7 @@ import (
 	"scroll-tech/common/database"
 	"scroll-tech/common/types"
 	"scroll-tech/common/types/message"
+	"scroll-tech/common/utils"
 
 	"scroll-tech/bridge/internal/config"
 	"scroll-tech/bridge/internal/controller/relayer"
@@ -91,20 +91,22 @@ func testCommitBatchAndFinalizeBatch(t *testing.T) {
 	assert.NotEmpty(t, batch.CommitTxHash)
 	assert.Equal(t, types.RollupCommitting, types.RollupStatus(batch.RollupStatus))
 
-	assert.NoError(t, err)
-	commitTx, _, err := l1Client.TransactionByHash(context.Background(), common.HexToHash(batch.CommitTxHash))
-	assert.NoError(t, err)
-	commitTxReceipt, err := bind.WaitMined(context.Background(), l1Client, commitTx)
-	assert.NoError(t, err)
-	assert.Equal(t, len(commitTxReceipt.Logs), 1)
+	success := utils.TryTimes(30, func() bool {
+		var receipt *gethTypes.Receipt
+		receipt, err = l1Client.TransactionReceipt(context.Background(), common.HexToHash(batch.CommitTxHash))
+		return err == nil && receipt.Status == 1
+	})
+	assert.True(t, success)
 
 	// fetch rollup events
-	err = l1Watcher.FetchContractEvent()
-	assert.NoError(t, err)
-	statuses, err := batchOrm.GetRollupStatusByHashList(context.Background(), []string{batchHash})
-	assert.NoError(t, err)
-	assert.Equal(t, 1, len(statuses))
-	assert.Equal(t, types.RollupCommitted, statuses[0])
+	success = utils.TryTimes(30, func() bool {
+		err = l1Watcher.FetchContractEvent()
+		assert.NoError(t, err)
+		var statuses []types.RollupStatus
+		statuses, err = batchOrm.GetRollupStatusByHashList(context.Background(), []string{batchHash})
+		return err == nil && len(statuses) == 1 && types.RollupCommitted == statuses[0]
+	})
+	assert.True(t, success)
 
 	// add dummy proof
 	proof := &message.BatchProof{
@@ -118,7 +120,7 @@ func testCommitBatchAndFinalizeBatch(t *testing.T) {
 	// process committed batch and check status
 	l2Relayer.ProcessCommittedBatches()
 
-	statuses, err = batchOrm.GetRollupStatusByHashList(context.Background(), []string{batchHash})
+	statuses, err := batchOrm.GetRollupStatusByHashList(context.Background(), []string{batchHash})
 	assert.NoError(t, err)
 	assert.Equal(t, 1, len(statuses))
 	assert.Equal(t, types.RollupFinalizing, statuses[0])
@@ -128,17 +130,20 @@ func testCommitBatchAndFinalizeBatch(t *testing.T) {
 	assert.NotNil(t, batch)
 	assert.NotEmpty(t, batch.FinalizeTxHash)
 
-	finalizeTx, _, err := l1Client.TransactionByHash(context.Background(), common.HexToHash(batch.FinalizeTxHash))
-	assert.NoError(t, err)
-	finalizeTxReceipt, err := bind.WaitMined(context.Background(), l1Client, finalizeTx)
-	assert.NoError(t, err)
-	assert.Equal(t, len(finalizeTxReceipt.Logs), 1)
+	success = utils.TryTimes(30, func() bool {
+		var receipt *gethTypes.Receipt
+		receipt, err = l1Client.TransactionReceipt(context.Background(), common.HexToHash(batch.FinalizeTxHash))
+		return err == nil && receipt.Status == 1
+	})
+	assert.True(t, success)
 
 	// fetch rollup events
-	err = l1Watcher.FetchContractEvent()
-	assert.NoError(t, err)
-	statuses, err = batchOrm.GetRollupStatusByHashList(context.Background(), []string{batchHash})
-	assert.NoError(t, err)
-	assert.Equal(t, 1, len(statuses))
-	assert.Equal(t, types.RollupFinalized, statuses[0])
+	success = utils.TryTimes(30, func() bool {
+		err = l1Watcher.FetchContractEvent()
+		assert.NoError(t, err)
+		var statuses []types.RollupStatus
+		statuses, err = batchOrm.GetRollupStatusByHashList(context.Background(), []string{batchHash})
+		return err == nil && len(statuses) == 1 && types.RollupFinalized == statuses[0]
+	})
+	assert.True(t, success)
 }

common/types/message/message.go

@@ -13,6 +13,18 @@ import (
 	"github.com/scroll-tech/go-ethereum/rlp"
 )
 
+// ProofFailureType the proof failure type
+type ProofFailureType int
+
+const (
+	// ProofFailureUndefined the undefined type proof failure type
+	ProofFailureUndefined ProofFailureType = iota
+	// ProofFailurePanic proof failure for prover panic
+	ProofFailurePanic
+	// ProofFailureNoPanic proof failure for no prover panic
+	ProofFailureNoPanic
+)
+
 // RespStatus represents status code from prover to scroll
 type RespStatus uint32
 

common/version/version.go

@@ -6,7 +6,7 @@ import (
 	"strings"
 )
 
-var tag = "v4.1.53"
+var tag = "v4.1.64"
 
 var commit = func() string {
 	if info, ok := debug.ReadBuildInfo(); ok {

contracts/src/misc/ScrollOwner.sol

@@ -0,0 +1,130 @@
+// SPDX-License-Identifier: MIT
+
+pragma solidity ^0.8.0;
+
+import {AccessControlEnumerable} from "@openzeppelin/contracts/access/AccessControlEnumerable.sol";
+import {EnumerableSet} from "@openzeppelin/contracts/utils/structs/EnumerableSet.sol";
+
+// solhint-disable no-empty-blocks
+
+contract ScrollOwner is AccessControlEnumerable {
+    using EnumerableSet for EnumerableSet.Bytes32Set;
+
+    /*************
+     * Variables *
+     *************/
+
+    /// @notice Mapping from target address to selector to the list of accessible roles.
+    mapping(address => mapping(bytes4 => EnumerableSet.Bytes32Set)) private targetAccess;
+
+    /**********************
+     * Function Modifiers *
+     **********************/
+
+    modifier hasAccess(
+        address _target,
+        bytes4 _selector,
+        bytes32 _role
+    ) {
+        // admin has access to all methods
+        require(_role == DEFAULT_ADMIN_ROLE || targetAccess[_target][_selector].contains(_role), "no access");
+        _;
+    }
+
+    /***************
+     * Constructor *
+     ***************/
+
+    constructor() {
+        _grantRole(DEFAULT_ADMIN_ROLE, msg.sender);
+    }
+
+    /*************************
+     * Public View Functions *
+     *************************/
+
+    /// @notice Return a list of roles which has access to the function.
+    /// @param _target The address of target contract.
+    /// @param _selector The function selector to query.
+    /// @return _roles The list of roles.
+    function callableRoles(address _target, bytes4 _selector) external view returns (bytes32[] memory _roles) {
+        EnumerableSet.Bytes32Set storage _lists = targetAccess[_target][_selector];
+        _roles = new bytes32[](_lists.length());
+        for (uint256 i = 0; i < _roles.length; i++) {
+            _roles[i] = _lists.at(i);
+        }
+    }
+
+    /*****************************
+     * Public Mutating Functions *
+     *****************************/
+
+    /// @notice Perform a function call from arbitrary role.
+    /// @param _target The address of target contract.
+    /// @param _value The value passing to target contract.
+    /// @param _data The calldata passing to target contract.
+    /// @param _role The expected role of the caller.
+    function execute(
+        address _target,
+        uint256 _value,
+        bytes calldata _data,
+        bytes32 _role
+    ) public payable onlyRole(_role) hasAccess(_target, bytes4(_data[0:4]), _role) {
+        _execute(_target, _value, _data);
+    }
+
+    // allow others to send ether to this contract.
+    receive() external payable {}
+
+    /************************
+     * Restricted Functions *
+     ************************/
+
+    /// @notice Update the access to target contract.
+    /// @param _target The address of target contract.
+    /// @param _selectors The list of function selectors to update.
+    /// @param _role The role to change.
+    /// @param _status True if we are going to add the role, otherwise remove the role.
+    function updateAccess(
+        address _target,
+        bytes4[] memory _selectors,
+        bytes32 _role,
+        bool _status
+    ) external onlyRole(DEFAULT_ADMIN_ROLE) {
+        if (_status) {
+            for (uint256 i = 0; i < _selectors.length; i++) {
+                targetAccess[_target][_selectors[i]].add(_role);
+            }
+        } else {
+            for (uint256 i = 0; i < _selectors.length; i++) {
+                targetAccess[_target][_selectors[i]].remove(_role);
+            }
+        }
+    }
+
+    /**********************
+     * Internal Functions *
+     **********************/
+
+    /// @dev Internal function to call contract. If the call reverted, the error will be popped up.
+    /// @param _target The address of target contract.
+    /// @param _value The value passing to target contract.
+    /// @param _data The calldata passing to target contract.
+    function _execute(
+        address _target,
+        uint256 _value,
+        bytes calldata _data
+    ) internal {
+        // solhint-disable-next-line avoid-low-level-calls
+        (bool success, ) = address(_target).call{value: _value}(_data);
+        if (!success) {
+            // solhint-disable-next-line no-inline-assembly
+            assembly {
+                let ptr := mload(0x40)
+                let size := returndatasize()
+                returndatacopy(ptr, 0, size)
+                revert(ptr, size)
+            }
+        }
+    }
+}

contracts/src/test/ScrollOwner.t.sol

@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: MIT
+
+pragma solidity ^0.8.0;
+
+import {DSTestPlus} from "solmate/test/utils/DSTestPlus.sol";
+
+import {ScrollOwner} from "../misc/ScrollOwner.sol";
+
+contract ScrollOwnerTest is DSTestPlus {
+    event Call();
+
+    ScrollOwner private owner;
+
+    function setUp() public {
+        owner = new ScrollOwner();
+    }
+
+    function testUpdateAccess() external {
+        // not admin, evert
+        hevm.startPrank(address(1));
+        hevm.expectRevert(
+            "AccessControl: account 0x0000000000000000000000000000000000000001 is missing role 0x0000000000000000000000000000000000000000000000000000000000000000"
+        );
+        owner.updateAccess(address(0), new bytes4[](0), bytes32(0), true);
+        hevm.stopPrank();
+
+        bytes4[] memory _selectors;
+        bytes32[] memory _roles;
+
+        // add access then remove access
+        _roles = owner.callableRoles(address(this), ScrollOwnerTest.revertOnCall.selector);
+        assertEq(0, _roles.length);
+        _selectors = new bytes4[](1);
+        _selectors[0] = ScrollOwnerTest.revertOnCall.selector;
+        owner.updateAccess(address(this), _selectors, bytes32(uint256(1)), true);
+        _roles = owner.callableRoles(address(this), ScrollOwnerTest.revertOnCall.selector);
+        assertEq(1, _roles.length);
+        assertEq(_roles[0], bytes32(uint256(1)));
+        owner.updateAccess(address(this), _selectors, bytes32(uint256(1)), false);
+        _roles = owner.callableRoles(address(this), ScrollOwnerTest.revertOnCall.selector);
+        assertEq(0, _roles.length);
+    }
+
+    function testAdminExecute() external {
+        // call with revert
+        hevm.expectRevert("Called");
+        owner.execute(address(this), 0, abi.encodeWithSelector(ScrollOwnerTest.revertOnCall.selector), bytes32(0));
+
+        // call with emit
+        hevm.expectEmit(false, false, false, true);
+        emit Call();
+        owner.execute(address(this), 0, abi.encodeWithSelector(ScrollOwnerTest.emitOnCall.selector), bytes32(0));
+    }
+
+    function testExecute(bytes32 _role) external {
+        hevm.assume(_role != bytes32(0));
+
+        bytes4[] memory _selectors = new bytes4[](2);
+        _selectors[0] = ScrollOwnerTest.revertOnCall.selector;
+        _selectors[1] = ScrollOwnerTest.emitOnCall.selector;
+
+        owner.grantRole(_role, address(this));
+
+        // no access, revert
+        hevm.expectRevert("no access");
+        owner.execute(address(this), 0, abi.encodeWithSelector(ScrollOwnerTest.revertOnCall.selector), _role);
+
+        owner.updateAccess(address(this), _selectors, _role, true);
+
+        // call with revert
+        hevm.expectRevert("Called");
+        owner.execute(address(this), 0, abi.encodeWithSelector(ScrollOwnerTest.revertOnCall.selector), _role);
+
+        // call with emit
+        hevm.expectEmit(false, false, false, true);
+        emit Call();
+        owner.execute(address(this), 0, abi.encodeWithSelector(ScrollOwnerTest.emitOnCall.selector), _role);
+    }
+
+    function revertOnCall() external pure {
+        revert("Called");
+    }
+
+    function emitOnCall() external {
+        emit Call();
+    }
+}

coordinator/cmd/app/mock_app.go

@@ -81,11 +81,12 @@ func (c *CoordinatorApp) MockConfig(store bool) error {
 	}
 	// Reset prover manager config for manager test cases.
 	cfg.ProverManager = &coordinatorConfig.ProverManager{
-		ProversPerSession:  1,
-		Verifier:           &coordinatorConfig.VerifierConfig{MockMode: true},
-		CollectionTimeSec:  60,
-		SessionAttempts:    10,
-		MaxVerifierWorkers: 4,
+		ProversPerSession:      1,
+		Verifier:               &coordinatorConfig.VerifierConfig{MockMode: true},
+		BatchCollectionTimeSec: 60,
+		ChunkCollectionTimeSec: 60,
+		SessionAttempts:        10,
+		MaxVerifierWorkers:     4,
 	}
 	cfg.DB.DSN = base.DBImg.Endpoint()
 	cfg.L2.ChainID = 111

coordinator/conf/config.json

@@ -2,7 +2,8 @@
   "prover_manager": {
     "provers_per_session": 1,
     "session_attempts": 5,
-    "collection_time_sec": 180,
+    "batch_collection_time_sec": 180,
+    "chunk_collection_time_sec": 180,
     "verifier": {
       "mock_mode": true,
       "params_path": "",

coordinator/internal/config/config.go

@@ -17,8 +17,10 @@ type ProverManager struct {
 	SessionAttempts uint8 `json:"session_attempts"`
 	// Zk verifier config.
 	Verifier *VerifierConfig `json:"verifier"`
-	// Proof collection time (in seconds).
-	CollectionTimeSec int `json:"collection_time_sec"`
+	// BatchCollectionTimeSec batch Proof collection time (in seconds).
+	BatchCollectionTimeSec int `json:"batch_collection_time_sec"`
+	// ChunkCollectionTimeSec chunk Proof collection time (in seconds).
+	ChunkCollectionTimeSec int `json:"chunk_collection_time_sec"`
 	// Max number of workers in verifier worker pool
 	MaxVerifierWorkers int `json:"max_verifier_workers"`
 }

coordinator/internal/config/config_test.go

@@ -15,7 +15,8 @@ func TestConfig(t *testing.T) {
 		"prover_manager": {
 			"provers_per_session": 1,
 			"session_attempts": 5,
-			"collection_time_sec": 180,
+			"batch_collection_time_sec": 180,
+			"chunk_collection_time_sec": 180,
 			"verifier": {
 				"mock_mode": true,
 				"params_path": "",

coordinator/internal/controller/api/submit_proof.go

@@ -64,7 +64,7 @@ func (spc *SubmitProofController) SubmitProof(ctx *gin.Context) {
 		proofMsg.BatchProof = &tmpBatchProof
 	}
 
-	if err := spc.submitProofReceiverLogic.HandleZkProof(ctx, &proofMsg); err != nil {
+	if err := spc.submitProofReceiverLogic.HandleZkProof(ctx, &proofMsg, spp); err != nil {
 		nerr := fmt.Errorf("handle zk proof failure, err:%w", err)
 		coodinatorType.RenderJSON(ctx, types.ErrCoordinatorHandleZkProofFailure, nerr, nil)
 		return

coordinator/internal/controller/cron/collect_proof.go

@@ -29,8 +29,10 @@ type Collector struct {
 	chunkOrm      *orm.Chunk
 	batchOrm      *orm.Batch
 
-	timeoutCheckerRunTotal prometheus.Counter
-	proverTaskTimeoutTotal prometheus.Counter
+	timeoutBatchCheckerRunTotal prometheus.Counter
+	batchProverTaskTimeoutTotal prometheus.Counter
+	timeoutChunkCheckerRunTotal prometheus.Counter
+	chunkProverTaskTimeoutTotal prometheus.Counter
 }
 
 // NewCollector create a collector to cron collect the data to send to prover
@@ -44,17 +46,26 @@ func NewCollector(ctx context.Context, db *gorm.DB, cfg *config.Config, reg prom
 		chunkOrm:        orm.NewChunk(db),
 		batchOrm:        orm.NewBatch(db),
 
-		timeoutCheckerRunTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-			Name: "coordinator_timeout_checker_run_total",
-			Help: "Total number of timeout checker run.",
+		timeoutBatchCheckerRunTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
+			Name: "coordinator_batch_timeout_checker_run_total",
+			Help: "Total number of batch timeout checker run.",
 		}),
-		proverTaskTimeoutTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-			Name: "coordinator_prover_task_timeout_total",
-			Help: "Total number of timeout prover task.",
+		batchProverTaskTimeoutTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
+			Name: "coordinator_batch_prover_task_timeout_total",
+			Help: "Total number of batch timeout prover task.",
+		}),
+		timeoutChunkCheckerRunTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
+			Name: "coordinator_chunk_timeout_checker_run_total",
+			Help: "Total number of chunk timeout checker run.",
+		}),
+		chunkProverTaskTimeoutTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
+			Name: "coordinator_chunk_prover_task_timeout_total",
+			Help: "Total number of chunk timeout prover task.",
 		}),
 	}
 
-	go c.timeoutProofTask()
+	go c.timeoutBatchProofTask()
+	go c.timeoutChunkProofTask()
 
 	log.Info("Start coordinator successfully.")
 
@@ -69,10 +80,10 @@ func (c *Collector) Stop() {
 // timeoutTask cron check the send task is timeout. if timeout reached, restore the
 // chunk/batch task to unassigned. then the batch/chunk collector can retry it.
 
-func (c *Collector) timeoutProofTask() {
+func (c *Collector) timeoutBatchProofTask() {
 	defer func() {
 		if err := recover(); err != nil {
-			nerr := fmt.Errorf("timeout proof task panic error:%v", err)
+			nerr := fmt.Errorf("timeout batch proof task panic error:%v", err)
 			log.Warn(nerr.Error())
 		}
 	}()
@@ -81,52 +92,14 @@ func (c *Collector) timeoutProofTask() {
 	for {
 		select {
 		case <-ticker.C:
-			c.timeoutCheckerRunTotal.Inc()
-			timeout := time.Duration(c.cfg.ProverManager.CollectionTimeSec) * time.Second
-			assignedProverTasks, err := c.proverTaskOrm.GetTimeoutAssignedProverTasks(c.ctx, 10, timeout)
+			c.timeoutBatchCheckerRunTotal.Inc()
+			timeout := time.Duration(c.cfg.ProverManager.BatchCollectionTimeSec) * time.Second
+			assignedProverTasks, err := c.proverTaskOrm.GetTimeoutAssignedProverTasks(c.ctx, 10, message.ProofTypeBatch, timeout)
 			if err != nil {
 				log.Error("get unassigned session info failure", "error", err)
 				break
 			}
-
-			// here not update the block batch proving status failed, because the collector loop will check
-			// the attempt times. if reach the times, the collector will set the block batch proving status.
-			for _, assignedProverTask := range assignedProverTasks {
-				c.proverTaskTimeoutTotal.Inc()
-				log.Warn("proof task have reach the timeout", "task id", assignedProverTask.TaskID,
-					"prover public key", assignedProverTask.ProverPublicKey, "prover name", assignedProverTask.ProverName, "task type", assignedProverTask.TaskType)
-				err = c.db.Transaction(func(tx *gorm.DB) error {
-					// update prover task proving status as ProverProofInvalid
-					if err = c.proverTaskOrm.UpdateProverTaskProvingStatus(c.ctx, message.ProofType(assignedProverTask.TaskType),
-						assignedProverTask.TaskID, assignedProverTask.ProverPublicKey, types.ProverProofInvalid, tx); err != nil {
-						log.Error("update prover task proving status failure", "hash", assignedProverTask.TaskID, "pubKey", assignedProverTask.ProverPublicKey, "err", err)
-						return err
-					}
-
-					// update prover task failure type
-					if err = c.proverTaskOrm.UpdateProverTaskFailureType(c.ctx, message.ProofType(assignedProverTask.TaskType),
-						assignedProverTask.TaskID, assignedProverTask.ProverPublicKey, types.ProverTaskFailureTypeTimeout, tx); err != nil {
-						log.Error("update prover task failure type failure", "hash", assignedProverTask.TaskID, "pubKey", assignedProverTask.ProverPublicKey, "err", err)
-						return err
-					}
-
-					// update the task to unassigned, let collector restart it
-					if message.ProofType(assignedProverTask.TaskType) == message.ProofTypeChunk {
-						if err = c.chunkOrm.UpdateProvingStatus(c.ctx, assignedProverTask.TaskID, types.ProvingTaskUnassigned, tx); err != nil {
-							log.Error("update chunk proving status to unassigned to restart it failure", "hash", assignedProverTask.TaskID, "err", err)
-						}
-					}
-					if message.ProofType(assignedProverTask.TaskType) == message.ProofTypeBatch {
-						if err = c.batchOrm.UpdateProvingStatus(c.ctx, assignedProverTask.TaskID, types.ProvingTaskUnassigned, tx); err != nil {
-							log.Error("update batch proving status to unassigned to restart it failure", "hash", assignedProverTask.TaskID, "err", err)
-						}
-					}
-					return nil
-				})
-				if err != nil {
-					log.Error("check task proof is timeout failure", "error", err)
-				}
-			}
+			c.check(assignedProverTasks, c.batchProverTaskTimeoutTotal)
 		case <-c.ctx.Done():
 			if c.ctx.Err() != nil {
 				log.Error("manager context canceled with error", "error", c.ctx.Err())
@@ -138,3 +111,81 @@ func (c *Collector) timeoutProofTask() {
 		}
 	}
 }
+
+func (c *Collector) timeoutChunkProofTask() {
+	defer func() {
+		if err := recover(); err != nil {
+			nerr := fmt.Errorf("timeout proof chunk task panic error:%v", err)
+			log.Warn(nerr.Error())
+		}
+	}()
+
+	ticker := time.NewTicker(time.Second * 2)
+	for {
+		select {
+		case <-ticker.C:
+			c.timeoutChunkCheckerRunTotal.Inc()
+			timeout := time.Duration(c.cfg.ProverManager.ChunkCollectionTimeSec) * time.Second
+			assignedProverTasks, err := c.proverTaskOrm.GetTimeoutAssignedProverTasks(c.ctx, 10, message.ProofTypeChunk, timeout)
+			if err != nil {
+				log.Error("get unassigned session info failure", "error", err)
+				break
+			}
+			c.check(assignedProverTasks, c.chunkProverTaskTimeoutTotal)
+
+		case <-c.ctx.Done():
+			if c.ctx.Err() != nil {
+				log.Error("manager context canceled with error", "error", c.ctx.Err())
+			}
+			return
+		case <-c.stopTimeoutChan:
+			log.Info("the coordinator run loop exit")
+			return
+		}
+	}
+}
+
+func (c *Collector) check(assignedProverTasks []orm.ProverTask, timeout prometheus.Counter) {
+	// here not update the block batch proving status failed, because the collector loop will check
+	// the attempt times. if reach the times, the collector will set the block batch proving status.
+	for _, assignedProverTask := range assignedProverTasks {
+		if c.proverTaskOrm.TaskTimeoutMoreThanOnce(c.ctx, assignedProverTask.TaskID) {
+			log.Warn("Task timeout more than once", "hash", assignedProverTask.TaskID)
+		}
+
+		timeout.Inc()
+		log.Warn("proof task have reach the timeout", "task id", assignedProverTask.TaskID,
+			"prover public key", assignedProverTask.ProverPublicKey, "prover name", assignedProverTask.ProverName, "task type", assignedProverTask.TaskType)
+		err := c.db.Transaction(func(tx *gorm.DB) error {
+			// update prover task proving status as ProverProofInvalid
+			if err := c.proverTaskOrm.UpdateProverTaskProvingStatus(c.ctx, message.ProofType(assignedProverTask.TaskType),
+				assignedProverTask.TaskID, assignedProverTask.ProverPublicKey, types.ProverProofInvalid, tx); err != nil {
+				log.Error("update prover task proving status failure", "hash", assignedProverTask.TaskID, "pubKey", assignedProverTask.ProverPublicKey, "err", err)
+				return err
+			}
+
+			// update prover task failure type
+			if err := c.proverTaskOrm.UpdateProverTaskFailureType(c.ctx, message.ProofType(assignedProverTask.TaskType),
+				assignedProverTask.TaskID, assignedProverTask.ProverPublicKey, types.ProverTaskFailureTypeTimeout, tx); err != nil {
+				log.Error("update prover task failure type failure", "hash", assignedProverTask.TaskID, "pubKey", assignedProverTask.ProverPublicKey, "err", err)
+				return err
+			}
+
+			// update the task to unassigned, let collector restart it
+			if message.ProofType(assignedProverTask.TaskType) == message.ProofTypeChunk {
+				if err := c.chunkOrm.UpdateProvingStatus(c.ctx, assignedProverTask.TaskID, types.ProvingTaskUnassigned, tx); err != nil {
+					log.Error("update chunk proving status to unassigned to restart it failure", "hash", assignedProverTask.TaskID, "err", err)
+				}
+			}
+			if message.ProofType(assignedProverTask.TaskType) == message.ProofTypeBatch {
+				if err := c.batchOrm.UpdateProvingStatus(c.ctx, assignedProverTask.TaskID, types.ProvingTaskUnassigned, tx); err != nil {
+					log.Error("update batch proving status to unassigned to restart it failure", "hash", assignedProverTask.TaskID, "err", err)
+				}
+			}
+			return nil
+		})
+		if err != nil {
+			log.Error("check task proof is timeout failure", "error", err)
+		}
+	}
+}

coordinator/internal/logic/submitproof/proof_receiver.go

@@ -121,14 +121,18 @@ func NewSubmitProofReceiverLogic(cfg *config.ProverManager, db *gorm.DB, reg pro
 // HandleZkProof handle a ZkProof submitted from a prover.
 // For now only proving/verifying error will lead to setting status as skipped.
 // db/unmarshal errors will not because they are errors on the business logic side.
-func (m *ProofReceiverLogic) HandleZkProof(ctx *gin.Context, proofMsg *message.ProofMsg) error {
+func (m *ProofReceiverLogic) HandleZkProof(ctx *gin.Context, proofMsg *message.ProofMsg, proofParameter coordinatorType.SubmitProofParameter) error {
 	m.proofReceivedTotal.Inc()
 	pk := ctx.GetString(coordinatorType.PublicKey)
 	if len(pk) == 0 {
 		return fmt.Errorf("get public key from context failed")
 	}
+	pv := ctx.GetString(coordinatorType.ProverVersion)
+	if len(pk) == 0 {
+		return fmt.Errorf("get ProverVersion from context failed")
+	}
 
-	proverTask, err := m.proverTaskOrm.GetProverTaskByTaskIDAndPubKey(ctx, proofMsg.ID, pk)
+	proverTask, err := m.proverTaskOrm.GetProverTaskByTaskIDAndProver(ctx, proofMsg.ID, pk, pv)
 	if proverTask == nil || err != nil {
 		log.Error("get none prover task for the proof", "key", pk, "taskID", proofMsg.ID, "error", err)
 		return ErrValidatorFailureProverTaskEmpty
@@ -140,7 +144,7 @@ func (m *ProofReceiverLogic) HandleZkProof(ctx *gin.Context, proofMsg *message.P
 	log.Info("handling zk proof", "proof id", proofMsg.ID, "prover name", proverTask.ProverName,
 		"prover pk", pk, "prove type", proverTask.TaskType, "proof time", proofTimeSec)
 
-	if err = m.validator(ctx, proverTask, pk, proofMsg); err != nil {
+	if err = m.validator(ctx, proverTask, pk, proofMsg, proofParameter); err != nil {
 		return err
 	}
 
@@ -157,7 +161,7 @@ func (m *ProofReceiverLogic) HandleZkProof(ctx *gin.Context, proofMsg *message.P
 
 	if verifyErr != nil || !success {
 		m.verifierFailureTotal.WithLabelValues(proverVersion).Inc()
-		m.proofFailure(ctx, proofMsg.ID, pk, proofMsg)
+		m.proofRecover(ctx, proofMsg.ID, pk, proofMsg)
 
 		log.Info("proof verified by coordinator failed", "proof id", proofMsg.ID, "prover name", proverTask.ProverName,
 			"prover pk", pk, "prove type", proofMsg.Type, "proof time", proofTimeSec, "error", verifyErr)
@@ -201,8 +205,13 @@ func (m *ProofReceiverLogic) checkAreAllChunkProofsReady(ctx context.Context, ch
 	return nil
 }
 
-func (m *ProofReceiverLogic) validator(ctx context.Context, proverTask *orm.ProverTask, pk string, proofMsg *message.ProofMsg) error {
-	m.validateFailureTotal.Inc()
+func (m *ProofReceiverLogic) validator(ctx context.Context, proverTask *orm.ProverTask, pk string, proofMsg *message.ProofMsg, proofParameter coordinatorType.SubmitProofParameter) (err error) {
+	defer func() {
+		if err != nil {
+			m.validateFailureTotal.Inc()
+		}
+	}()
+
 	// Ensure this prover is eligible to participate in the prover task.
 	if types.ProverProveStatus(proverTask.ProvingStatus) == types.ProverProofValid {
 		m.validateFailureProverTaskSubmitTwice.Inc()
@@ -210,8 +219,12 @@ func (m *ProofReceiverLogic) validator(ctx context.Context, proverTask *orm.Prov
 		// TODO: Defend invalid proof resubmissions by one of the following two methods:
 		// (i) slash the prover for each submission of invalid proof
 		// (ii) set the maximum failure retry times
-		log.Warn("the prover task cannot submit twice", "hash", proofMsg.ID, "prover pk", proverTask.ProverPublicKey,
-			"prover name", proverTask.ProverName, "proof type", proverTask.TaskType)
+		log.Warn(
+			"cannot submit valid proof for a prover task twice",
+			"proof type", proverTask.TaskType, "hash", proofMsg.ID,
+			"prover name", proverTask.ProverName, "prover version", proverTask.ProverVersion,
+			"prover pk", proverTask.ProverPublicKey,
+		)
 		return ErrValidatorFailureProverTaskCannotSubmitTwice
 	}
 
@@ -219,14 +232,12 @@ func (m *ProofReceiverLogic) validator(ctx context.Context, proverTask *orm.Prov
 	proofTimeSec := uint64(proofTime.Seconds())
 
 	if proofMsg.Status != message.StatusOk {
+		m.proofRecover(ctx, proofMsg.ID, pk, proofMsg)
 		m.validateFailureProverTaskStatusNotOk.Inc()
-		log.Info("proof generated by prover failed", "proof id", proofMsg.ID, "prover name", proverTask.ProverName,
-			"prover pk", pk, "prove type", proofMsg.Type, "error", proofMsg.Error)
-
-		if updateErr := m.proverTaskOrm.UpdateProverTaskProvingStatus(ctx, proofMsg.Type, proofMsg.ID, pk, types.ProverProofInvalid); updateErr != nil {
-			log.Error("proof generated by prover failed update prover task proving status failure", "proof id", proofMsg.ID,
-				"prover name", proverTask.ProverName, "prover pk", pk, "prove type", proofMsg.Type, "error", proofMsg.Error)
-		}
+		log.Info("proof generated by prover failed",
+			"prove type", proofMsg.Type, "proof id", proofMsg.ID,
+			"prover name", proverTask.ProverName, "prover version", proverTask.ProverVersion,
+			"prover pk", "failure type", proofParameter.FailureType, "failure message", proofParameter.FailureMsg)
 		return ErrValidatorFailureProofMsgStatusNotOk
 	}
 
@@ -254,14 +265,14 @@ func (m *ProofReceiverLogic) validator(ctx context.Context, proverTask *orm.Prov
 	return nil
 }
 
-func (m *ProofReceiverLogic) proofFailure(ctx context.Context, hash string, pubKey string, proofMsg *message.ProofMsg) {
-	log.Info("proof failure update proof status", "hash", hash, "public key", pubKey,
-		"proof type", proofMsg.Type.String(), "status", types.ProvingTaskFailed.String())
-
-	if err := m.updateProofStatus(ctx, hash, pubKey, proofMsg, types.ProvingTaskFailed, 0); err != nil {
-		log.Error("failed to updated proof status ProvingTaskFailed", "hash", hash, "pubKey", pubKey, "error", err)
-	}
-}
+//func (m *ProofReceiverLogic) proofFailure(ctx context.Context, hash string, pubKey string, proofMsg *message.ProofMsg) {
+//	log.Info("proof failure update proof status", "hash", hash, "public key", pubKey,
+//		"proof type", proofMsg.Type.String(), "status", types.ProvingTaskFailed.String())
+//
+//	if err := m.updateProofStatus(ctx, hash, pubKey, proofMsg, types.ProvingTaskFailed, 0); err != nil {
+//		log.Error("failed to updated proof status ProvingTaskFailed", "hash", hash, "pubKey", pubKey, "error", err)
+//	}
+//}
 
 func (m *ProofReceiverLogic) proofRecover(ctx context.Context, hash string, pubKey string, proofMsg *message.ProofMsg) {
 	log.Info("proof recover update proof status", "hash", hash, "public key", pubKey,

coordinator/internal/orm/prover_task.go

@@ -113,16 +113,18 @@ func (o *ProverTask) GetProverTasksByHashes(ctx context.Context, hashes []string
 	return proverTasks, nil
 }
 
-// GetProverTaskByTaskIDAndPubKey get prover task taskID and public key
-func (o *ProverTask) GetProverTaskByTaskIDAndPubKey(ctx context.Context, taskID, proverPublicKey string) (*ProverTask, error) {
+// GetProverTaskByTaskIDAndProver get prover task taskID and public key
+func (o *ProverTask) GetProverTaskByTaskIDAndProver(ctx context.Context, taskID, proverPublicKey, proverVersion string) (*ProverTask, error) {
 	db := o.db.WithContext(ctx)
 	db = db.Model(&ProverTask{})
-	db = db.Where("task_id", taskID).Where("prover_public_key", proverPublicKey)
+	db = db.Where("task_id", taskID)
+	db = db.Where("prover_public_key", proverPublicKey)
+	db = db.Where("prover_version", proverVersion)
 
 	var proverTask ProverTask
 	err := db.First(&proverTask).Error
 	if err != nil {
-		return nil, fmt.Errorf("ProverTask.GetProverTaskByTaskIDAndPubKey err:%w, taskID:%s, pubukey:%s", err, taskID, proverPublicKey)
+		return nil, fmt.Errorf("ProverTask.GetProverTaskByTaskIDAndProver err:%w, taskID:%s, pubkey:%s, prover_version:%s", err, taskID, proverPublicKey, proverVersion)
 	}
 	return &proverTask, nil
 }
@@ -142,10 +144,11 @@ func (o *ProverTask) GetProvingStatusByTaskID(ctx context.Context, taskID string
 }
 
 // GetTimeoutAssignedProverTasks get the timeout and assigned proving_status prover task
-func (o *ProverTask) GetTimeoutAssignedProverTasks(ctx context.Context, limit int, timeout time.Duration) ([]ProverTask, error) {
+func (o *ProverTask) GetTimeoutAssignedProverTasks(ctx context.Context, limit int, taskType message.ProofType, timeout time.Duration) ([]ProverTask, error) {
 	db := o.db.WithContext(ctx)
 	db = db.Model(&ProverTask{})
 	db = db.Where("proving_status", int(types.ProverAssigned))
+	db = db.Where("task_type", int(taskType))
 	db = db.Where("assigned_at < ?", utils.NowUTC().Add(-timeout))
 	db = db.Limit(limit)
 
@@ -157,6 +160,25 @@ func (o *ProverTask) GetTimeoutAssignedProverTasks(ctx context.Context, limit in
 	return proverTasks, nil
 }
 
+// TaskTimeoutMoreThanOnce get the timeout twice task. a temp design
+func (o *ProverTask) TaskTimeoutMoreThanOnce(ctx context.Context, taskID string) bool {
+	db := o.db.WithContext(ctx)
+	db = db.Model(&ProverTask{})
+	db = db.Where("task_id", taskID)
+	db = db.Where("proving_status", int(types.ProverProofInvalid))
+
+	var count int64
+	if err := db.Count(&count).Error; err != nil {
+		return true
+	}
+
+	if count >= 1 {
+		return true
+	}
+
+	return false
+}
+
 // SetProverTask updates or inserts a ProverTask record.
 func (o *ProverTask) SetProverTask(ctx context.Context, proverTask *ProverTask, dbTX ...*gorm.DB) error {
 	db := o.db.WithContext(ctx)

coordinator/internal/types/submit_proof.go

@@ -2,8 +2,10 @@ package types
 
 // SubmitProofParameter the SubmitProof api request parameter
 type SubmitProofParameter struct {
-	TaskID   string `form:"task_id" json:"task_id" binding:"required"`
-	TaskType int    `form:"task_type" json:"task_type" binding:"required"`
-	Status   int    `form:"status" json:"status"`
-	Proof    string `form:"proof" json:"proof"`
+	TaskID      string `form:"task_id" json:"task_id" binding:"required"`
+	TaskType    int    `form:"task_type" json:"task_type" binding:"required"`
+	Status      int    `form:"status" json:"status"`
+	Proof       string `form:"proof" json:"proof"`
+	FailureType int    `form:"failure_type" json:"failure_type"`
+	FailureMsg  string `form:"failure_msg" json:"failure_msg"`
 }

coordinator/test/api_test.go

@@ -76,11 +76,12 @@ func setupCoordinator(t *testing.T, proversPerSession uint8, coordinatorURL stri
 			ChainID: 111,
 		},
 		ProverManager: &config.ProverManager{
-			ProversPerSession:  proversPerSession,
-			Verifier:           &config.VerifierConfig{MockMode: true},
-			CollectionTimeSec:  10,
-			MaxVerifierWorkers: 10,
-			SessionAttempts:    5,
+			ProversPerSession:      proversPerSession,
+			Verifier:               &config.VerifierConfig{MockMode: true},
+			BatchCollectionTimeSec: 10,
+			ChunkCollectionTimeSec: 10,
+			MaxVerifierWorkers:     10,
+			SessionAttempts:        5,
 		},
 		Auth: &config.Auth{
 			ChallengeExpireDurationSec: tokenTimeout,
@@ -316,7 +317,7 @@ func testInvalidProof(t *testing.T) {
 			assert.NoError(t, err)
 			batchProofStatus, err = batchOrm.GetProvingStatusByHash(context.Background(), batch.Hash)
 			assert.NoError(t, err)
-			if chunkProofStatus == types.ProvingTaskFailed && batchProofStatus == types.ProvingTaskFailed {
+			if chunkProofStatus == types.ProvingTaskUnassigned && batchProofStatus == types.ProvingTaskUnassigned {
 				return
 			}
 		case <-tickStop:
@@ -438,7 +439,7 @@ func testTimeoutProof(t *testing.T) {
 	assert.Equal(t, batchProofStatus, types.ProvingTaskAssigned)
 
 	// wait coordinator to reset the prover task proving status
-	time.Sleep(time.Duration(conf.ProverManager.CollectionTimeSec*2) * time.Second)
+	time.Sleep(time.Duration(conf.ProverManager.BatchCollectionTimeSec*2) * time.Second)
 
 	// create second mock prover, that will send valid proof.
 	chunkProver2 := newMockProver(t, "prover_test"+strconv.Itoa(2), coordinatorURL, message.ProofTypeChunk)

go.work.sum

@@ -193,7 +193,7 @@ github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/crate-crypto/go-ipa v0.0.0-20220523130400-f11357ae11c7 h1:6IrxszG5G+O7zhtkWxq6+unVvnrm1fqV2Pe+T95DUzw=
 github.com/crate-crypto/go-ipa v0.0.0-20220523130400-f11357ae11c7/go.mod h1:gFnFS95y8HstDP6P9pPwzrxOOC5TRDkwbM+ao15ChAI=
-github.com/crate-crypto/go-ipa v0.0.0-20230601170251-1830d0757c80 h1:DuBDHVjgGMPki7bAyh91+3cF1Vh34sAEdH8JQgbc2R0=
+github.com/crate-crypto/go-ipa v0.0.0-20230601170251-1830d0757c80/go.mod h1:gzbVz57IDJgQ9rLQwfSk696JGWof8ftznEL9GoAv3NI=
 github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
 github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
 github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
@@ -238,7 +238,7 @@ github.com/dop251/goja v0.0.0-20220405120441-9037c2b61cbf h1:Yt+4K30SdjOkRoRRm3v
 github.com/dop251/goja v0.0.0-20220405120441-9037c2b61cbf/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk=
 github.com/dop251/goja v0.0.0-20230122112309-96b1610dd4f7 h1:kgvzE5wLsLa7XKfV85VZl40QXaMCaeFtHpPwJ8fhotY=
 github.com/dop251/goja v0.0.0-20230122112309-96b1610dd4f7/go.mod h1:yRkwfj0CBpOGre+TwBsqPV0IH0Pk73e4PXJOeNDboGs=
-github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3 h1:+3HCtB74++ClLy8GgjUQYeC8R4ILzVcIe8+5edAJJnE=
+github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4=
 github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7 h1:tYwu/z8Y0NkkzGEh3z21mSWggMg4LwLRFucLS7TjARg=
 github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y=
 github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d h1:W1n4DvpzZGOISgp7wWNtraLcHtnmnTwBlJidqtMIuwQ=
@@ -277,7 +277,7 @@ github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61/go.mod h1:Q0X6pkwTILD
 github.com/gavv/httpexpect v2.0.0+incompatible h1:1X9kcRshkSKEjNJJxX9Y9mQ5BRfbxU5kORdjhlA1yX8=
 github.com/gballet/go-verkle v0.0.0-20220902153445-097bd83b7732 h1:AB7YjNrzlVHsYz06zCULVV2zYCEft82P86dSmtwxKL0=
 github.com/gballet/go-verkle v0.0.0-20220902153445-097bd83b7732/go.mod h1:o/XfIXWi4/GqbQirfRm5uTbXMG5NpqxkxblnbZ+QM9I=
-github.com/gballet/go-verkle v0.0.0-20230607174250-df487255f46b h1:vMT47RYsrftsHSTQhqXwC3BYflo38OLC3Y4LtXtLyU0=
+github.com/gballet/go-verkle v0.0.0-20230607174250-df487255f46b/go.mod h1:CDncRYVRSDqwakm282WEkjfaAj1hxU/v5RXxk5nXOiI=
 github.com/getkin/kin-openapi v0.61.0 h1:6awGqF5nG5zkVpMsAih1QH4VgzS8phTxECUWIFo7zko=
 github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9 h1:r5GgOLGbza2wVHRzK7aAj6lWZjfbAwiu/RDCVOKjRyM=
 github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs=
@@ -301,6 +301,7 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb
 github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU=
 github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
 github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab h1:xveKWz2iaueeTaUgdetzel+U7exyigDYBryyVfV/rZk=
@@ -373,7 +374,7 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99 h1:Ak8CrdlwwXwAZxzS66vgPt4U8yUZX7JwLvVR58FN5jM=
 github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20230207041349-798e818bf904 h1:4/hN5RUoecvl+RmJRE2YxKWtnnQls6rQjjW5oV7qg2U=
+github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg=
 github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
@@ -464,13 +465,14 @@ github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o=
 github.com/jsternberg/zap-logfmt v1.0.0 h1:0Dz2s/eturmdUS34GM82JwNEdQ9hPoJgqptcEKcbpzY=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5 h1:PJr+ZMXIecYc1Ey2zucXdR73SMBtgjPgwa31099IMv0=
 github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef h1:2jNeR4YUziVtswNP9sEFAI913cVrzH85T+8Q6LpYbT0=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM=
 github.com/karalabe/usb v0.0.0-20211005121534-4c5740d64559/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
 github.com/karalabe/usb v0.0.2 h1:M6QQBNxF+CQ8OFvxrT90BA0qBOXymndZnk5q235mFc4=
 github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
-github.com/karalabe/usb v0.0.3-0.20230711191512-61db3e06439c h1:AqsttAyEyIEsNz5WLRwuRwjiT5CMDUfLk6cFJDVPebs=
+github.com/karalabe/usb v0.0.3-0.20230711191512-61db3e06439c/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
 github.com/kataras/blocks v0.0.7 h1:cF3RDY/vxnSRezc7vLFlQFTYXG/yAr1o7WImJuZbzC4=
 github.com/kataras/blocks v0.0.7/go.mod h1:UJIU97CluDo0f+zEjbnbkeMRlvYORtmc1304EeyXf4I=
 github.com/kataras/golog v0.1.7 h1:0TY5tHn5L5DlRIikepcaRR/6oInIr9AiWsxzt0vvlBE=
@@ -709,7 +711,7 @@ go.opentelemetry.io/otel v1.9.0/go.mod h1:np4EoPGzoPs3O67xUVNoPPcmSvsfOxNlNA4F4A
 go.opentelemetry.io/otel/trace v1.9.0 h1:oZaCNJUjWcg60VXWee8lJKlqhPbXAPB51URuR47pQYc=
 go.opentelemetry.io/otel/trace v1.9.0/go.mod h1:2737Q0MuG8q1uILYm2YYVkAyLtOofiTNGg6VODnOiPo=
 go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4=
-go.uber.org/automaxprocs v1.5.2 h1:2LxUOGiR3O6tw8ui5sZa2LAaHnsviZdVOUZw4fvbnME=
+go.uber.org/automaxprocs v1.5.2/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0=
 go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
 go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o=
 golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -780,7 +782,7 @@ golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
 golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
-golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0=
+golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=

prover/client/types.go

@@ -53,10 +53,12 @@ type GetTaskResponse struct {
 
 // SubmitProofRequest defines the request structure for the SubmitProof API.
 type SubmitProofRequest struct {
-	TaskID   string `json:"task_id"`
-	TaskType int    `json:"task_type"`
-	Status   int    `json:"status"`
-	Proof    string `json:"proof"`
+	TaskID      string `json:"task_id"`
+	TaskType    int    `json:"task_type"`
+	Status      int    `json:"status"`
+	Proof       string `json:"proof"`
+	FailureType int    `json:"failure_type,omitempty"`
+	FailureMsg  string `json:"failure_msg,omitempty"`
 }
 
 // SubmitProofResponse defines the response structure for the SubmitProof API.

prover/prover.go

@@ -151,13 +151,6 @@ func (r *Prover) proveAndSubmit() error {
 		}
 	}
 
-	defer func() {
-		err = r.stack.Delete(task.Task.ID)
-		if err != nil {
-			log.Error("prover stack pop failed!", "err", err)
-		}
-	}()
-
 	var proofMsg *message.ProofDetail
 	if task.Times <= 2 {
 		// If panic times <= 2, try to proof the task.
@@ -168,7 +161,8 @@ func (r *Prover) proveAndSubmit() error {
 		log.Info("start to prove task", "task-type", task.Task.Type, "task-id", task.Task.ID)
 		proofMsg, err = r.prove(task)
 		if err != nil { // handling error from prove
-			return fmt.Errorf("failed to prove task, task-type: %v, err: %v", task.Task.Type, err)
+			log.Error("failed to prove task", "task_type", task.Task.Type, "task-id", task.Task.ID, "err", err)
+			return r.submitErr(task, message.ProofFailureNoPanic, err)
 		}
 
 		return r.submitProof(proofMsg)
@@ -176,7 +170,11 @@ func (r *Prover) proveAndSubmit() error {
 
 	// when the prover has more than 3 times panic,
 	// it will omit to prove the task, submit StatusProofError and then Delete the task.
-	return fmt.Errorf("zk proving panic for task, task-type: %v, task-id: %v", task.Task.Type, task.Task.ID)
+	if err = r.stack.Delete(task.Task.ID); err != nil {
+		log.Error("prover stack pop failed", "task_type", task.Task.Type, "task_id", task.Task.ID, "err", err)
+	}
+	log.Error("zk proving panic for task", "task-type", task.Task.Type, "task-id", task.Task.ID)
+	return r.submitErr(task, message.ProofFailurePanic, errors.New("zk proving panic for task"))
 }
 
 // fetchTaskFromCoordinator fetches a new task from the server
@@ -336,6 +334,27 @@ func (r *Prover) submitProof(msg *message.ProofDetail) error {
 	return nil
 }
 
+func (r *Prover) submitErr(task *store.ProvingTask, proofFailureType message.ProofFailureType, err error) error {
+	// prepare the submit request
+	req := &client.SubmitProofRequest{
+		TaskID:      task.Task.ID,
+		TaskType:    int(task.Task.Type),
+		Status:      int(message.StatusProofError),
+		Proof:       "",
+		FailureType: int(proofFailureType),
+		FailureMsg:  err.Error(),
+	}
+
+	// send the submit request
+	if submitErr := r.coordinatorClient.SubmitProof(r.ctx, req); submitErr != nil {
+		return fmt.Errorf("error submitting proof: %v", submitErr)
+	}
+
+	log.Info("proof submitted report failure successfully", "task-id", task.Task.ID, "task-type",
+		task.Task.Type, "task-status", message.StatusProofError, "err", err)
+	return nil
+}
+
 func (r *Prover) getSortedTracesByHashes(blockHashes []common.Hash) ([]*types.BlockTrace, error) {
 	if len(blockHashes) == 0 {
 		return nil, fmt.Errorf("blockHashes is empty")