feat(rollup-relayer): auto finalize batch when timeout in test env (#1008)

Co-authored-by: HAOYUatHZ <37070449+HAOYUatHZ@users.noreply.github.com>

common/version/version.go

@@ -5,7 +5,7 @@ import (
 	"runtime/debug"
 )
 
-var tag = "v4.3.38"
+var tag = "v4.3.39"
 
 var commit = func() string {
 	if info, ok := debug.ReadBuildInfo(); ok {

rollup/conf/config.json

@@ -24,7 +24,6 @@
         "min_gas_price": 0,
         "gas_price_diff": 50000
       },
-      "finalize_batch_interval_sec": 0,
       "gas_oracle_sender_private_key": "1313131313131313131313131313131313131313131313131313131313131313"
     }
   },
@@ -59,7 +58,8 @@
         "try_times": 5,
         "base_url": "http://localhost:8750"
       },
-      "finalize_batch_interval_sec": 0,
+      "enable_test_env_bypass_features": true,
+      "finalize_batch_without_proof_timeout_sec": 7200,
       "gas_oracle_sender_private_key": "1313131313131313131313131313131313131313131313131313131313131313",
       "commit_sender_private_key": "1414141414141414141414141414141414141414141414141414141414141414",
       "finalize_sender_private_key": "1515151515151515151515151515151515151515151515151515151515151515",

rollup/internal/config/relayer.go

@@ -64,6 +64,11 @@ type RelayerConfig struct {
 	GasOracleSenderPrivateKey *ecdsa.PrivateKey `json:"-"`
 	CommitSenderPrivateKey    *ecdsa.PrivateKey `json:"-"`
 	FinalizeSenderPrivateKey  *ecdsa.PrivateKey `json:"-"`
+
+	// Indicates if bypass features specific to testing environments are enabled.
+	EnableTestEnvBypassFeatures bool `json:"enable_test_env_bypass_features"`
+	// The timeout in seconds for finalizing a batch without proof, only used when EnableTestEnvBypassFeatures is true.
+	FinalizeBatchWithoutProofTimeoutSec uint64 `json:"finalize_batch_without_proof_timeout_sec"`
 }
 
 // GasOracleConfig The config for updating gas price oracle.

rollup/internal/controller/relayer/l1_relayer.go

@@ -50,6 +50,11 @@ func NewLayer1Relayer(ctx context.Context, db *gorm.DB, cfg *config.RelayerConfi
 		return nil, fmt.Errorf("new gas oracle sender failed for address %s, err: %v", addr.Hex(), err)
 	}
 
+	// Ensure test features aren't enabled on the mainnet.
+	if gasOracleSender.GetChainID() == big.NewInt(1) && cfg.EnableTestEnvBypassFeatures {
+		return nil, fmt.Errorf("cannot enable test env features in mainnet")
+	}
+
 	var minGasPrice uint64
 	var gasPriceDiff uint64
 	if cfg.GasOracleConfig != nil {

rollup/internal/controller/relayer/l2_relayer.go

@@ -19,6 +19,7 @@ import (
 	"gorm.io/gorm"
 
 	"scroll-tech/common/types"
+	"scroll-tech/common/utils"
 
 	bridgeAbi "scroll-tech/rollup/abi"
 	"scroll-tech/rollup/internal/config"
@@ -88,6 +89,11 @@ func NewLayer2Relayer(ctx context.Context, l2Client *ethclient.Client, db *gorm.
 		return nil, fmt.Errorf("new gas oracle sender failed for address %s, err: %w", addr.Hex(), err)
 	}
 
+	// Ensure test features aren't enabled on the mainnet.
+	if commitSender.GetChainID() == big.NewInt(1) && cfg.EnableTestEnvBypassFeatures {
+		return nil, fmt.Errorf("cannot enable test env features in mainnet")
+	}
+
 	var minGasPrice uint64
 	var gasPriceDiff uint64
 	if cfg.GasOracleConfig != nil {
@@ -434,108 +440,27 @@ func (r *Layer2Relayer) ProcessCommittedBatches() {
 	r.metrics.rollupL2RelayerProcessCommittedBatchesTotal.Inc()
 
 	batch := batches[0]
-	hash := batch.Hash
 	status := types.ProvingStatus(batch.ProvingStatus)
 	switch status {
 	case types.ProvingTaskUnassigned, types.ProvingTaskAssigned:
-		// The proof for this block is not ready yet.
-		return
+		if batch.CommittedAt == nil {
+			log.Error("batch.CommittedAt is nil", "index", batch.Index, "hash", batch.Hash)
+			return
+		}
+
+		if r.cfg.EnableTestEnvBypassFeatures && utils.NowUTC().Sub(*batch.CommittedAt) > time.Duration(r.cfg.FinalizeBatchWithoutProofTimeoutSec)*time.Second {
+			if err := r.finalizeBatch(batch, false); err != nil {
+				log.Error("Failed to finalize timeout batch without proof", "index", batch.Index, "hash", batch.Hash, "err", err)
+			}
+		}
+
 	case types.ProvingTaskVerified:
-		log.Info("Start to roll up zk proof", "hash", hash)
+		log.Info("Start to roll up zk proof", "hash", batch.Hash)
 		r.metrics.rollupL2RelayerProcessCommittedBatchesFinalizedTotal.Inc()
-
-		// Check batch status before send `finalizeBatchWithProof` tx.
-		if r.cfg.ChainMonitor.Enabled {
-			var batchStatus bool
-			batchStatus, err = r.getBatchStatusByIndex(batch.Index)
-			if err != nil {
-				r.metrics.rollupL2ChainMonitorLatestFailedCall.Inc()
-				log.Warn("failed to get batch status, please check chain_monitor api server", "batch_index", batch.Index, "err", err)
-				return
-			}
-			if !batchStatus {
-				r.metrics.rollupL2ChainMonitorLatestFailedBatchStatus.Inc()
-				log.Error("the batch status is not right, stop finalize batch and check the reason", "batch_index", batch.Index)
-				return
-			}
+		if err := r.finalizeBatch(batch, true); err != nil {
+			log.Error("Failed to finalize batch with proof", "index", batch.Index, "hash", batch.Hash, "err", err)
 		}
 
-		var parentBatchStateRoot string
-		if batch.Index > 0 {
-			var parentBatch *orm.Batch
-			parentBatch, err = r.batchOrm.GetBatchByIndex(r.ctx, batch.Index-1)
-			// handle unexpected db error
-			if err != nil {
-				log.Error("Failed to get batch", "index", batch.Index-1, "err", err)
-				return
-			}
-			parentBatchStateRoot = parentBatch.StateRoot
-		}
-
-		aggProof, err := r.batchOrm.GetVerifiedProofByHash(r.ctx, hash)
-		if err != nil {
-			log.Error("get verified proof by hash failed", "hash", hash, "err", err)
-			return
-		}
-
-		if err = aggProof.SanityCheck(); err != nil {
-			log.Error("agg_proof sanity check fails", "hash", hash, "error", err)
-			return
-		}
-
-		data, err := r.l1RollupABI.Pack(
-			"finalizeBatchWithProof",
-			batch.BatchHeader,
-			common.HexToHash(parentBatchStateRoot),
-			common.HexToHash(batch.StateRoot),
-			common.HexToHash(batch.WithdrawRoot),
-			aggProof.Proof,
-		)
-		if err != nil {
-			log.Error("Pack finalizeBatchWithProof failed", "err", err)
-			return
-		}
-
-		txID := hash + "-finalize"
-		// add suffix `-finalize` to avoid duplication with commit tx in unit tests
-		txHash, err := r.finalizeSender.SendTransaction(txID, &r.cfg.RollupContractAddress, big.NewInt(0), data, 0)
-		finalizeTxHash := &txHash
-		if err != nil {
-			if !errors.Is(err, sender.ErrNoAvailableAccount) && !errors.Is(err, sender.ErrFullPending) {
-				// This can happen normally if we try to finalize 2 or more
-				// batches around the same time. The 2nd tx might fail since
-				// the client does not see the 1st tx's updates at this point.
-				// TODO: add more fine-grained error handling
-				log.Error(
-					"finalizeBatchWithProof in layer1 failed",
-					"index", batch.Index,
-					"hash", batch.Hash,
-					"RollupContractAddress", r.cfg.RollupContractAddress,
-					"err", err,
-				)
-				log.Debug(
-					"finalizeBatchWithProof in layer1 failed",
-					"index", batch.Index,
-					"hash", batch.Hash,
-					"RollupContractAddress", r.cfg.RollupContractAddress,
-					"calldata", common.Bytes2Hex(data),
-					"err", err,
-				)
-			}
-			return
-		}
-		log.Info("finalizeBatchWithProof in layer1", "index", batch.Index, "batch hash", batch.Hash, "tx hash", hash)
-
-		// record and sync with db, @todo handle db error
-		err = r.batchOrm.UpdateFinalizeTxHashAndRollupStatus(r.ctx, hash, finalizeTxHash.String(), types.RollupFinalizing)
-		if err != nil {
-			log.Error("UpdateFinalizeTxHashAndRollupStatus failed",
-				"index", batch.Index, "batch hash", batch.Hash,
-				"tx hash", finalizeTxHash.String(), "err", err)
-		}
-		r.processingFinalization.Store(txID, hash)
-		r.metrics.rollupL2RelayerProcessCommittedBatchesFinalizedSuccessTotal.Inc()
-
 	case types.ProvingTaskFailed:
 		// We were unable to prove this batch. There are two possibilities:
 		// (a) Prover bug. In this case, we should fix and redeploy the prover.
@@ -553,13 +478,121 @@ func (r *Layer2Relayer) ProcessCommittedBatches() {
 			"ProvedAt", batch.ProvedAt,
 			"ProofTimeSec", batch.ProofTimeSec,
 		)
-		return
 
 	default:
 		log.Error("encounter unreachable case in ProcessCommittedBatches", "proving status", status)
 	}
 }
 
+func (r *Layer2Relayer) finalizeBatch(batch *orm.Batch, withProof bool) error {
+	// Check batch status before send `finalizeBatchWithProof` tx.
+	if r.cfg.ChainMonitor.Enabled {
+		var batchStatus bool
+		batchStatus, err := r.getBatchStatusByIndex(batch.Index)
+		if err != nil {
+			r.metrics.rollupL2ChainMonitorLatestFailedCall.Inc()
+			log.Warn("failed to get batch status, please check chain_monitor api server", "batch_index", batch.Index, "err", err)
+			return err
+		}
+		if !batchStatus {
+			r.metrics.rollupL2ChainMonitorLatestFailedBatchStatus.Inc()
+			log.Error("the batch status is not right, stop finalize batch and check the reason", "batch_index", batch.Index)
+			return err
+		}
+	}
+
+	var parentBatchStateRoot string
+	if batch.Index > 0 {
+		var parentBatch *orm.Batch
+		parentBatch, err := r.batchOrm.GetBatchByIndex(r.ctx, batch.Index-1)
+		// handle unexpected db error
+		if err != nil {
+			log.Error("Failed to get batch", "index", batch.Index-1, "err", err)
+			return err
+		}
+		parentBatchStateRoot = parentBatch.StateRoot
+	}
+
+	var txCalldata []byte
+	if withProof {
+		aggProof, err := r.batchOrm.GetVerifiedProofByHash(r.ctx, batch.Hash)
+		if err != nil {
+			log.Error("get verified proof by hash failed", "hash", batch.Hash, "err", err)
+			return err
+		}
+
+		if err = aggProof.SanityCheck(); err != nil {
+			log.Error("agg_proof sanity check fails", "hash", batch.Hash, "error", err)
+			return err
+		}
+
+		txCalldata, err = r.l1RollupABI.Pack(
+			"finalizeBatchWithProof",
+			batch.BatchHeader,
+			common.HexToHash(parentBatchStateRoot),
+			common.HexToHash(batch.StateRoot),
+			common.HexToHash(batch.WithdrawRoot),
+			aggProof.Proof,
+		)
+		if err != nil {
+			log.Error("Pack finalizeBatchWithProof failed", "err", err)
+			return err
+		}
+	} else {
+		var err error
+		txCalldata, err = r.l1RollupABI.Pack(
+			"finalizeBatch",
+			batch.BatchHeader,
+			common.HexToHash(parentBatchStateRoot),
+			common.HexToHash(batch.StateRoot),
+			common.HexToHash(batch.WithdrawRoot),
+		)
+		if err != nil {
+			log.Error("Pack finalizeBatch failed", "err", err)
+			return err
+		}
+	}
+
+	txID := batch.Hash + "-finalize"
+	// add suffix `-finalize` to avoid duplication with commit tx in unit tests
+	txHash, err := r.finalizeSender.SendTransaction(txID, &r.cfg.RollupContractAddress, big.NewInt(0), txCalldata, 0)
+	finalizeTxHash := &txHash
+	if err != nil {
+		if !errors.Is(err, sender.ErrNoAvailableAccount) && !errors.Is(err, sender.ErrFullPending) {
+			// This can happen normally if we try to finalize 2 or more
+			// batches around the same time. The 2nd tx might fail since
+			// the client does not see the 1st tx's updates at this point.
+			// TODO: add more fine-grained error handling
+			log.Error(
+				"finalizeBatchWithProof in layer1 failed",
+				"index", batch.Index,
+				"hash", batch.Hash,
+				"RollupContractAddress", r.cfg.RollupContractAddress,
+				"err", err,
+			)
+			log.Debug(
+				"finalizeBatchWithProof in layer1 failed",
+				"index", batch.Index,
+				"hash", batch.Hash,
+				"RollupContractAddress", r.cfg.RollupContractAddress,
+				"calldata", common.Bytes2Hex(txCalldata),
+				"err", err,
+			)
+		}
+		return err
+	}
+	log.Info("finalizeBatch in layer1", "index", batch.Index, "batch hash", batch.Hash, "tx hash", batch.Hash, "with proof", withProof)
+
+	// record and sync with db, @todo handle db error
+	if err := r.batchOrm.UpdateFinalizeTxHashAndRollupStatus(r.ctx, batch.Hash, finalizeTxHash.String(), types.RollupFinalizing); err != nil {
+		log.Error("UpdateFinalizeTxHashAndRollupStatus failed", "index", batch.Index, "batch hash", batch.Hash, "tx hash", finalizeTxHash.String(), "err", err)
+		return err
+	}
+	r.processingFinalization.Store(txID, batch.Hash)
+	r.metrics.rollupL2RelayerProcessCommittedBatchesFinalizedSuccessTotal.Inc()
+	return nil
+}
+
 // batchStatusResponse the response schema
 type batchStatusResponse struct {
 	ErrCode int    `json:"errcode"`

rollup/internal/controller/relayer/l2_relayer_test.go

@@ -121,6 +121,37 @@ func testL2RelayerProcessCommittedBatches(t *testing.T) {
 	assert.Equal(t, types.RollupFinalizing, statuses[0])
 }
 
+func testL2RelayerFinalizeTimeoutBatches(t *testing.T) {
+	db := setupL2RelayerDB(t)
+	defer database.CloseDB(db)
+
+	l2Cfg := cfg.L2Config
+	l2Cfg.RelayerConfig.EnableTestEnvBypassFeatures = true
+	l2Cfg.RelayerConfig.FinalizeBatchWithoutProofTimeoutSec = 0
+	relayer, err := NewLayer2Relayer(context.Background(), l2Cli, db, l2Cfg.RelayerConfig, false, nil)
+	assert.NoError(t, err)
+	batchMeta := &types.BatchMeta{
+		StartChunkIndex: 0,
+		StartChunkHash:  chunkHash1.Hex(),
+		EndChunkIndex:   1,
+		EndChunkHash:    chunkHash2.Hex(),
+	}
+	batchOrm := orm.NewBatch(db)
+	batch, err := batchOrm.InsertBatch(context.Background(), []*types.Chunk{chunk1, chunk2}, batchMeta)
+	assert.NoError(t, err)
+
+	err = batchOrm.UpdateRollupStatus(context.Background(), batch.Hash, types.RollupCommitted)
+	assert.NoError(t, err)
+
+	// Check the database for the updated status using TryTimes.
+	ok := utils.TryTimes(5, func() bool {
+		relayer.ProcessCommittedBatches()
+		statuses, err := batchOrm.GetRollupStatusByHashList(context.Background(), []string{batch.Hash})
+		return err == nil && len(statuses) == 1 && statuses[0] == types.RollupFinalizing
+	})
+	assert.True(t, ok)
+}
+
 func testL2RelayerCommitConfirm(t *testing.T) {
 	db := setupL2RelayerDB(t)
 	defer database.CloseDB(db)

rollup/internal/controller/relayer/relayer_test.go

@@ -99,6 +99,7 @@ func TestFunctions(t *testing.T) {
 	t.Run("TestCreateNewRelayer", testCreateNewRelayer)
 	t.Run("TestL2RelayerProcessPendingBatches", testL2RelayerProcessPendingBatches)
 	t.Run("TestL2RelayerProcessCommittedBatches", testL2RelayerProcessCommittedBatches)
+	t.Run("TestL2RelayerFinalizeTimeoutBatches", testL2RelayerFinalizeTimeoutBatches)
 	t.Run("TestL2RelayerCommitConfirm", testL2RelayerCommitConfirm)
 	t.Run("TestL2RelayerFinalizeConfirm", testL2RelayerFinalizeConfirm)
 	t.Run("TestL2RelayerGasOracleConfirm", testL2RelayerGasOracleConfirm)

rollup/internal/controller/sender/sender.go

@@ -165,6 +165,11 @@ func (s *Sender) IsFull() bool {
 	return s.pendingTxs.Count() >= s.config.PendingLimit
 }
 
+// GetChainID returns the chain ID associated with the sender.
+func (s *Sender) GetChainID() *big.Int {
+	return s.chainID
+}
+
 // Stop stop the sender module.
 func (s *Sender) Stop() {
 	close(s.stopCh)

rollup/internal/orm/batch.go

@@ -9,6 +9,7 @@ import (
 
 	"scroll-tech/common/types"
 	"scroll-tech/common/types/message"
+	"scroll-tech/common/utils"
 
 	"github.com/scroll-tech/go-ethereum/common"
 	"github.com/scroll-tech/go-ethereum/log"
@@ -355,9 +356,9 @@ func (o *Batch) UpdateRollupStatus(ctx context.Context, hash string, status type
 
 	switch status {
 	case types.RollupCommitted:
-		updateFields["committed_at"] = time.Now()
+		updateFields["committed_at"] = utils.NowUTC()
 	case types.RollupFinalized:
-		updateFields["finalized_at"] = time.Now()
+		updateFields["finalized_at"] = utils.NowUTC()
 	}
 
 	db := o.db
@@ -380,7 +381,7 @@ func (o *Batch) UpdateCommitTxHashAndRollupStatus(ctx context.Context, hash stri
 	updateFields["commit_tx_hash"] = commitTxHash
 	updateFields["rollup_status"] = int(status)
 	if status == types.RollupCommitted {
-		updateFields["committed_at"] = time.Now()
+		updateFields["committed_at"] = utils.NowUTC()
 	}
 
 	db := o.db.WithContext(ctx)