add log to coordiantor log check (#1473)

Co-authored-by: georgehao <georgehao@users.noreply.github.com>
Co-authored-by: colinlyguo <colinlyguo@scroll.io>

README.md

@@ -18,7 +18,7 @@
 ├── <a href="./database">database</a>: Database client and schema definition
 ├── <a href="./prover">prover</a>: Prover client that runs proof generation for zkEVM circuit and aggregation circuit
 ├── <a href="./rollup">rollup</a>: Rollup-related services
-├── <a href="./scroll-contracts">scroll-contracts</a>: solidity code for Scroll L1 bridge and rollup contracts and L2 bridge and pre-deployed contracts.
+├── <a href="https://github.com/scroll-tech/scroll-contracts.git">scroll-contracts</a>: solidity code for Scroll L1 bridge and rollup contracts and L2 bridge and pre-deployed contracts.
 └── <a href="./tests">tests</a>: Integration tests
 </pre>
 

common/version/version.go

@@ -5,7 +5,7 @@ import (
 	"runtime/debug"
 )
 
-var tag = "v4.4.37"
+var tag = "v4.4.41"
 
 var commit = func() string {
 	if info, ok := debug.ReadBuildInfo(); ok {

coordinator/internal/controller/api/auth.go

@@ -6,6 +6,7 @@ import (
 
 	jwt "github.com/appleboy/gin-jwt/v2"
 	"github.com/gin-gonic/gin"
+	"github.com/scroll-tech/go-ethereum/log"
 	"gorm.io/gorm"
 
 	"scroll-tech/coordinator/internal/config"
@@ -58,8 +59,19 @@ func (a *AuthController) PayloadFunc(data interface{}) jwt.MapClaims {
 		return jwt.MapClaims{}
 	}
 
+	publicKey := v.PublicKey
+	if publicKey == "" {
+		var err error
+		publicKey, err = v.RecoverPublicKeyFromSignature()
+		if err != nil {
+			// do not handle error here since already called v.Verify() beforehands so there should be no error
+			// add log just in case some error happens
+			log.Error("impossible path: failed to recover public key from signature", "error", err.Error())
+		}
+	}
+
 	return jwt.MapClaims{
-		types.PublicKey:     v.PublicKey,
+		types.PublicKey:     publicKey,
 		types.ProverName:    v.Message.ProverName,
 		types.ProverVersion: v.Message.ProverVersion,
 	}

coordinator/internal/logic/auth/login.go

@@ -59,6 +59,8 @@ func (l *LoginLogic) Check(login *types.LoginParameter) error {
 	if login.PublicKey != "" {
 		verify, err := login.Verify()
 		if err != nil || !verify {
+			log.Error("auth message verify failure", "prover_name", login.Message.ProverName,
+				"prover_version", login.Message.ProverVersion, "message", login.Message)
 			return errors.New("auth message verify failure")
 		}
 	}
@@ -84,13 +86,14 @@ func (l *LoginLogic) Check(login *types.LoginParameter) error {
 					vks[vk] = struct{}{}
 				}
 			default:
-				log.Error("invalid prover_type", "value", proverType)
+				log.Error("invalid prover_type", "value", proverType, "prover name", login.Message.ProverName, "prover_version", login.Message.ProverVersion)
 			}
 		}
 
 		for _, vk := range login.Message.VKs {
 			if _, ok := vks[vk]; !ok {
-				log.Error("vk inconsistency", "prover vk", vk)
+				log.Error("vk inconsistency", "prover vk", vk, "prover name", login.Message.ProverName,
+					"prover_version", login.Message.ProverVersion, "message", login.Message)
 				if !version.CheckScrollProverVersion(login.Message.ProverVersion) {
 					return fmt.Errorf("incompatible prover version. please upgrade your prover, expect version: %s, actual version: %s",
 						version.Version, login.Message.ProverVersion)

coordinator/internal/types/auth.go

@@ -77,6 +77,22 @@ func (a *LoginParameter) Verify() (bool, error) {
 	return isValid, nil
 }
 
+// RecoverPublicKeyFromSignature get public key from signature.
+// This method is for pre-darwin's compatible.
+func (a *LoginParameter) RecoverPublicKeyFromSignature() (string, error) {
+	hash, err := a.Message.Hash()
+	if err != nil {
+		return "", err
+	}
+	sig := common.FromHex(a.Signature)
+	// recover public key
+	pk, err := crypto.SigToPub(hash, sig)
+	if err != nil {
+		return "", err
+	}
+	return common.Bytes2Hex(crypto.CompressPubkey(pk)), nil
+}
+
 // Hash returns the hash of the auth message, which should be the message used
 // to construct the Signature.
 func (i *Message) Hash() ([]byte, error) {

rollup/conf/config.json

@@ -41,7 +41,8 @@
         "max_blob_gas_price": 10000000000000,
         "tx_type": "DynamicFeeTx",
         "check_pending_time": 1,
-        "min_gas_tip": 100000000
+        "min_gas_tip": 100000000,
+        "max_pending_blob_txs": 3
       },
       "gas_oracle_config": {
         "min_gas_price": 0,

rollup/internal/config/relayer.go

@@ -32,6 +32,8 @@ type SenderConfig struct {
 	MaxBlobGasPrice uint64 `json:"max_blob_gas_price"`
 	// The transaction type to use: LegacyTx, DynamicFeeTx, BlobTx
 	TxType string `json:"tx_type"`
+	// The maximum number of pending blob-carrying transactions
+	MaxPendingBlobTxs int64 `json:"max_pending_blob_txs"`
 }
 
 // ChainMonitor this config is used to get batch status from chain_monitor API.

rollup/internal/controller/relayer/l2_relayer.go

@@ -414,6 +414,15 @@ func (r *Layer2Relayer) ProcessPendingBatches() {
 
 		txHash, err := r.commitSender.SendTransaction(dbBatch.Hash, &r.cfg.RollupContractAddress, calldata, blob, fallbackGasLimit)
 		if err != nil {
+			if errors.Is(err, sender.ErrTooManyPendingBlobTxs) {
+				r.metrics.rollupL2RelayerProcessPendingBatchErrTooManyPendingBlobTxsTotal.Inc()
+				log.Debug(
+					"Skipped sending commitBatch tx to L1: too many pending blob txs",
+					"maxPending", r.cfg.SenderConfig.MaxPendingBlobTxs,
+					"err", err,
+				)
+				return
+			}
 			log.Error(
 				"Failed to send commitBatch tx to layer1",
 				"index", dbBatch.Index,

rollup/internal/controller/relayer/l2_relayer_metrics.go

@@ -8,26 +8,27 @@ import (
 )
 
 type l2RelayerMetrics struct {
-	rollupL2RelayerProcessPendingBatchTotal                     prometheus.Counter
-	rollupL2RelayerProcessPendingBatchSuccessTotal              prometheus.Counter
-	rollupL2RelayerGasPriceOraclerRunTotal                      prometheus.Counter
-	rollupL2RelayerLastGasPrice                                 prometheus.Gauge
-	rollupL2RelayerProcessCommittedBatchesTotal                 prometheus.Counter
-	rollupL2RelayerProcessCommittedBatchesFinalizedTotal        prometheus.Counter
-	rollupL2RelayerProcessCommittedBatchesFinalizedSuccessTotal prometheus.Counter
-	rollupL2BatchesCommittedConfirmedTotal                      prometheus.Counter
-	rollupL2BatchesCommittedConfirmedFailedTotal                prometheus.Counter
-	rollupL2BatchesFinalizedConfirmedTotal                      prometheus.Counter
-	rollupL2BatchesFinalizedConfirmedFailedTotal                prometheus.Counter
-	rollupL2UpdateGasOracleConfirmedTotal                       prometheus.Counter
-	rollupL2UpdateGasOracleConfirmedFailedTotal                 prometheus.Counter
-	rollupL2ChainMonitorLatestFailedCall                        prometheus.Counter
-	rollupL2ChainMonitorLatestFailedBatchStatus                 prometheus.Counter
-	rollupL2RelayerProcessPendingBundlesTotal                   prometheus.Counter
-	rollupL2RelayerProcessPendingBundlesFinalizedTotal          prometheus.Counter
-	rollupL2RelayerProcessPendingBundlesFinalizedSuccessTotal   prometheus.Counter
-	rollupL2BundlesFinalizedConfirmedTotal                      prometheus.Counter
-	rollupL2BundlesFinalizedConfirmedFailedTotal                prometheus.Counter
+	rollupL2RelayerProcessPendingBatchTotal                         prometheus.Counter
+	rollupL2RelayerProcessPendingBatchSuccessTotal                  prometheus.Counter
+	rollupL2RelayerProcessPendingBatchErrTooManyPendingBlobTxsTotal prometheus.Counter
+	rollupL2RelayerGasPriceOraclerRunTotal                          prometheus.Counter
+	rollupL2RelayerLastGasPrice                                     prometheus.Gauge
+	rollupL2RelayerProcessCommittedBatchesTotal                     prometheus.Counter
+	rollupL2RelayerProcessCommittedBatchesFinalizedTotal            prometheus.Counter
+	rollupL2RelayerProcessCommittedBatchesFinalizedSuccessTotal     prometheus.Counter
+	rollupL2BatchesCommittedConfirmedTotal                          prometheus.Counter
+	rollupL2BatchesCommittedConfirmedFailedTotal                    prometheus.Counter
+	rollupL2BatchesFinalizedConfirmedTotal                          prometheus.Counter
+	rollupL2BatchesFinalizedConfirmedFailedTotal                    prometheus.Counter
+	rollupL2UpdateGasOracleConfirmedTotal                           prometheus.Counter
+	rollupL2UpdateGasOracleConfirmedFailedTotal                     prometheus.Counter
+	rollupL2ChainMonitorLatestFailedCall                            prometheus.Counter
+	rollupL2ChainMonitorLatestFailedBatchStatus                     prometheus.Counter
+	rollupL2RelayerProcessPendingBundlesTotal                       prometheus.Counter
+	rollupL2RelayerProcessPendingBundlesFinalizedTotal              prometheus.Counter
+	rollupL2RelayerProcessPendingBundlesFinalizedSuccessTotal       prometheus.Counter
+	rollupL2BundlesFinalizedConfirmedTotal                          prometheus.Counter
+	rollupL2BundlesFinalizedConfirmedFailedTotal                    prometheus.Counter
 }
 
 var (
@@ -46,6 +47,10 @@ func initL2RelayerMetrics(reg prometheus.Registerer) *l2RelayerMetrics {
 				Name: "rollup_layer2_process_pending_batch_success_total",
 				Help: "The total number of layer2 process pending success batch",
 			}),
+			rollupL2RelayerProcessPendingBatchErrTooManyPendingBlobTxsTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
+				Name: "rollup_layer2_process_pending_batch_err_too_many_pending_blob_txs_total",
+				Help: "The total number of layer2 process pending batch failed on too many pending blob txs",
+			}),
 			rollupL2RelayerGasPriceOraclerRunTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
 				Name: "rollup_layer2_gas_price_oracler_total",
 				Help: "The total number of layer2 gas price oracler run total",

rollup/internal/controller/sender/sender.go

@@ -39,6 +39,11 @@ const (
 	DynamicFeeTxType = "DynamicFeeTx"
 )
 
+var (
+	// ErrTooManyPendingBlobTxs
+	ErrTooManyPendingBlobTxs = errors.New("the limit of pending blob-carrying transactions has been exceeded")
+)
+
 // Confirmation struct used to indicate transaction confirmation details
 type Confirmation struct {
 	ContextID    string
@@ -180,6 +185,23 @@ func (s *Sender) SendTransaction(contextID string, target *common.Address, data
 	)
 
 	if blob != nil {
+		// check that number of pending blob-carrying txs is not too big
+		if s.senderType == types.SenderTypeCommitBatch {
+			var numPendingTransactions int64
+			// We should count here only blob-carrying txs, but due to check that blob != nil, we know that we already switched to blobs.
+			// Now all txs with SenderTypeCommitBatch will be blob-carrying, but some of previous pending txs could still be non-blob.
+			// But this can happen only once at the moment of switching from non-blob to blob (pre-Bernoulli and post-Bernoulli) and it doesn't break anything.
+			// So don't need to add check that tx carries blob
+			numPendingTransactions, err = s.pendingTransactionOrm.GetCountPendingTransactionsBySenderType(s.ctx, s.senderType)
+			if err != nil {
+				log.Error("failed to count pending transactions", "err: %w", err)
+				return common.Hash{}, fmt.Errorf("failed to count pending transactions, err: %w", err)
+			}
+			if numPendingTransactions >= s.config.MaxPendingBlobTxs {
+				return common.Hash{}, ErrTooManyPendingBlobTxs
+			}
+
+		}
 		sidecar, err = makeSidecar(blob)
 		if err != nil {
 			log.Error("failed to make sidecar for blob transaction", "error", err)
@@ -518,6 +540,22 @@ func (s *Sender) checkPendingTransaction() {
 			}
 		} else if txnToCheck.Status == types.TxStatusPending && // Only try resubmitting a new transaction based on gas price of the last transaction (status pending) with same ContextID.
 			s.config.EscalateBlocks+txnToCheck.SubmitBlockNumber <= blockNumber {
+
+			// blockNumber is the block number with "latest" tag, so we need to check the current nonce of the sender address to ensure that the previous transaction has been confirmed.
+			// otherwise it's not very necessary to bump the gas price. Also worth noting is that, during bumping gas prices, the sender would consider the new basefee and blobbasefee of L1.
+			currentNonce, err := s.client.NonceAt(s.ctx, common.HexToAddress(txnToCheck.SenderAddress), new(big.Int).SetUint64(blockNumber))
+			if err != nil {
+				log.Error("failed to get current nonce from node", "address", txnToCheck.SenderAddress, "blockNumber", blockNumber, "err", err)
+				return
+			}
+
+			// early return if the previous transaction has not been confirmed yet.
+			// currentNonce is already the confirmed nonce + 1.
+			if tx.Nonce() > currentNonce {
+				log.Debug("previous transaction not yet confirmed, skip bumping gas price", "address", txnToCheck.SenderAddress, "currentNonce", currentNonce, "txNonce", tx.Nonce())
+				continue
+			}
+
 			// It's possible that the pending transaction was marked as failed earlier in this loop (e.g., if one of its replacements has already been confirmed).
 			// Therefore, we fetch the current transaction status again for accuracy before proceeding.
 			status, err := s.pendingTransactionOrm.GetTxStatusByTxHash(s.ctx, tx.Hash())

rollup/internal/controller/sender/sender_test.go

@@ -6,6 +6,7 @@ import (
 	"crypto/rand"
 	"errors"
 	"fmt"
+	"math"
 	"math/big"
 	"os"
 	"testing"
@@ -120,7 +121,6 @@ func setupEnv(t *testing.T) {
 
 func TestSender(t *testing.T) {
 	setupEnv(t)
-
 	t.Run("test new sender", testNewSender)
 	t.Run("test send and retrieve transaction", testSendAndRetrieveTransaction)
 	t.Run("test fallback gas limit", testFallbackGasLimit)
@@ -130,11 +130,13 @@ func TestSender(t *testing.T) {
 	t.Run("test resubmit under priced transaction", testResubmitUnderpricedTransaction)
 	t.Run("test resubmit dynamic fee transaction with rising base fee", testResubmitDynamicFeeTransactionWithRisingBaseFee)
 	t.Run("test resubmit blob transaction with rising base fee and blob base fee", testResubmitBlobTransactionWithRisingBaseFeeAndBlobBaseFee)
+	t.Run("test resubmit nonce gapped transaction", testResubmitNonceGappedTransaction)
 	t.Run("test check pending transaction tx confirmed", testCheckPendingTransactionTxConfirmed)
 	t.Run("test check pending transaction resubmit tx confirmed", testCheckPendingTransactionResubmitTxConfirmed)
 	t.Run("test check pending transaction replaced tx confirmed", testCheckPendingTransactionReplacedTxConfirmed)
 	t.Run("test check pending transaction multiple times with only one transaction pending", testCheckPendingTransactionTxMultipleTimesWithOnlyOneTxPending)
 	t.Run("test blob transaction with blobhash op contract call", testBlobTransactionWithBlobhashOpContractCall)
+	t.Run("test test send blob-carrying tx over limit", testSendBlobCarryingTxOverLimit)
 }
 
 func testNewSender(t *testing.T) {
@@ -517,6 +519,67 @@ func testResubmitBlobTransactionWithRisingBaseFeeAndBlobBaseFee(t *testing.T) {
 	s.Stop()
 }
 
+func testResubmitNonceGappedTransaction(t *testing.T) {
+	for i, txType := range txTypes {
+		sqlDB, err := db.DB()
+		assert.NoError(t, err)
+		assert.NoError(t, migrate.ResetDB(sqlDB))
+
+		cfgCopy := *cfg.L2Config.RelayerConfig.SenderConfig
+
+		// Bump gas price, gas tip cap and gas fee cap just touch the minimum threshold of 10% (default config of geth).
+		cfgCopy.EscalateMultipleNum = 110
+		cfgCopy.EscalateMultipleDen = 100
+		cfgCopy.TxType = txType
+
+		// resubmit immediately if not nonce gapped
+		cfgCopy.Confirmations = rpc.LatestBlockNumber
+		cfgCopy.EscalateBlocks = 0
+
+		// stop background check pending transaction
+		cfgCopy.CheckPendingTime = math.MaxUint32
+
+		s, err := NewSender(context.Background(), &cfgCopy, privateKey, "test", "test", types.SenderTypeUnknown, db, nil)
+		assert.NoError(t, err)
+
+		patchGuard1 := gomonkey.ApplyMethodFunc(s.client, "SendTransaction", func(_ context.Context, _ *gethTypes.Transaction) error {
+			return nil
+		})
+
+		// simulating not confirmed transaction
+		patchGuard2 := gomonkey.ApplyMethodFunc(s.client, "TransactionReceipt", func(_ context.Context, hash common.Hash) (*gethTypes.Receipt, error) {
+			return nil, errors.New("simulated transaction receipt error")
+		})
+
+		_, err = s.SendTransaction("test-1", &common.Address{}, nil, txBlob[i], 0)
+		assert.NoError(t, err)
+
+		_, err = s.SendTransaction("test-2", &common.Address{}, nil, txBlob[i], 0)
+		assert.NoError(t, err)
+
+		s.checkPendingTransaction()
+
+		txs, err := s.pendingTransactionOrm.GetPendingOrReplacedTransactionsBySenderType(context.Background(), s.senderType, 10)
+		assert.NoError(t, err)
+		assert.Len(t, txs, 3)
+
+		assert.Equal(t, txs[0].Nonce, txs[1].Nonce)
+		assert.Equal(t, txs[0].Nonce+1, txs[2].Nonce)
+
+		// the first 2 transactions have the same nonce, with one replaced and another pending
+		assert.Equal(t, types.TxStatusReplaced, txs[0].Status)
+		assert.Equal(t, types.TxStatusPending, txs[1].Status)
+
+		// the third transaction has nonce + 1, which will not be replaced due to the nonce gap,
+		// thus the status should be pending
+		assert.Equal(t, types.TxStatusPending, txs[2].Status)
+
+		s.Stop()
+		patchGuard1.Reset()
+		patchGuard2.Reset()
+	}
+}
+
 func testCheckPendingTransactionTxConfirmed(t *testing.T) {
 	for _, txType := range txTypes {
 		sqlDB, err := db.DB()
@@ -818,3 +881,22 @@ func randFieldElement() [32]byte {
 
 	return gokzg4844.SerializeScalar(r)
 }
+
+func testSendBlobCarryingTxOverLimit(t *testing.T) {
+	cfgCopy := *cfg.L2Config.RelayerConfig.SenderConfig
+	cfgCopy.TxType = "DynamicFeeTx"
+
+	sqlDB, err := db.DB()
+	assert.NoError(t, err)
+	assert.NoError(t, migrate.ResetDB(sqlDB))
+	s, err := NewSender(context.Background(), &cfgCopy, privateKey, "test", "test", types.SenderTypeCommitBatch, db, nil)
+	assert.NoError(t, err)
+
+	for i := 0; i < int(cfgCopy.MaxPendingBlobTxs); i++ {
+		_, err = s.SendTransaction("0", &common.Address{}, nil, randBlob(), 0)
+		assert.NoError(t, err)
+	}
+	_, err = s.SendTransaction("0", &common.Address{}, nil, randBlob(), 0)
+	assert.ErrorIs(t, err, ErrTooManyPendingBlobTxs)
+	s.Stop()
+}

rollup/internal/orm/pending_transaction.go

@@ -85,6 +85,19 @@ func (o *PendingTransaction) GetPendingOrReplacedTransactionsBySenderType(ctx co
 	return transactions, nil
 }
 
+// GetCountPendingTransactionsBySenderType retrieves number of pending transactions filtered by sender type
+func (o *PendingTransaction) GetCountPendingTransactionsBySenderType(ctx context.Context, senderType types.SenderType) (int64, error) {
+	var count int64
+	db := o.db.WithContext(ctx)
+	db = db.Model(&PendingTransaction{})
+	db = db.Where("sender_type = ?", senderType)
+	db = db.Where("status = ?", types.TxStatusPending)
+	if err := db.Count(&count).Error; err != nil {
+		return 0, fmt.Errorf("failed to count pending transactions by sender type, error: %w", err)
+	}
+	return count, nil
+}
+
 // GetConfirmedTransactionsBySenderType retrieves confirmed transactions filtered by sender type, limited to a specified count.
 // for unit test
 func (o *PendingTransaction) GetConfirmedTransactionsBySenderType(ctx context.Context, senderType types.SenderType, limit int) ([]PendingTransaction, error) {