improve logs

This reverts commit 42190feb6c.

.gitignore

@@ -3,7 +3,6 @@ assets/params*
 assets/seed
 coverage.txt
 build/bin
-*.integration.txt
 
 # misc
 sftp-config.json

bridge/cmd/app/app.go

@@ -1,7 +1,6 @@
 package app
 
 import (
-	"context"
 	"fmt"
 	"os"
 	"os/signal"
@@ -11,7 +10,6 @@ import (
 
 	"scroll-tech/database"
 
-	"scroll-tech/common/metrics"
 	"scroll-tech/common/utils"
 	"scroll-tech/common/version"
 
@@ -51,10 +49,7 @@ func action(ctx *cli.Context) error {
 		log.Crit("failed to load config file", "config file", cfgFile, "error", err)
 	}
 
-	// Start metrics server.
-	metrics.Serve(context.Background(), ctx)
-
-	// Init db connection.
+	// init db connection
 	var ormFactory database.OrmFactory
 	if ormFactory, err = database.NewOrmFactory(cfg.DBConfig); err != nil {
 		log.Crit("failed to init db connection", "err", err)

bridge/l1/backend.go

@@ -26,7 +26,7 @@ func New(ctx context.Context, cfg *config.L1Config, orm database.OrmFactory) (*B
 		return nil, err
 	}
 
-	relayer, err := NewLayer1Relayer(ctx, int64(cfg.Confirmations), orm, cfg.RelayerConfig)
+	relayer, err := NewLayer1Relayer(ctx, client, int64(cfg.Confirmations), orm, cfg.RelayerConfig)
 	if err != nil {
 		return nil, err
 	}

bridge/l1/relayer.go

@@ -11,6 +11,7 @@ import (
 	"github.com/scroll-tech/go-ethereum/accounts/abi"
 	"github.com/scroll-tech/go-ethereum/common"
 	"github.com/scroll-tech/go-ethereum/crypto"
+	"github.com/scroll-tech/go-ethereum/ethclient"
 	"github.com/scroll-tech/go-ethereum/log"
 
 	"scroll-tech/database/orm"
@@ -28,6 +29,7 @@ import (
 // @todo It's better to be triggered by watcher.
 type Layer1Relayer struct {
 	ctx    context.Context
+	client *ethclient.Client
 	sender *sender.Sender
 
 	db  orm.L1MessageOrm
@@ -41,7 +43,7 @@ type Layer1Relayer struct {
 }
 
 // NewLayer1Relayer will return a new instance of Layer1RelayerClient
-func NewLayer1Relayer(ctx context.Context, l1ConfirmNum int64, db orm.L1MessageOrm, cfg *config.RelayerConfig) (*Layer1Relayer, error) {
+func NewLayer1Relayer(ctx context.Context, ethClient *ethclient.Client, l1ConfirmNum int64, db orm.L1MessageOrm, cfg *config.RelayerConfig) (*Layer1Relayer, error) {
 	l2MessengerABI, err := bridge_abi.L2MessengerMetaData.GetAbi()
 	if err != nil {
 		log.Warn("new L2MessengerABI failed", "err", err)
@@ -57,6 +59,7 @@ func NewLayer1Relayer(ctx context.Context, l1ConfirmNum int64, db orm.L1MessageO
 
 	return &Layer1Relayer{
 		ctx:            ctx,
+		client:         ethClient,
 		sender:         sender,
 		db:             db,
 		l2MessengerABI: l2MessengerABI,
@@ -110,7 +113,7 @@ func (r *Layer1Relayer) processSavedEvent(msg *orm.L1Message) error {
 		return err
 	}
 
-	hash, err := r.sender.SendTransaction(msg.MsgHash, &r.cfg.MessengerContractAddress, big.NewInt(0), data)
+	hash, err := r.sender.SendTransaction(msg.MsgHash, &r.cfg.MessengerContractAddress, big.NewInt(0), data, 0)
 	if err != nil && err.Error() == "execution reverted: Message expired" {
 		return r.db.UpdateLayer1Status(r.ctx, msg.MsgHash, orm.MsgExpired)
 	}
@@ -131,6 +134,8 @@ func (r *Layer1Relayer) processSavedEvent(msg *orm.L1Message) error {
 
 // Start the relayer process
 func (r *Layer1Relayer) Start() {
+	log.Info("Starting l1/relayer")
+
 	go func() {
 		// trigger by timer
 		ticker := time.NewTicker(3 * time.Second)

bridge/l1/relayer_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"testing"
 
+	"github.com/scroll-tech/go-ethereum/ethclient"
 	"github.com/stretchr/testify/assert"
 
 	"scroll-tech/database/migrate"
@@ -19,7 +20,10 @@ func testCreateNewL1Relayer(t *testing.T) {
 	assert.NoError(t, migrate.ResetDB(db.GetDB().DB))
 	defer db.Close()
 
-	relayer, err := NewLayer1Relayer(context.Background(), 1, db, cfg.L2Config.RelayerConfig)
+	client, err := ethclient.Dial(l1gethImg.Endpoint())
+	assert.NoError(t, err)
+
+	relayer, err := NewLayer1Relayer(context.Background(), client, 1, db, cfg.L2Config.RelayerConfig)
 	assert.NoError(t, err)
 	defer relayer.Stop()
 

bridge/l1/watcher.go

@@ -11,7 +11,6 @@ import (
 	"github.com/scroll-tech/go-ethereum/core/types"
 	"github.com/scroll-tech/go-ethereum/ethclient"
 	"github.com/scroll-tech/go-ethereum/log"
-	"github.com/scroll-tech/go-ethereum/metrics"
 
 	"scroll-tech/database"
 	"scroll-tech/database/orm"
@@ -20,10 +19,6 @@ import (
 	"scroll-tech/bridge/utils"
 )
 
-var (
-	bridgeL1MsgSyncHeightGauge = metrics.NewRegisteredGauge("bridge/l1/msg/sync/height", nil)
-)
-
 type relayedMessage struct {
 	msgHash      common.Hash
 	txHash       common.Hash
@@ -86,6 +81,8 @@ func NewWatcher(ctx context.Context, client *ethclient.Client, startHeight uint6
 
 // Start the Watcher module.
 func (w *Watcher) Start() {
+	log.Info("Starting l1/watcher")
+
 	go func() {
 		ticker := time.NewTicker(10 * time.Second)
 		defer ticker.Stop()
@@ -156,7 +153,6 @@ func (w *Watcher) FetchContractEvent(blockHeight uint64) error {
 		}
 		if len(logs) == 0 {
 			w.processedMsgHeight = uint64(to)
-			bridgeL1MsgSyncHeightGauge.Update(to)
 			continue
 		}
 		log.Info("Received new L1 events", "fromBlock", from, "toBlock", to, "cnt", len(logs))
@@ -221,7 +217,6 @@ func (w *Watcher) FetchContractEvent(blockHeight uint64) error {
 		}
 
 		w.processedMsgHeight = uint64(to)
-		bridgeL1MsgSyncHeightGauge.Update(to)
 	}
 
 	return nil

bridge/l2/relayer.go

@@ -166,7 +166,7 @@ func (r *Layer2Relayer) processSavedEvent(msg *orm.L2Message, index uint64) erro
 		return err
 	}
 
-	hash, err := r.messageSender.SendTransaction(msg.MsgHash, &r.cfg.MessengerContractAddress, big.NewInt(0), data)
+	hash, err := r.messageSender.SendTransaction(msg.MsgHash, &r.cfg.MessengerContractAddress, big.NewInt(0), data, 0)
 	if err != nil && err.Error() == "execution reverted: Message expired" {
 		return r.db.UpdateLayer2Status(r.ctx, msg.MsgHash, orm.MsgExpired)
 	}
@@ -269,7 +269,26 @@ func (r *Layer2Relayer) ProcessPendingBatches(wg *sync.WaitGroup) {
 
 	txID := id + "-commit"
 	// add suffix `-commit` to avoid duplication with finalize tx in unit tests
-	hash, err := r.rollupSender.SendTransaction(txID, &r.cfg.RollupContractAddress, big.NewInt(0), data)
+	hash, err := r.rollupSender.SendTransaction(txID, &r.cfg.RollupContractAddress, big.NewInt(0), data, 0)
+
+	if err != nil && err.Error() == "execution reverted: Parent batch hasn't been committed" {
+
+		// check parent is committing
+		batches, err = r.db.GetBlockBatches(map[string]interface{}{"end_block_hash": batch.ParentHash})
+		if err != nil || len(batches) == 0 {
+			log.Error("Failed to get parent batch from db", "batch_id", id, "parent_hash", batch.ParentHash, "err", err)
+			return
+		}
+		parentBatch := batches[0]
+
+		if parentBatch.RollupStatus >= orm.RollupCommitting {
+			// retry with manual gas estimation
+			gasLimit := estimateCommitBatchGas(len(data), len(layer2Batch.Blocks))
+			hash, err = r.rollupSender.SendTransaction(txID, &r.cfg.RollupContractAddress, big.NewInt(0), data, gasLimit)
+			log.Info("commitBatch tx resent with manual gas estimation ", "id", id, "index", batch.Index, "gasLimit", gasLimit, "hash", hash.String(), "err", err)
+		}
+	}
+
 	if err != nil {
 		if !errors.Is(err, sender.ErrNoAvailableAccount) {
 			log.Error("Failed to send commitBatch tx to layer1 ", "id", id, "index", batch.Index, "err", err)
@@ -375,7 +394,7 @@ func (r *Layer2Relayer) ProcessCommittedBatches(wg *sync.WaitGroup) {
 
 		txID := id + "-finalize"
 		// add suffix `-finalize` to avoid duplication with commit tx in unit tests
-		txHash, err := r.rollupSender.SendTransaction(txID, &r.cfg.RollupContractAddress, big.NewInt(0), data)
+		txHash, err := r.rollupSender.SendTransaction(txID, &r.cfg.RollupContractAddress, big.NewInt(0), data, 0)
 		hash := &txHash
 		if err != nil {
 			if !errors.Is(err, sender.ErrNoAvailableAccount) {
@@ -402,6 +421,8 @@ func (r *Layer2Relayer) ProcessCommittedBatches(wg *sync.WaitGroup) {
 
 // Start the relayer process
 func (r *Layer2Relayer) Start() {
+	log.Info("Starting l2/relayer")
+
 	go func() {
 		// trigger by timer
 		ticker := time.NewTicker(time.Second)
@@ -473,3 +494,12 @@ func (r *Layer2Relayer) handleConfirmation(confirmation *sender.Confirmation) {
 	}
 	log.Info("transaction confirmed in layer1", "type", transactionType, "confirmation", confirmation)
 }
+
+func estimateCommitBatchGas(callDataLength int, numBlocks int) uint64 {
+	gasLimit := uint64(0)
+	gasLimit += 16 * uint64(callDataLength)   // calldata cost
+	gasLimit += 4*2100 + 3*22100              // fixed cost per batch
+	gasLimit += 4 * 22100 * uint64(numBlocks) // cost per block in batch
+	gasLimit = gasLimit * 12 / 10             // apply multiplier
+	return gasLimit
+}

bridge/l2/watcher.go

@@ -14,7 +14,6 @@ import (
 	"github.com/scroll-tech/go-ethereum/ethclient"
 	"github.com/scroll-tech/go-ethereum/event"
 	"github.com/scroll-tech/go-ethereum/log"
-	"github.com/scroll-tech/go-ethereum/metrics"
 
 	bridge_abi "scroll-tech/bridge/abi"
 	"scroll-tech/bridge/utils"
@@ -25,11 +24,6 @@ import (
 	"scroll-tech/bridge/config"
 )
 
-// Metrics
-var (
-	bridgeL2MsgSyncHeightGauge = metrics.NewRegisteredGauge("bridge/l2/msg/sync/height", nil)
-)
-
 type relayedMessage struct {
 	msgHash      common.Hash
 	txHash       common.Hash
@@ -82,6 +76,8 @@ func NewL2WatcherClient(ctx context.Context, client *ethclient.Client, confirmat
 
 // Start the Listening process
 func (w *WatcherClient) Start() {
+	log.Info("Starting l2/watcher")
+
 	go func() {
 		if reflect.ValueOf(w.orm).IsNil() {
 			panic("must run L2 watcher with DB")
@@ -269,7 +265,6 @@ func (w *WatcherClient) FetchContractEvent(blockHeight uint64) {
 		}
 		if len(logs) == 0 {
 			w.processedMsgHeight = uint64(to)
-			bridgeL2MsgSyncHeightGauge.Update(to)
 			continue
 		}
 		log.Info("received new L2 messages", "fromBlock", from, "toBlock", to, "cnt", len(logs))
@@ -302,7 +297,6 @@ func (w *WatcherClient) FetchContractEvent(blockHeight uint64) {
 		}
 
 		w.processedMsgHeight = uint64(to)
-		bridgeL2MsgSyncHeightGauge.Update(to)
 	}
 }
 

bridge/l2/watcher_test.go

@@ -44,7 +44,7 @@ func testCreateNewWatcherAndStop(t *testing.T) {
 	numTransactions := 3
 	toAddress := common.HexToAddress("0x4592d8f8d7b001e72cb26a73e4fa1806a51ac79d")
 	for i := 0; i < numTransactions; i++ {
-		_, err = newSender.SendTransaction(strconv.Itoa(1000+i), &toAddress, big.NewInt(1000000000), nil)
+		_, err = newSender.SendTransaction(strconv.Itoa(1000+i), &toAddress, big.NewInt(1000000000), nil, 0)
 		assert.NoError(t, err)
 		<-newSender.ConfirmChan()
 	}

bridge/sender/sender.go

@@ -154,22 +154,21 @@ func (s *Sender) NumberOfAccounts() int {
 	return len(s.auths.accounts)
 }
 
-func (s *Sender) getFeeData(auth *bind.TransactOpts, target *common.Address, value *big.Int, data []byte) (*FeeData, error) {
-	// estimate gas limit
-	gasLimit, err := s.client.EstimateGas(s.ctx, geth.CallMsg{From: auth.From, To: target, Value: value, Data: data})
-	if err != nil {
-		return nil, err
+func (s *Sender) getFeeData(auth *bind.TransactOpts, target *common.Address, value *big.Int, data []byte, gasLimit uint64) (*FeeData, error) {
+	if gasLimit == 0 {
+		// estimate gas limit
+		var err error
+		gasLimit, err = s.client.EstimateGas(s.ctx, geth.CallMsg{From: auth.From, To: target, Value: value, Data: data})
+		if err != nil {
+			return nil, err
+		}
+		gasLimit = gasLimit * 15 / 10 // 50% extra gas to void out of gas error
 	}
-	gasLimit = gasLimit * 15 / 10 // 50% extra gas to void out of gas error
+
 	// @todo change it when Scroll enable EIP1559
 	if s.config.TxType != DynamicFeeTxType {
 		// estimate gas price
-		var gasPrice *big.Int
-		gasPrice, err = s.client.SuggestGasPrice(s.ctx)
-
-		gasPrice = gasPrice.Mul(gasPrice, big.NewInt(15))
-		gasPrice = gasPrice.Div(gasPrice, big.NewInt(10))
-
+		gasPrice, err := s.client.SuggestGasPrice(s.ctx)
 		if err != nil {
 			return nil, err
 		}
@@ -193,7 +192,7 @@ func (s *Sender) getFeeData(auth *bind.TransactOpts, target *common.Address, val
 }
 
 // SendTransaction send a signed L2tL1 transaction.
-func (s *Sender) SendTransaction(ID string, target *common.Address, value *big.Int, data []byte) (hash common.Hash, err error) {
+func (s *Sender) SendTransaction(ID string, target *common.Address, value *big.Int, data []byte, gasLimit uint64) (hash common.Hash, err error) {
 	// We occupy the ID, in case some other threads call with the same ID in the same time
 	if _, loaded := s.pendingTxs.LoadOrStore(ID, nil); loaded {
 		return common.Hash{}, fmt.Errorf("has the repeat tx ID, ID: %s", ID)
@@ -217,9 +216,10 @@ func (s *Sender) SendTransaction(ID string, target *common.Address, value *big.I
 		tx      *types.Transaction
 	)
 	// estimate gas fee
-	if feeData, err = s.getFeeData(auth, target, value, data); err != nil {
+	if feeData, err = s.getFeeData(auth, target, value, data, gasLimit); err != nil {
 		return
 	}
+
 	if tx, err = s.createAndSendTx(auth, feeData, target, value, data, nil); err == nil {
 		// add pending transaction to queue
 		pending := &PendingTransaction{

bridge/sender/sender_test.go

@@ -87,7 +87,7 @@ func testBatchSender(t *testing.T, batchSize int) {
 			for i := 0; i < TXBatch; i++ {
 				toAddr := common.HexToAddress("0x4592d8f8d7b001e72cb26a73e4fa1806a51ac79d")
 				id := strconv.Itoa(i + index*1000)
-				_, err := newSender.SendTransaction(id, &toAddr, big.NewInt(1), nil)
+				_, err := newSender.SendTransaction(id, &toAddr, big.NewInt(1), nil, 0)
 				if errors.Is(err, sender.ErrNoAvailableAccount) {
 					<-time.After(time.Second)
 					continue

common/metrics/metrics.go

@@ -1,53 +0,0 @@
-package metrics
-
-import (
-	"context"
-	"net"
-	"net/http"
-	"strconv"
-
-	"github.com/scroll-tech/go-ethereum/log"
-	"github.com/scroll-tech/go-ethereum/metrics"
-	"github.com/scroll-tech/go-ethereum/metrics/prometheus"
-	"github.com/scroll-tech/go-ethereum/rpc"
-
-	"github.com/urfave/cli/v2"
-
-	"scroll-tech/common/utils"
-)
-
-// Serve starts the metrics server on the given address, will be closed when the given
-// context is canceled.
-func Serve(ctx context.Context, c *cli.Context) {
-	if !c.Bool(utils.MetricsEnabled.Name) {
-		return
-	}
-
-	address := net.JoinHostPort(
-		c.String(utils.MetricsAddr.Name),
-		strconv.Itoa(c.Int(utils.MetricsPort.Name)),
-	)
-
-	server := &http.Server{
-		Addr:         address,
-		Handler:      prometheus.Handler(metrics.DefaultRegistry),
-		ReadTimeout:  rpc.DefaultHTTPTimeouts.ReadTimeout,
-		WriteTimeout: rpc.DefaultHTTPTimeouts.WriteTimeout,
-		IdleTimeout:  rpc.DefaultHTTPTimeouts.IdleTimeout,
-	}
-
-	go func() {
-		<-ctx.Done()
-		if err := server.Close(); err != nil {
-			log.Error("Failed to close metrics server", "error", err)
-		}
-	}()
-
-	log.Info("Starting metrics server", "address", address)
-
-	go func() {
-		if err := server.ListenAndServe(); err != nil {
-			log.Error("start metrics server error", "error", err)
-		}
-	}()
-}

common/utils/flags.go

@@ -12,9 +12,6 @@ var (
 		&LogFileFlag,
 		&LogJSONFormat,
 		&LogDebugFlag,
-		&MetricsEnabled,
-		&MetricsAddr,
-		&MetricsPort,
 	}
 	// ConfigFileFlag load json type config file.
 	ConfigFileFlag = cli.StringFlag{
@@ -45,25 +42,4 @@ var (
 		Name:  "log.debug",
 		Usage: "Prepends log messages with call-site location (file and line number)",
 	}
-	// MetricsEnabled enable metrics collection and reporting
-	MetricsEnabled = cli.BoolFlag{
-		Name:     "metrics",
-		Usage:    "Enable metrics collection and reporting",
-		Category: "METRICS",
-		Value:    false,
-	}
-	// MetricsAddr is listening address of Metrics reporting server
-	MetricsAddr = cli.StringFlag{
-		Name:     "metrics.addr",
-		Usage:    "Metrics reporting server listening address",
-		Category: "METRICS",
-		Value:    "0.0.0.0",
-	}
-	// MetricsPort is listening port of Metrics reporting server
-	MetricsPort = cli.IntFlag{
-		Name:     "metrics.port",
-		Usage:    "Metrics reporting server listening port",
-		Category: "METRICS",
-		Value:    6060,
-	}
 )

common/version/version.go

@@ -5,7 +5,7 @@ import (
 	"runtime/debug"
 )
 
-var tag = "prealpha-v11.16"
+var tag = "prealpha-v11.14"
 
 var commit = func() string {
 	if info, ok := debug.ReadBuildInfo(); ok {

tests/integration-test/integration_test.go

@@ -1,16 +1,8 @@
 package integration
 
 import (
-	"crypto/rand"
-	"io/ioutil"
-	"math/big"
-	"net/http"
-	"strconv"
-	"strings"
 	"testing"
 	"time"
-
-	"github.com/stretchr/testify/assert"
 )
 
 func TestIntegration(t *testing.T) {
@@ -24,9 +16,6 @@ func TestIntegration(t *testing.T) {
 	// test bridge service
 	t.Run("testStartProcess", testStartProcess)
 
-	// test monitor metrics
-	t.Run("testMonitorMetrics", testMonitorMetrics)
-
 	t.Cleanup(func() {
 		free(t)
 	})
@@ -54,28 +43,3 @@ func testStartProcess(t *testing.T) {
 	bridgeCmd.WaitExit()
 	coordinatorCmd.WaitExit()
 }
-
-func testMonitorMetrics(t *testing.T) {
-	// migrate db.
-	runDBCliApp(t, "reset", "successful to reset")
-	runDBCliApp(t, "migrate", "current version:")
-
-	// Start bridge process with metrics server.
-	port, _ := rand.Int(rand.Reader, big.NewInt(2000))
-	svrPort := strconv.FormatInt(port.Int64()+50000, 10)
-	bridgeCmd := runBridgeApp(t, "--metrics", "--metrics.addr", "localhost", "--metrics.port", svrPort)
-	bridgeCmd.RunApp(func() bool { return bridgeCmd.WaitResult(time.Second*20, "Start bridge successfully") })
-
-	// Get monitor metrics.
-	resp, err := http.Get("http://localhost:" + svrPort)
-	assert.NoError(t, err)
-	defer resp.Body.Close()
-	body, err := ioutil.ReadAll(resp.Body)
-	assert.NoError(t, err)
-	bodyStr := string(body)
-	assert.Equal(t, 200, resp.StatusCode)
-	assert.Equal(t, true, strings.Contains(bodyStr, "bridge_l1_msg_sync_height"))
-	assert.Equal(t, true, strings.Contains(bodyStr, "bridge_l2_msg_sync_height"))
-
-	bridgeCmd.WaitExit()
-}