feat(watcher&rollup): add block commit calldata size and commit/finalize batch revert metrics (#974)

Co-authored-by: colinlyguo <colinlyguo@users.noreply.github.com>

bridge-history-api/internal/controller/controller.go

@@ -11,10 +11,6 @@ var (
 	HistoryCtrler *HistoryController
 	// BatchCtrler is controller instance
 	BatchCtrler *BatchController
-	// HealthCheck the health check controller
-	HealthCheck *HealthCheckController
-	// Ready the ready controller
-	Ready *ReadyController
 
 	initControllerOnce sync.Once
 )
@@ -24,7 +20,5 @@ func InitController(db *gorm.DB) {
 	initControllerOnce.Do(func() {
 		HistoryCtrler = NewHistoryController(db)
 		BatchCtrler = NewBatchController(db)
-		HealthCheck = NewHealthCheckController(db)
-		Ready = NewReadyController()
 	})
 }

bridge-history-api/internal/controller/health_check.go

@@ -1,30 +0,0 @@
-package controller
-
-import (
-	"github.com/gin-gonic/gin"
-	"gorm.io/gorm"
-
-	"bridge-history-api/internal/types"
-	"bridge-history-api/utils"
-)
-
-// HealthCheckController is health check API
-type HealthCheckController struct {
-	db *gorm.DB
-}
-
-// NewHealthCheckController returns an HealthCheckController instance
-func NewHealthCheckController(db *gorm.DB) *HealthCheckController {
-	return &HealthCheckController{
-		db: db,
-	}
-}
-
-// HealthCheck the api controller for coordinator health check
-func (a *HealthCheckController) HealthCheck(c *gin.Context) {
-	if _, err := utils.Ping(a.db); err != nil {
-		types.RenderFatal(c, err)
-		return
-	}
-	types.RenderSuccess(c, nil)
-}

bridge-history-api/internal/controller/history_controller.go

@@ -26,6 +26,7 @@ type HistoryController struct {
 	historyLogic *logic.HistoryLogic
 	cache        *cache.Cache
 	singleFlight singleflight.Group
+	cacheMetrics *cacheMetrics
 }
 
 // NewHistoryController return HistoryController instance
@@ -33,6 +34,7 @@ func NewHistoryController(db *gorm.DB) *HistoryController {
 	return &HistoryController{
 		historyLogic: logic.NewHistoryLogic(db),
 		cache:        cache.New(30*time.Second, 10*time.Minute),
+		cacheMetrics: initCacheMetrics(),
 	}
 }
 
@@ -46,12 +48,22 @@ func (c *HistoryController) GetAllClaimableTxsByAddr(ctx *gin.Context) {
 
 	cacheKey := cacheKeyPrefixClaimableTxsByAddr + req.Address
 	if cachedData, found := c.cache.Get(cacheKey); found {
-		if resultData, ok := cachedData.(*types.ResultData); ok {
+		c.cacheMetrics.cacheHits.WithLabelValues("GetAllClaimableTxsByAddr").Inc()
+		// Log cache hit along with request param.
+		log.Info("cache hit", "request", req)
+		if cachedData == nil {
+			types.RenderSuccess(ctx, &types.ResultData{})
+			return
+		} else if resultData, ok := cachedData.(*types.ResultData); ok {
 			types.RenderSuccess(ctx, resultData)
 			return
 		}
 		// Log error for unexpected type, then fetch data from the database.
 		log.Error("unexpected type in cache", "expected", "*types.ResultData", "got", reflect.TypeOf(cachedData))
+	} else {
+		c.cacheMetrics.cacheMisses.WithLabelValues("GetAllClaimableTxsByAddr").Inc()
+		// Log cache miss along with request param.
+		log.Info("cache miss", "request", req)
 	}
 
 	result, err, _ := c.singleFlight.Do(cacheKey, func() (interface{}, error) {
@@ -86,7 +98,7 @@ func (c *HistoryController) PostQueryTxsByHash(ctx *gin.Context) {
 	}
 
 	if len(req.Txs) > 10 {
-		types.RenderFailure(ctx, types.ErrParameterInvalidNo, errors.New("the number of hashes in the request exceeds the allowed maximum"))
+		types.RenderFailure(ctx, types.ErrParameterInvalidNo, errors.New("the number of hashes in the request exceeds the allowed maximum of 10"))
 		return
 	}
 	hashesMap := make(map[string]struct{}, len(req.Txs))
@@ -101,13 +113,21 @@ func (c *HistoryController) PostQueryTxsByHash(ctx *gin.Context) {
 
 		cacheKey := cacheKeyPrefixQueryTxsByHash + hash
 		if cachedData, found := c.cache.Get(cacheKey); found {
-			if txInfo, ok := cachedData.(*types.TxHistoryInfo); ok {
+			c.cacheMetrics.cacheHits.WithLabelValues("PostQueryTxsByHash").Inc()
+			// Log cache hit along with tx hash.
+			log.Info("cache hit", "tx hash", hash)
+			if cachedData == nil {
+				continue
+			} else if txInfo, ok := cachedData.(*types.TxHistoryInfo); ok {
 				results = append(results, txInfo)
 			} else {
 				log.Error("unexpected type in cache", "expected", "*types.TxHistoryInfo", "got", reflect.TypeOf(cachedData))
 				uncachedHashes = append(uncachedHashes, hash)
 			}
 		} else {
+			c.cacheMetrics.cacheMisses.WithLabelValues("PostQueryTxsByHash").Inc()
+			// Log cache miss along with tx hash.
+			log.Info("cache miss", "tx hash", hash)
 			uncachedHashes = append(uncachedHashes, hash)
 		}
 	}
@@ -119,10 +139,20 @@ func (c *HistoryController) PostQueryTxsByHash(ctx *gin.Context) {
 			return
 		}
 
+		resultMap := make(map[string]*types.TxHistoryInfo)
 		for _, result := range dbResults {
 			results = append(results, result)
-			cacheKey := cacheKeyPrefixQueryTxsByHash + result.Hash
-			c.cache.Set(cacheKey, result, cache.DefaultExpiration)
+			resultMap[result.Hash] = result
+		}
+
+		for _, hash := range uncachedHashes {
+			cacheKey := cacheKeyPrefixQueryTxsByHash + hash
+			result, found := resultMap[hash]
+			if found {
+				c.cache.Set(cacheKey, result, cache.DefaultExpiration)
+			} else {
+				c.cache.Set(cacheKey, nil, cache.DefaultExpiration)
+			}
 		}
 	}
 

bridge-history-api/internal/controller/metrics.go

@@ -0,0 +1,40 @@
+package controller
+
+import (
+	"sync"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+type cacheMetrics struct {
+	cacheHits   *prometheus.CounterVec
+	cacheMisses *prometheus.CounterVec
+}
+
+var (
+	initMetricsOnce sync.Once
+	cm              *cacheMetrics
+)
+
+func initCacheMetrics() *cacheMetrics {
+	initMetricsOnce.Do(func() {
+		cm = &cacheMetrics{
+			cacheHits: promauto.NewCounterVec(
+				prometheus.CounterOpts{
+					Name: "bridge_history_api_cache_hits_total",
+					Help: "The total number of cache hits",
+				},
+				[]string{"api"},
+			),
+			cacheMisses: promauto.NewCounterVec(
+				prometheus.CounterOpts{
+					Name: "bridge_history_api_cache_misses_total",
+					Help: "The total number of cache misses",
+				},
+				[]string{"api"},
+			),
+		}
+	})
+	return cm
+}

bridge-history-api/internal/controller/ready.go

@@ -1,21 +0,0 @@
-package controller
-
-import (
-	"github.com/gin-gonic/gin"
-
-	"bridge-history-api/internal/types"
-)
-
-// ReadyController ready API
-type ReadyController struct {
-}
-
-// NewReadyController returns an ReadyController instance
-func NewReadyController() *ReadyController {
-	return &ReadyController{}
-}
-
-// Ready the api controller for coordinator ready
-func (r *ReadyController) Ready(c *gin.Context) {
-	types.RenderSuccess(c, nil)
-}

bridge-history-api/internal/route/route.go

@@ -22,7 +22,7 @@ func Route(router *gin.Engine, conf *config.Config, reg prometheus.Registerer) {
 		MaxAge:           12 * time.Hour,
 	}))
 
-	observability.Use(router, "bridge-history", reg)
+	observability.Use(router, "bridge_history_api", reg)
 
 	r := router.Group("api/")
 	r.POST("/txsbyhashes", controller.HistoryCtrler.PostQueryTxsByHash)

bridge-history-api/internal/types/history_types.go

@@ -26,9 +26,7 @@ const (
 
 // QueryByAddressRequest the request parameter of address api
 type QueryByAddressRequest struct {
-	Address  string `form:"address" binding:"required"`
-	Page     int    `form:"page,default=1"`
-	PageSize int    `form:"page_size,default=10"`
+	Address string `form:"address" binding:"required"`
 }
 
 // QueryByHashRequest the request parameter of hash api

bridge-history-api/observability/middleware.go

@@ -4,7 +4,7 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/prometheus/client_golang/prometheus"
 
-	"scroll-tech/common/observability/ginmetrics"
+	"bridge-history-api/observability/ginmetrics"
 )
 
 // Use register the gin metric

bridge-history-api/observability/probes.go

@@ -4,8 +4,8 @@ import (
 	"github.com/gin-gonic/gin"
 	"gorm.io/gorm"
 
-	"scroll-tech/common/database"
-	"scroll-tech/common/types"
+	"bridge-history-api/internal/types"
+	"bridge-history-api/utils"
 )
 
 // ProbesController probe check controller
@@ -22,7 +22,7 @@ func NewProbesController(db *gorm.DB) *ProbesController {
 
 // HealthCheck the api controller for health check
 func (a *ProbesController) HealthCheck(c *gin.Context) {
-	if _, err := database.Ping(a.db); err != nil {
+	if _, err := utils.Ping(a.db); err != nil {
 		types.RenderFatal(c, err)
 		return
 	}

bridge-history-api/observability/server.go

@@ -16,7 +16,7 @@ import (
 	"github.com/urfave/cli/v2"
 	"gorm.io/gorm"
 
-	"scroll-tech/common/utils"
+	"bridge-history-api/utils"
 )
 
 // Server starts the metrics server on the given address, will be closed when the given

common/version/version.go

@@ -5,7 +5,7 @@ import (
 	"runtime/debug"
 )
 
-var tag = "v4.3.22"
+var tag = "v4.3.27"
 
 var commit = func() string {
 	if info, ok := debug.ReadBuildInfo(); ok {

rollup/internal/controller/relayer/l2_relayer.go

@@ -582,7 +582,8 @@ func (r *Layer2Relayer) handleConfirmation(confirmation *sender.Confirmation) {
 			status = types.RollupCommitted
 		} else {
 			status = types.RollupCommitFailed
-			log.Warn("transaction confirmed but failed in layer1", "confirmation", confirmation)
+			r.metrics.rollupL2BatchesCommittedConfirmedFailedTotal.Inc()
+			log.Warn("commitBatch transaction confirmed but failed in layer1", "confirmation", confirmation)
 		}
 		// @todo handle db error
 		err := r.batchOrm.UpdateCommitTxHashAndRollupStatus(r.ctx, batchHash.(string), confirmation.TxHash.String(), status)
@@ -603,7 +604,8 @@ func (r *Layer2Relayer) handleConfirmation(confirmation *sender.Confirmation) {
 			status = types.RollupFinalized
 		} else {
 			status = types.RollupFinalizeFailed
-			log.Warn("transaction confirmed but failed in layer1", "confirmation", confirmation)
+			r.metrics.rollupL2BatchesFinalizedConfirmedFailedTotal.Inc()
+			log.Warn("finalizeBatchWithProof transaction confirmed but failed in layer1", "confirmation", confirmation)
 		}
 
 		// @todo handle db error

rollup/internal/controller/relayer/l2_relayer_metrics.go

@@ -16,7 +16,9 @@ type l2RelayerMetrics struct {
 	rollupL2RelayerProcessCommittedBatchesFinalizedTotal        prometheus.Counter
 	rollupL2RelayerProcessCommittedBatchesFinalizedSuccessTotal prometheus.Counter
 	rollupL2BatchesCommittedConfirmedTotal                      prometheus.Counter
+	rollupL2BatchesCommittedConfirmedFailedTotal                prometheus.Counter
 	rollupL2BatchesFinalizedConfirmedTotal                      prometheus.Counter
+	rollupL2BatchesFinalizedConfirmedFailedTotal                prometheus.Counter
 	rollupL2BatchesGasOraclerConfirmedTotal                     prometheus.Counter
 	rollupL2ChainMonitorLatestFailedCall                        prometheus.Counter
 	rollupL2ChainMonitorLatestFailedBatchStatus                 prometheus.Counter
@@ -62,10 +64,18 @@ func initL2RelayerMetrics(reg prometheus.Registerer) *l2RelayerMetrics {
 				Name: "rollup_layer2_process_committed_batches_confirmed_total",
 				Help: "The total number of layer2 process committed batches confirmed total",
 			}),
+			rollupL2BatchesCommittedConfirmedFailedTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
+				Name: "rollup_layer2_process_committed_batches_confirmed_failed_total",
+				Help: "The total number of layer2 process committed batches confirmed failed total",
+			}),
 			rollupL2BatchesFinalizedConfirmedTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
 				Name: "rollup_layer2_process_finalized_batches_confirmed_total",
 				Help: "The total number of layer2 process finalized batches confirmed total",
 			}),
+			rollupL2BatchesFinalizedConfirmedFailedTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
+				Name: "rollup_layer2_process_finalized_batches_confirmed_failed_total",
+				Help: "The total number of layer2 process finalized batches confirmed failed total",
+			}),
 			rollupL2BatchesGasOraclerConfirmedTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
 				Name: "rollup_layer2_process_gras_oracler_confirmed_total",
 				Help: "The total number of layer2 process finalized batches confirmed total",

rollup/internal/controller/watcher/l2_watcher.go

@@ -180,6 +180,9 @@ func (w *L2WatcherClient) getAndStoreBlockTraces(ctx context.Context, from, to u
 	}
 
 	if len(blocks) > 0 {
+		for _, block := range blocks {
+			w.metrics.rollupL2BlockL1CommitCalldataSize.Set(float64(block.EstimateL1CommitCalldataSize()))
+		}
 		if err := w.l2BlockOrm.InsertL2Blocks(w.ctx, blocks); err != nil {
 			return fmt.Errorf("failed to batch insert BlockTraces: %v", err)
 		}

rollup/internal/controller/watcher/l2_watcher_metrics.go

@@ -8,12 +8,13 @@ import (
 )
 
 type l2WatcherMetrics struct {
-	fetchRunningMissingBlocksTotal  prometheus.Counter
-	fetchRunningMissingBlocksHeight prometheus.Gauge
-	fetchContractEventTotal         prometheus.Counter
-	fetchContractEventHeight        prometheus.Gauge
-	rollupL2MsgsRelayedEventsTotal  prometheus.Counter
-	rollupL2BlocksFetchedGap        prometheus.Gauge
+	fetchRunningMissingBlocksTotal    prometheus.Counter
+	fetchRunningMissingBlocksHeight   prometheus.Gauge
+	fetchContractEventTotal           prometheus.Counter
+	fetchContractEventHeight          prometheus.Gauge
+	rollupL2MsgsRelayedEventsTotal    prometheus.Counter
+	rollupL2BlocksFetchedGap          prometheus.Gauge
+	rollupL2BlockL1CommitCalldataSize prometheus.Gauge
 }
 
 var (
@@ -48,6 +49,10 @@ func initL2WatcherMetrics(reg prometheus.Registerer) *l2WatcherMetrics {
 				Name: "rollup_l2_watcher_blocks_fetched_gap",
 				Help: "The gap of l2 fetch",
 			}),
+			rollupL2BlockL1CommitCalldataSize: promauto.With(reg).NewGauge(prometheus.GaugeOpts{
+				Name: "rollup_l2_block_l1_commit_calldata_size",
+				Help: "The l1 commitBatch calldata size of the l2 block",
+			}),
 		}
 	})
 	return l2WatcherMetric