Compare commits

..

7 Commits

Author SHA1 Message Date
Steven
6f72d0447e fix(libzkp): upgrade libzkp to v0.9.4 (#973)
Co-authored-by: silathdiir <silathdiir@users.noreply.github.com>
Co-authored-by: colin <102356659+colinlyguo@users.noreply.github.com>
Co-authored-by: colinlyguo <colinlyguo@users.noreply.github.com>
2023-10-02 15:39:51 +08:00
georgehao
76d66eba58 feat: add chunk/batch get_task index (#976)
Co-authored-by: maskpp <maskpp266@gmail.com>
2023-09-29 08:55:10 -05:00
colin
c551609e17 feat(watcher&rollup): add block commit calldata size and commit/finalize batch revert metrics (#974)
Co-authored-by: colinlyguo <colinlyguo@users.noreply.github.com>
2023-09-29 15:33:08 +08:00
colin
47e5a43646 fix(bridge-history): add cache hit tracing logs (#972) 2023-09-27 13:11:01 +08:00
colin
7604612581 refactor(bridge-history): rename metrics (#971)
Co-authored-by: colinlyguo <colinlyguo@users.noreply.github.com>
2023-09-27 01:07:22 +08:00
colin
35d4ec5ad0 feat(bridge-history): add cache hit metrics and cache non-existent keys (#970)
Co-authored-by: colinlyguo <colinlyguo@users.noreply.github.com>
2023-09-26 23:43:09 +08:00
colin
8f745e9836 fix(bridge-history): duplicated symbol and metric prefix (#969)
Co-authored-by: colinlyguo <colinlyguo@users.noreply.github.com>
2023-09-26 18:52:18 +08:00
19 changed files with 153 additions and 95 deletions

View File

@@ -11,10 +11,6 @@ var (
HistoryCtrler *HistoryController
// BatchCtrler is controller instance
BatchCtrler *BatchController
// HealthCheck the health check controller
HealthCheck *HealthCheckController
// Ready the ready controller
Ready *ReadyController
initControllerOnce sync.Once
)
@@ -24,7 +20,5 @@ func InitController(db *gorm.DB) {
initControllerOnce.Do(func() {
HistoryCtrler = NewHistoryController(db)
BatchCtrler = NewBatchController(db)
HealthCheck = NewHealthCheckController(db)
Ready = NewReadyController()
})
}

View File

@@ -1,30 +0,0 @@
package controller
import (
"github.com/gin-gonic/gin"
"gorm.io/gorm"
"bridge-history-api/internal/types"
"bridge-history-api/utils"
)
// HealthCheckController is health check API
type HealthCheckController struct {
db *gorm.DB
}
// NewHealthCheckController returns an HealthCheckController instance
func NewHealthCheckController(db *gorm.DB) *HealthCheckController {
return &HealthCheckController{
db: db,
}
}
// HealthCheck the api controller for coordinator health check
func (a *HealthCheckController) HealthCheck(c *gin.Context) {
if _, err := utils.Ping(a.db); err != nil {
types.RenderFatal(c, err)
return
}
types.RenderSuccess(c, nil)
}

View File

@@ -26,6 +26,7 @@ type HistoryController struct {
historyLogic *logic.HistoryLogic
cache *cache.Cache
singleFlight singleflight.Group
cacheMetrics *cacheMetrics
}
// NewHistoryController return HistoryController instance
@@ -33,6 +34,7 @@ func NewHistoryController(db *gorm.DB) *HistoryController {
return &HistoryController{
historyLogic: logic.NewHistoryLogic(db),
cache: cache.New(30*time.Second, 10*time.Minute),
cacheMetrics: initCacheMetrics(),
}
}
@@ -46,12 +48,22 @@ func (c *HistoryController) GetAllClaimableTxsByAddr(ctx *gin.Context) {
cacheKey := cacheKeyPrefixClaimableTxsByAddr + req.Address
if cachedData, found := c.cache.Get(cacheKey); found {
if resultData, ok := cachedData.(*types.ResultData); ok {
c.cacheMetrics.cacheHits.WithLabelValues("GetAllClaimableTxsByAddr").Inc()
// Log cache hit along with request param.
log.Info("cache hit", "request", req)
if cachedData == nil {
types.RenderSuccess(ctx, &types.ResultData{})
return
} else if resultData, ok := cachedData.(*types.ResultData); ok {
types.RenderSuccess(ctx, resultData)
return
}
// Log error for unexpected type, then fetch data from the database.
log.Error("unexpected type in cache", "expected", "*types.ResultData", "got", reflect.TypeOf(cachedData))
} else {
c.cacheMetrics.cacheMisses.WithLabelValues("GetAllClaimableTxsByAddr").Inc()
// Log cache miss along with request param.
log.Info("cache miss", "request", req)
}
result, err, _ := c.singleFlight.Do(cacheKey, func() (interface{}, error) {
@@ -86,7 +98,7 @@ func (c *HistoryController) PostQueryTxsByHash(ctx *gin.Context) {
}
if len(req.Txs) > 10 {
types.RenderFailure(ctx, types.ErrParameterInvalidNo, errors.New("the number of hashes in the request exceeds the allowed maximum"))
types.RenderFailure(ctx, types.ErrParameterInvalidNo, errors.New("the number of hashes in the request exceeds the allowed maximum of 10"))
return
}
hashesMap := make(map[string]struct{}, len(req.Txs))
@@ -101,13 +113,21 @@ func (c *HistoryController) PostQueryTxsByHash(ctx *gin.Context) {
cacheKey := cacheKeyPrefixQueryTxsByHash + hash
if cachedData, found := c.cache.Get(cacheKey); found {
if txInfo, ok := cachedData.(*types.TxHistoryInfo); ok {
c.cacheMetrics.cacheHits.WithLabelValues("PostQueryTxsByHash").Inc()
// Log cache hit along with tx hash.
log.Info("cache hit", "tx hash", hash)
if cachedData == nil {
continue
} else if txInfo, ok := cachedData.(*types.TxHistoryInfo); ok {
results = append(results, txInfo)
} else {
log.Error("unexpected type in cache", "expected", "*types.TxHistoryInfo", "got", reflect.TypeOf(cachedData))
uncachedHashes = append(uncachedHashes, hash)
}
} else {
c.cacheMetrics.cacheMisses.WithLabelValues("PostQueryTxsByHash").Inc()
// Log cache miss along with tx hash.
log.Info("cache miss", "tx hash", hash)
uncachedHashes = append(uncachedHashes, hash)
}
}
@@ -119,10 +139,20 @@ func (c *HistoryController) PostQueryTxsByHash(ctx *gin.Context) {
return
}
resultMap := make(map[string]*types.TxHistoryInfo)
for _, result := range dbResults {
results = append(results, result)
cacheKey := cacheKeyPrefixQueryTxsByHash + result.Hash
c.cache.Set(cacheKey, result, cache.DefaultExpiration)
resultMap[result.Hash] = result
}
for _, hash := range uncachedHashes {
cacheKey := cacheKeyPrefixQueryTxsByHash + hash
result, found := resultMap[hash]
if found {
c.cache.Set(cacheKey, result, cache.DefaultExpiration)
} else {
c.cache.Set(cacheKey, nil, cache.DefaultExpiration)
}
}
}

View File

@@ -0,0 +1,40 @@
package controller
import (
"sync"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
type cacheMetrics struct {
cacheHits *prometheus.CounterVec
cacheMisses *prometheus.CounterVec
}
var (
initMetricsOnce sync.Once
cm *cacheMetrics
)
func initCacheMetrics() *cacheMetrics {
initMetricsOnce.Do(func() {
cm = &cacheMetrics{
cacheHits: promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "bridge_history_api_cache_hits_total",
Help: "The total number of cache hits",
},
[]string{"api"},
),
cacheMisses: promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "bridge_history_api_cache_misses_total",
Help: "The total number of cache misses",
},
[]string{"api"},
),
}
})
return cm
}

View File

@@ -1,21 +0,0 @@
package controller
import (
"github.com/gin-gonic/gin"
"bridge-history-api/internal/types"
)
// ReadyController ready API
type ReadyController struct {
}
// NewReadyController returns an ReadyController instance
func NewReadyController() *ReadyController {
return &ReadyController{}
}
// Ready the api controller for coordinator ready
func (r *ReadyController) Ready(c *gin.Context) {
types.RenderSuccess(c, nil)
}

View File

@@ -22,7 +22,7 @@ func Route(router *gin.Engine, conf *config.Config, reg prometheus.Registerer) {
MaxAge: 12 * time.Hour,
}))
observability.Use(router, "bridge-history", reg)
observability.Use(router, "bridge_history_api", reg)
r := router.Group("api/")
r.POST("/txsbyhashes", controller.HistoryCtrler.PostQueryTxsByHash)

View File

@@ -26,9 +26,7 @@ const (
// QueryByAddressRequest the request parameter of address api
type QueryByAddressRequest struct {
Address string `form:"address" binding:"required"`
Page int `form:"page,default=1"`
PageSize int `form:"page_size,default=10"`
Address string `form:"address" binding:"required"`
}
// QueryByHashRequest the request parameter of hash api

View File

@@ -4,7 +4,7 @@ import (
"github.com/gin-gonic/gin"
"github.com/prometheus/client_golang/prometheus"
"scroll-tech/common/observability/ginmetrics"
"bridge-history-api/observability/ginmetrics"
)
// Use register the gin metric

View File

@@ -4,8 +4,8 @@ import (
"github.com/gin-gonic/gin"
"gorm.io/gorm"
"scroll-tech/common/database"
"scroll-tech/common/types"
"bridge-history-api/internal/types"
"bridge-history-api/utils"
)
// ProbesController probe check controller
@@ -22,7 +22,7 @@ func NewProbesController(db *gorm.DB) *ProbesController {
// HealthCheck the api controller for health check
func (a *ProbesController) HealthCheck(c *gin.Context) {
if _, err := database.Ping(a.db); err != nil {
if _, err := utils.Ping(a.db); err != nil {
types.RenderFatal(c, err)
return
}

View File

@@ -16,7 +16,7 @@ import (
"github.com/urfave/cli/v2"
"gorm.io/gorm"
"scroll-tech/common/utils"
"bridge-history-api/utils"
)
// Server starts the metrics server on the given address, will be closed when the given

View File

@@ -31,7 +31,7 @@ dependencies = [
[[package]]
name = "aggregator"
version = "0.1.0"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.3#f7fb2900514c38b0ee15b1666c696df4b75a61ca"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.4#40f4758f5b4b5a5c82fb312ee58492487f181185"
dependencies = [
"ark-std",
"env_logger 0.10.0",
@@ -333,7 +333,7 @@ checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1"
[[package]]
name = "bus-mapping"
version = "0.1.0"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.3#f7fb2900514c38b0ee15b1666c696df4b75a61ca"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.4#40f4758f5b4b5a5c82fb312ee58492487f181185"
dependencies = [
"eth-types",
"ethers-core",
@@ -959,7 +959,7 @@ dependencies = [
[[package]]
name = "eth-types"
version = "0.1.0"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.3#f7fb2900514c38b0ee15b1666c696df4b75a61ca"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.4#40f4758f5b4b5a5c82fb312ee58492487f181185"
dependencies = [
"ethers-core",
"ethers-signers",
@@ -1116,7 +1116,7 @@ dependencies = [
[[package]]
name = "external-tracer"
version = "0.1.0"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.3#f7fb2900514c38b0ee15b1666c696df4b75a61ca"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.4#40f4758f5b4b5a5c82fb312ee58492487f181185"
dependencies = [
"eth-types",
"geth-utils",
@@ -1296,7 +1296,7 @@ dependencies = [
[[package]]
name = "gadgets"
version = "0.1.0"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.3#f7fb2900514c38b0ee15b1666c696df4b75a61ca"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.4#40f4758f5b4b5a5c82fb312ee58492487f181185"
dependencies = [
"digest 0.7.6",
"eth-types",
@@ -1328,7 +1328,7 @@ dependencies = [
[[package]]
name = "geth-utils"
version = "0.1.0"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.3#f7fb2900514c38b0ee15b1666c696df4b75a61ca"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.4#40f4758f5b4b5a5c82fb312ee58492487f181185"
dependencies = [
"env_logger 0.9.3",
"gobuild 0.1.0-alpha.2 (git+https://github.com/scroll-tech/gobuild.git)",
@@ -1497,7 +1497,7 @@ dependencies = [
[[package]]
name = "halo2-mpt-circuits"
version = "0.1.0"
source = "git+https://github.com/scroll-tech/mpt-circuit.git?tag=v0.6.5#0bae9eeb813583c11f6db1f961a7e92f8c9bda82"
source = "git+https://github.com/scroll-tech/mpt-circuit.git?tag=v0.7.0#578c210ceb88d3c143ee2a013ad836d19285d9c1"
dependencies = [
"ethers-core",
"halo2_proofs",
@@ -1519,7 +1519,7 @@ dependencies = [
[[package]]
name = "halo2_proofs"
version = "0.2.0"
source = "git+https://github.com/scroll-tech/halo2.git?branch=develop#aa86c107aeb62282d81ebce5c4930ec0c0aa540b"
source = "git+https://github.com/scroll-tech/halo2.git?branch=develop#e3fe25eadd714fd991f35190d17ff0b8fb031188"
dependencies = [
"ark-std",
"blake2b_simd",
@@ -1937,7 +1937,7 @@ dependencies = [
[[package]]
name = "keccak256"
version = "0.1.0"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.3#f7fb2900514c38b0ee15b1666c696df4b75a61ca"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.4#40f4758f5b4b5a5c82fb312ee58492487f181185"
dependencies = [
"env_logger 0.9.3",
"eth-types",
@@ -2135,7 +2135,7 @@ dependencies = [
[[package]]
name = "mock"
version = "0.1.0"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.3#f7fb2900514c38b0ee15b1666c696df4b75a61ca"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.4#40f4758f5b4b5a5c82fb312ee58492487f181185"
dependencies = [
"eth-types",
"ethers-core",
@@ -2151,7 +2151,7 @@ dependencies = [
[[package]]
name = "mpt-zktrie"
version = "0.1.0"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.3#f7fb2900514c38b0ee15b1666c696df4b75a61ca"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.4#40f4758f5b4b5a5c82fb312ee58492487f181185"
dependencies = [
"eth-types",
"halo2-mpt-circuits",
@@ -2582,7 +2582,7 @@ dependencies = [
[[package]]
name = "prover"
version = "0.1.0"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.3#f7fb2900514c38b0ee15b1666c696df4b75a61ca"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.4#40f4758f5b4b5a5c82fb312ee58492487f181185"
dependencies = [
"aggregator",
"anyhow",
@@ -4125,7 +4125,7 @@ checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9"
[[package]]
name = "zkevm-circuits"
version = "0.1.0"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.3#f7fb2900514c38b0ee15b1666c696df4b75a61ca"
source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.9.4#40f4758f5b4b5a5c82fb312ee58492487f181185"
dependencies = [
"array-init",
"bus-mapping",

View File

@@ -21,7 +21,7 @@ halo2curves = { git = "https://github.com/scroll-tech/halo2curves.git", branch =
[dependencies]
halo2_proofs = { git = "https://github.com/scroll-tech/halo2.git", branch = "develop" }
prover = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.9.3", default-features = false, features = ["parallel_syn", "scroll", "shanghai"] }
prover = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.9.4", default-features = false, features = ["parallel_syn", "scroll", "shanghai"] }
base64 = "0.13.0"
env_logger = "0.9.0"

View File

@@ -5,7 +5,7 @@ import (
"runtime/debug"
)
var tag = "v4.3.22"
var tag = "v4.3.28"
var commit = func() string {
if info, ok := debug.ReadBuildInfo(); ok {

View File

@@ -63,7 +63,7 @@ func testResetDB(t *testing.T) {
cur, err := Current(pgDB.DB)
assert.NoError(t, err)
// total number of tables.
assert.Equal(t, 13, int(cur))
assert.Equal(t, 14, int(cur))
}
func testMigrate(t *testing.T) {

View File

@@ -0,0 +1,27 @@
-- +goose Up
-- +goose StatementBegin
drop index if exists idx_total_attempts_active_attempts_end_block_number;
drop index if exists idx_total_attempts_active_attempts_chunk_proofs_status;
create index if not exists idx_chunk_proving_status_index on chunk (proving_status, index) where deleted_at IS NULL;
create index if not exists idx_batch_proving_status_index on batch (proving_status, chunk_proofs_status, index) where deleted_at IS NULL;
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
create index if not exists idx_total_attempts_active_attempts_end_block_number
on chunk (total_attempts, active_attempts, end_block_number)
where deleted_at IS NULL;
create index if not exists idx_total_attempts_active_attempts_chunk_proofs_status
on batch (total_attempts, active_attempts, chunk_proofs_status)
where deleted_at IS NULL;
drop index if exists idx_chunk_proving_status_index;
drop index if exists idx_batch_proving_status_index;
-- +goose StatementEnd

View File

@@ -582,7 +582,8 @@ func (r *Layer2Relayer) handleConfirmation(confirmation *sender.Confirmation) {
status = types.RollupCommitted
} else {
status = types.RollupCommitFailed
log.Warn("transaction confirmed but failed in layer1", "confirmation", confirmation)
r.metrics.rollupL2BatchesCommittedConfirmedFailedTotal.Inc()
log.Warn("commitBatch transaction confirmed but failed in layer1", "confirmation", confirmation)
}
// @todo handle db error
err := r.batchOrm.UpdateCommitTxHashAndRollupStatus(r.ctx, batchHash.(string), confirmation.TxHash.String(), status)
@@ -603,7 +604,8 @@ func (r *Layer2Relayer) handleConfirmation(confirmation *sender.Confirmation) {
status = types.RollupFinalized
} else {
status = types.RollupFinalizeFailed
log.Warn("transaction confirmed but failed in layer1", "confirmation", confirmation)
r.metrics.rollupL2BatchesFinalizedConfirmedFailedTotal.Inc()
log.Warn("finalizeBatchWithProof transaction confirmed but failed in layer1", "confirmation", confirmation)
}
// @todo handle db error

View File

@@ -16,7 +16,9 @@ type l2RelayerMetrics struct {
rollupL2RelayerProcessCommittedBatchesFinalizedTotal prometheus.Counter
rollupL2RelayerProcessCommittedBatchesFinalizedSuccessTotal prometheus.Counter
rollupL2BatchesCommittedConfirmedTotal prometheus.Counter
rollupL2BatchesCommittedConfirmedFailedTotal prometheus.Counter
rollupL2BatchesFinalizedConfirmedTotal prometheus.Counter
rollupL2BatchesFinalizedConfirmedFailedTotal prometheus.Counter
rollupL2BatchesGasOraclerConfirmedTotal prometheus.Counter
rollupL2ChainMonitorLatestFailedCall prometheus.Counter
rollupL2ChainMonitorLatestFailedBatchStatus prometheus.Counter
@@ -62,10 +64,18 @@ func initL2RelayerMetrics(reg prometheus.Registerer) *l2RelayerMetrics {
Name: "rollup_layer2_process_committed_batches_confirmed_total",
Help: "The total number of layer2 process committed batches confirmed total",
}),
rollupL2BatchesCommittedConfirmedFailedTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
Name: "rollup_layer2_process_committed_batches_confirmed_failed_total",
Help: "The total number of layer2 process committed batches confirmed failed total",
}),
rollupL2BatchesFinalizedConfirmedTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
Name: "rollup_layer2_process_finalized_batches_confirmed_total",
Help: "The total number of layer2 process finalized batches confirmed total",
}),
rollupL2BatchesFinalizedConfirmedFailedTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
Name: "rollup_layer2_process_finalized_batches_confirmed_failed_total",
Help: "The total number of layer2 process finalized batches confirmed failed total",
}),
rollupL2BatchesGasOraclerConfirmedTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
Name: "rollup_layer2_process_gras_oracler_confirmed_total",
Help: "The total number of layer2 process finalized batches confirmed total",

View File

@@ -180,6 +180,9 @@ func (w *L2WatcherClient) getAndStoreBlockTraces(ctx context.Context, from, to u
}
if len(blocks) > 0 {
for _, block := range blocks {
w.metrics.rollupL2BlockL1CommitCalldataSize.Set(float64(block.EstimateL1CommitCalldataSize()))
}
if err := w.l2BlockOrm.InsertL2Blocks(w.ctx, blocks); err != nil {
return fmt.Errorf("failed to batch insert BlockTraces: %v", err)
}

View File

@@ -8,12 +8,13 @@ import (
)
type l2WatcherMetrics struct {
fetchRunningMissingBlocksTotal prometheus.Counter
fetchRunningMissingBlocksHeight prometheus.Gauge
fetchContractEventTotal prometheus.Counter
fetchContractEventHeight prometheus.Gauge
rollupL2MsgsRelayedEventsTotal prometheus.Counter
rollupL2BlocksFetchedGap prometheus.Gauge
fetchRunningMissingBlocksTotal prometheus.Counter
fetchRunningMissingBlocksHeight prometheus.Gauge
fetchContractEventTotal prometheus.Counter
fetchContractEventHeight prometheus.Gauge
rollupL2MsgsRelayedEventsTotal prometheus.Counter
rollupL2BlocksFetchedGap prometheus.Gauge
rollupL2BlockL1CommitCalldataSize prometheus.Gauge
}
var (
@@ -48,6 +49,10 @@ func initL2WatcherMetrics(reg prometheus.Registerer) *l2WatcherMetrics {
Name: "rollup_l2_watcher_blocks_fetched_gap",
Help: "The gap of l2 fetch",
}),
rollupL2BlockL1CommitCalldataSize: promauto.With(reg).NewGauge(prometheus.GaugeOpts{
Name: "rollup_l2_block_l1_commit_calldata_size",
Help: "The l1 commitBatch calldata size of the l2 block",
}),
}
})
return l2WatcherMetric