feat(rollup-relayer): add l1 commit estimation fields in batch schema (#891)

Co-authored-by: colinlyguo <colinlyguo@users.noreply.github.com>
Co-authored-by: Péter Garamvölgyi <peter@scroll.io>

bridge/internal/controller/relayer/l2_relayer.go

@@ -192,8 +192,14 @@ func (r *Layer2Relayer) initializeGenesis() error {
 			return fmt.Errorf("failed to update genesis chunk proving status: %v", err)
 		}
 
+		batchMeta := &types.BatchMeta{
+			StartChunkIndex: 0,
+			StartChunkHash:  dbChunk.Hash,
+			EndChunkIndex:   0,
+			EndChunkHash:    dbChunk.Hash,
+		}
 		var batch *orm.Batch
-		batch, err = r.batchOrm.InsertBatch(r.ctx, 0, 0, dbChunk.Hash, dbChunk.Hash, []*types.Chunk{chunk}, dbTX)
+		batch, err = r.batchOrm.InsertBatch(r.ctx, []*types.Chunk{chunk}, batchMeta, dbTX)
 		if err != nil {
 			return fmt.Errorf("failed to insert batch: %v", err)
 		}

bridge/internal/controller/relayer/l2_relayer_test.go

@@ -56,8 +56,14 @@ func testL2RelayerProcessPendingBatches(t *testing.T) {
 	assert.NoError(t, err)
 	dbChunk2, err := chunkOrm.InsertChunk(context.Background(), chunk2)
 	assert.NoError(t, err)
+	batchMeta := &types.BatchMeta{
+		StartChunkIndex: 0,
+		StartChunkHash:  dbChunk1.Hash,
+		EndChunkIndex:   1,
+		EndChunkHash:    dbChunk2.Hash,
+	}
 	batchOrm := orm.NewBatch(db)
-	batch, err := batchOrm.InsertBatch(context.Background(), 0, 1, dbChunk1.Hash, dbChunk2.Hash, []*types.Chunk{chunk1, chunk2})
+	batch, err := batchOrm.InsertBatch(context.Background(), []*types.Chunk{chunk1, chunk2}, batchMeta)
 	assert.NoError(t, err)
 
 	relayer.ProcessPendingBatches()
@@ -75,8 +81,14 @@ func testL2RelayerProcessCommittedBatches(t *testing.T) {
 	l2Cfg := cfg.L2Config
 	relayer, err := NewLayer2Relayer(context.Background(), l2Cli, db, l2Cfg.RelayerConfig, false, nil)
 	assert.NoError(t, err)
+	batchMeta := &types.BatchMeta{
+		StartChunkIndex: 0,
+		StartChunkHash:  chunkHash1.Hex(),
+		EndChunkIndex:   1,
+		EndChunkHash:    chunkHash2.Hex(),
+	}
 	batchOrm := orm.NewBatch(db)
-	batch, err := batchOrm.InsertBatch(context.Background(), 0, 1, chunkHash1.Hex(), chunkHash2.Hex(), []*types.Chunk{chunk1, chunk2})
+	batch, err := batchOrm.InsertBatch(context.Background(), []*types.Chunk{chunk1, chunk2}, batchMeta)
 	assert.NoError(t, err)
 
 	err = batchOrm.UpdateRollupStatus(context.Background(), batch.Hash, types.RollupCommitted)
@@ -124,7 +136,13 @@ func testL2RelayerCommitConfirm(t *testing.T) {
 	batchOrm := orm.NewBatch(db)
 	batchHashes := make([]string, len(processingKeys))
 	for i := range batchHashes {
-		batch, err := batchOrm.InsertBatch(context.Background(), 0, 1, chunkHash1.Hex(), chunkHash2.Hex(), []*types.Chunk{chunk1, chunk2})
+		batchMeta := &types.BatchMeta{
+			StartChunkIndex: 0,
+			StartChunkHash:  chunkHash1.Hex(),
+			EndChunkIndex:   1,
+			EndChunkHash:    chunkHash2.Hex(),
+		}
+		batch, err := batchOrm.InsertBatch(context.Background(), []*types.Chunk{chunk1, chunk2}, batchMeta)
 		assert.NoError(t, err)
 		batchHashes[i] = batch.Hash
 	}
@@ -174,7 +192,13 @@ func testL2RelayerFinalizeConfirm(t *testing.T) {
 	batchOrm := orm.NewBatch(db)
 	batchHashes := make([]string, len(processingKeys))
 	for i := range batchHashes {
-		batch, err := batchOrm.InsertBatch(context.Background(), 0, 1, chunkHash1.Hex(), chunkHash2.Hex(), []*types.Chunk{chunk1, chunk2})
+		batchMeta := &types.BatchMeta{
+			StartChunkIndex: 0,
+			StartChunkHash:  chunkHash1.Hex(),
+			EndChunkIndex:   1,
+			EndChunkHash:    chunkHash2.Hex(),
+		}
+		batch, err := batchOrm.InsertBatch(context.Background(), []*types.Chunk{chunk1, chunk2}, batchMeta)
 		assert.NoError(t, err)
 		batchHashes[i] = batch.Hash
 	}
@@ -210,11 +234,23 @@ func testL2RelayerGasOracleConfirm(t *testing.T) {
 	db := setupL2RelayerDB(t)
 	defer database.CloseDB(db)
 
+	batchMeta1 := &types.BatchMeta{
+		StartChunkIndex: 0,
+		StartChunkHash:  chunkHash1.Hex(),
+		EndChunkIndex:   0,
+		EndChunkHash:    chunkHash1.Hex(),
+	}
 	batchOrm := orm.NewBatch(db)
-	batch1, err := batchOrm.InsertBatch(context.Background(), 0, 0, chunkHash1.Hex(), chunkHash1.Hex(), []*types.Chunk{chunk1})
+	batch1, err := batchOrm.InsertBatch(context.Background(), []*types.Chunk{chunk1}, batchMeta1)
 	assert.NoError(t, err)
 
-	batch2, err := batchOrm.InsertBatch(context.Background(), 1, 1, chunkHash2.Hex(), chunkHash2.Hex(), []*types.Chunk{chunk2})
+	batchMeta2 := &types.BatchMeta{
+		StartChunkIndex: 1,
+		StartChunkHash:  chunkHash2.Hex(),
+		EndChunkIndex:   1,
+		EndChunkHash:    chunkHash2.Hex(),
+	}
+	batch2, err := batchOrm.InsertBatch(context.Background(), []*types.Chunk{chunk2}, batchMeta2)
 	assert.NoError(t, err)
 
 	// Create and set up the Layer2 Relayer.

bridge/internal/controller/watcher/batch_proposer.go

@@ -104,19 +104,19 @@ func NewBatchProposer(ctx context.Context, cfg *config.BatchProposerConfig, db *
 // TryProposeBatch tries to propose a new batches.
 func (p *BatchProposer) TryProposeBatch() {
 	p.batchProposerCircleTotal.Inc()
-	dbChunks, err := p.proposeBatchChunks()
+	dbChunks, batchMeta, err := p.proposeBatchChunks()
 	if err != nil {
 		p.proposeBatchFailureTotal.Inc()
 		log.Error("proposeBatchChunks failed", "err", err)
 		return
 	}
-	if err := p.updateBatchInfoInDB(dbChunks); err != nil {
+	if err := p.updateBatchInfoInDB(dbChunks, batchMeta); err != nil {
 		p.proposeBatchUpdateInfoFailureTotal.Inc()
 		log.Error("update batch info in db failed", "err", err)
 	}
 }
 
-func (p *BatchProposer) updateBatchInfoInDB(dbChunks []*orm.Chunk) error {
+func (p *BatchProposer) updateBatchInfoInDB(dbChunks []*orm.Chunk, batchMeta *types.BatchMeta) error {
 	p.proposeBatchUpdateInfoTotal.Inc()
 	numChunks := len(dbChunks)
 	if numChunks <= 0 {
@@ -127,17 +127,18 @@ func (p *BatchProposer) updateBatchInfoInDB(dbChunks []*orm.Chunk) error {
 		return err
 	}
 
-	startChunkIndex := dbChunks[0].Index
-	startChunkHash := dbChunks[0].Hash
-	endChunkIndex := dbChunks[numChunks-1].Index
-	endChunkHash := dbChunks[numChunks-1].Hash
+	batchMeta.StartChunkIndex = dbChunks[0].Index
+	batchMeta.StartChunkHash = dbChunks[0].Hash
+	batchMeta.EndChunkIndex = dbChunks[numChunks-1].Index
+	batchMeta.EndChunkHash = dbChunks[numChunks-1].Hash
 	err = p.db.Transaction(func(dbTX *gorm.DB) error {
-		batch, dbErr := p.batchOrm.InsertBatch(p.ctx, startChunkIndex, endChunkIndex, startChunkHash, endChunkHash, chunks, dbTX)
+		batch, dbErr := p.batchOrm.InsertBatch(p.ctx, chunks, batchMeta, dbTX)
 		if dbErr != nil {
-			log.Warn("BatchProposer.updateBatchInfoInDB insert batch failure", "error", "start chunk index", startChunkIndex, "end chunk index", endChunkIndex, dbErr)
+			log.Warn("BatchProposer.updateBatchInfoInDB insert batch failure",
+				"start chunk index", batchMeta.StartChunkIndex, "end chunk index", batchMeta.EndChunkIndex, "error", dbErr)
 			return dbErr
 		}
-		dbErr = p.chunkOrm.UpdateBatchHashInRange(p.ctx, startChunkIndex, endChunkIndex, batch.Hash, dbTX)
+		dbErr = p.chunkOrm.UpdateBatchHashInRange(p.ctx, batchMeta.StartChunkIndex, batchMeta.EndChunkIndex, batch.Hash, dbTX)
 		if dbErr != nil {
 			log.Warn("BatchProposer.UpdateBatchHashInRange update the chunk's batch hash failure", "hash", batch.Hash, "error", dbErr)
 			return dbErr
@@ -147,29 +148,30 @@ func (p *BatchProposer) updateBatchInfoInDB(dbChunks []*orm.Chunk) error {
 	return err
 }
 
-func (p *BatchProposer) proposeBatchChunks() ([]*orm.Chunk, error) {
+func (p *BatchProposer) proposeBatchChunks() ([]*orm.Chunk, *types.BatchMeta, error) {
 	unbatchedChunkIndex, err := p.batchOrm.GetFirstUnbatchedChunkIndex(p.ctx)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	dbChunks, err := p.chunkOrm.GetChunksGEIndex(p.ctx, unbatchedChunkIndex, int(p.maxChunkNumPerBatch)+1)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	if len(dbChunks) == 0 {
-		return nil, nil
+		return nil, nil, nil
 	}
 
 	var totalL1CommitCalldataSize uint32
 	var totalL1CommitGas uint64
 	var totalChunks uint64
 	var totalL1MessagePopped uint64
+	var batchMeta types.BatchMeta
 
 	parentBatch, err := p.batchOrm.GetLatestBatch(p.ctx)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	// Add extra gas costs
@@ -189,8 +191,8 @@ func (p *BatchProposer) proposeBatchChunks() ([]*orm.Chunk, error) {
 
 	for i, chunk := range dbChunks {
 		// metric values
-		lastTotalL1CommitCalldataSize := totalL1CommitCalldataSize
-		lastTotalL1CommitGas := totalL1CommitGas
+		batchMeta.TotalL1CommitGas = totalL1CommitGas
+		batchMeta.TotalL1CommitCalldataSize = totalL1CommitCalldataSize
 
 		totalL1CommitCalldataSize += chunk.TotalL1CommitCalldataSize
 		totalL1CommitGas += chunk.TotalL1CommitGas
@@ -212,7 +214,7 @@ func (p *BatchProposer) proposeBatchChunks() ([]*orm.Chunk, error) {
 			// If so, it indicates there are bugs in chunk-proposer, manual fix is needed.
 			if i == 0 {
 				if totalOverEstimateL1CommitGas > p.maxL1CommitGasPerBatch {
-					return nil, fmt.Errorf(
+					return nil, nil, fmt.Errorf(
 						"the first chunk exceeds l1 commit gas limit; start block number: %v, end block number: %v, commit gas: %v, max commit gas limit: %v",
 						dbChunks[0].StartBlockNumber,
 						dbChunks[0].EndBlockNumber,
@@ -221,7 +223,7 @@ func (p *BatchProposer) proposeBatchChunks() ([]*orm.Chunk, error) {
 					)
 				}
 				if totalL1CommitCalldataSize > p.maxL1CommitCalldataSizePerBatch {
-					return nil, fmt.Errorf(
+					return nil, nil, fmt.Errorf(
 						"the first chunk exceeds l1 commit calldata size limit; start block number: %v, end block number %v, calldata size: %v, max calldata size limit: %v",
 						dbChunks[0].StartBlockNumber,
 						dbChunks[0].EndBlockNumber,
@@ -239,10 +241,10 @@ func (p *BatchProposer) proposeBatchChunks() ([]*orm.Chunk, error) {
 				"currentOverEstimateL1CommitGas", totalOverEstimateL1CommitGas,
 				"maxL1CommitGasPerBatch", p.maxL1CommitGasPerBatch)
 
-			p.totalL1CommitGas.Set(float64(lastTotalL1CommitGas))
-			p.totalL1CommitCalldataSize.Set(float64(lastTotalL1CommitCalldataSize))
+			p.totalL1CommitGas.Set(float64(batchMeta.TotalL1CommitGas))
+			p.totalL1CommitCalldataSize.Set(float64(batchMeta.TotalL1CommitCalldataSize))
 			p.batchChunksNum.Set(float64(i))
-			return dbChunks[:i], nil
+			return dbChunks[:i], &batchMeta, nil
 		}
 	}
 
@@ -253,16 +255,18 @@ func (p *BatchProposer) proposeBatchChunks() ([]*orm.Chunk, error) {
 			"first block timestamp", dbChunks[0].StartBlockTime,
 			"chunk outdated time threshold", currentTimeSec,
 		)
+		batchMeta.TotalL1CommitGas = totalL1CommitGas
+		batchMeta.TotalL1CommitCalldataSize = totalL1CommitCalldataSize
 		p.batchFirstBlockTimeoutReached.Inc()
-		p.totalL1CommitGas.Set(float64(totalL1CommitGas))
-		p.totalL1CommitCalldataSize.Set(float64(totalL1CommitCalldataSize))
+		p.totalL1CommitGas.Set(float64(batchMeta.TotalL1CommitGas))
+		p.totalL1CommitCalldataSize.Set(float64(batchMeta.TotalL1CommitCalldataSize))
 		p.batchChunksNum.Set(float64(len(dbChunks)))
-		return dbChunks, nil
+		return dbChunks, &batchMeta, nil
 	}
 
 	log.Debug("pending chunks do not reach one of the constraints or contain a timeout block")
 	p.batchChunksProposeNotEnoughTotal.Inc()
-	return nil, nil
+	return nil, nil, nil
 }
 
 func (p *BatchProposer) dbChunksToBridgeChunks(dbChunks []*orm.Chunk) ([]*types.Chunk, error) {

bridge/internal/orm/batch.go

@@ -53,9 +53,11 @@ type Batch struct {
 	OracleTxHash string `json:"oracle_tx_hash" gorm:"column:oracle_tx_hash;default:NULL"`
 
 	// metadata
-	CreatedAt time.Time      `json:"created_at" gorm:"column:created_at"`
-	UpdatedAt time.Time      `json:"updated_at" gorm:"column:updated_at"`
-	DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"column:deleted_at;default:NULL"`
+	TotalL1CommitGas          uint64         `json:"total_l1_commit_gas" gorm:"column:total_l1_commit_gas;default:0"`
+	TotalL1CommitCalldataSize uint32         `json:"total_l1_commit_calldata_size" gorm:"column:total_l1_commit_calldata_size;default:0"`
+	CreatedAt                 time.Time      `json:"created_at" gorm:"column:created_at"`
+	UpdatedAt                 time.Time      `json:"updated_at" gorm:"column:updated_at"`
+	DeletedAt                 gorm.DeletedAt `json:"deleted_at" gorm:"column:deleted_at;default:NULL"`
 }
 
 // NewBatch creates a new Batch database instance.
@@ -224,7 +226,7 @@ func (o *Batch) GetBatchByIndex(ctx context.Context, index uint64) (*Batch, erro
 }
 
 // InsertBatch inserts a new batch into the database.
-func (o *Batch) InsertBatch(ctx context.Context, startChunkIndex, endChunkIndex uint64, startChunkHash, endChunkHash string, chunks []*types.Chunk, dbTX ...*gorm.DB) (*Batch, error) {
+func (o *Batch) InsertBatch(ctx context.Context, chunks []*types.Chunk, batchMeta *types.BatchMeta, dbTX ...*gorm.DB) (*Batch, error) {
 	if len(chunks) == 0 {
 		return nil, errors.New("invalid args")
 	}
@@ -270,20 +272,22 @@ func (o *Batch) InsertBatch(ctx context.Context, startChunkIndex, endChunkIndex
 	lastChunkBlockNum := len(chunks[numChunks-1].Blocks)
 
 	newBatch := Batch{
-		Index:             batchIndex,
-		Hash:              batchHeader.Hash().Hex(),
-		StartChunkHash:    startChunkHash,
-		StartChunkIndex:   startChunkIndex,
-		EndChunkHash:      endChunkHash,
-		EndChunkIndex:     endChunkIndex,
-		StateRoot:         chunks[numChunks-1].Blocks[lastChunkBlockNum-1].Header.Root.Hex(),
-		WithdrawRoot:      chunks[numChunks-1].Blocks[lastChunkBlockNum-1].WithdrawRoot.Hex(),
-		ParentBatchHash:   parentBatchHash.Hex(),
-		BatchHeader:       batchHeader.Encode(),
-		ChunkProofsStatus: int16(types.ChunkProofsStatusPending),
-		ProvingStatus:     int16(types.ProvingTaskUnassigned),
-		RollupStatus:      int16(types.RollupPending),
-		OracleStatus:      int16(types.GasOraclePending),
+		Index:                     batchIndex,
+		Hash:                      batchHeader.Hash().Hex(),
+		StartChunkHash:            batchMeta.StartChunkHash,
+		StartChunkIndex:           batchMeta.StartChunkIndex,
+		EndChunkHash:              batchMeta.EndChunkHash,
+		EndChunkIndex:             batchMeta.EndChunkIndex,
+		StateRoot:                 chunks[numChunks-1].Blocks[lastChunkBlockNum-1].Header.Root.Hex(),
+		WithdrawRoot:              chunks[numChunks-1].Blocks[lastChunkBlockNum-1].WithdrawRoot.Hex(),
+		ParentBatchHash:           parentBatchHash.Hex(),
+		BatchHeader:               batchHeader.Encode(),
+		ChunkProofsStatus:         int16(types.ChunkProofsStatusPending),
+		ProvingStatus:             int16(types.ProvingTaskUnassigned),
+		RollupStatus:              int16(types.RollupPending),
+		OracleStatus:              int16(types.GasOraclePending),
+		TotalL1CommitGas:          batchMeta.TotalL1CommitGas,
+		TotalL1CommitCalldataSize: batchMeta.TotalL1CommitCalldataSize,
 	}
 
 	db := o.db

bridge/internal/orm/orm_test.go

@@ -173,7 +173,13 @@ func TestBatchOrm(t *testing.T) {
 	assert.NoError(t, err)
 	assert.NoError(t, migrate.ResetDB(sqlDB))
 
-	batch1, err := batchOrm.InsertBatch(context.Background(), 0, 0, chunkHash1.Hex(), chunkHash1.Hex(), []*types.Chunk{chunk1})
+	batchMeta1 := &types.BatchMeta{
+		StartChunkIndex: 0,
+		StartChunkHash:  chunkHash1.Hex(),
+		EndChunkIndex:   0,
+		EndChunkHash:    chunkHash1.Hex(),
+	}
+	batch1, err := batchOrm.InsertBatch(context.Background(), []*types.Chunk{chunk1}, batchMeta1)
 	assert.NoError(t, err)
 	hash1 := batch1.Hash
 
@@ -184,7 +190,13 @@ func TestBatchOrm(t *testing.T) {
 	batchHash1 := batchHeader1.Hash().Hex()
 	assert.Equal(t, hash1, batchHash1)
 
-	batch2, err := batchOrm.InsertBatch(context.Background(), 1, 1, chunkHash2.Hex(), chunkHash2.Hex(), []*types.Chunk{chunk2})
+	batchMeta2 := &types.BatchMeta{
+		StartChunkIndex: 1,
+		StartChunkHash:  chunkHash2.Hex(),
+		EndChunkIndex:   1,
+		EndChunkHash:    chunkHash2.Hex(),
+	}
+	batch2, err := batchOrm.InsertBatch(context.Background(), []*types.Chunk{chunk2}, batchMeta2)
 	assert.NoError(t, err)
 	hash2 := batch2.Hash
 

bridge/tests/gas_oracle_test.go

@@ -90,8 +90,14 @@ func testImportL2GasPrice(t *testing.T) {
 	chunkHash, err := chunk.Hash(0)
 	assert.NoError(t, err)
 
+	batchMeta := &types.BatchMeta{
+		StartChunkIndex: 0,
+		StartChunkHash:  chunkHash.Hex(),
+		EndChunkIndex:   0,
+		EndChunkHash:    chunkHash.Hex(),
+	}
 	batchOrm := orm.NewBatch(db)
-	_, err = batchOrm.InsertBatch(context.Background(), 0, 0, chunkHash.Hex(), chunkHash.Hex(), []*types.Chunk{chunk})
+	_, err = batchOrm.InsertBatch(context.Background(), []*types.Chunk{chunk}, batchMeta)
 	assert.NoError(t, err)
 
 	// check db status

common/types/batch_header.go

@@ -10,6 +10,16 @@ import (
 	"github.com/scroll-tech/go-ethereum/crypto"
 )
 
+// BatchMeta contains metadata of a batch.
+type BatchMeta struct {
+	StartChunkIndex           uint64
+	StartChunkHash            string
+	EndChunkIndex             uint64
+	EndChunkHash              string
+	TotalL1CommitGas          uint64
+	TotalL1CommitCalldataSize uint32
+}
+
 // BatchHeader contains batch header info to be committed.
 type BatchHeader struct {
 	// Encoded in BatchHeaderV0Codec

common/version/version.go

@@ -7,7 +7,7 @@ import (
 	"strings"
 )
 
-var tag = "v4.2.7"
+var tag = "v4.2.8"
 
 var commit = func() string {
 	if info, ok := debug.ReadBuildInfo(); ok {

database/migrate/migrate_test.go

@@ -63,7 +63,7 @@ func testResetDB(t *testing.T) {
 	cur, err := Current(pgDB.DB)
 	assert.NoError(t, err)
 	// total number of tables.
-	assert.Equal(t, 10, int(cur))
+	assert.Equal(t, 11, int(cur))
 }
 
 func testMigrate(t *testing.T) {

database/migrate/migrations/00011_add_batch_stats_fields.sql

@@ -0,0 +1,17 @@
+-- +goose Up
+-- +goose StatementBegin
+
+ALTER TABLE batch
+ADD COLUMN total_l1_commit_gas BIGINT NOT NULL DEFAULT 0,
+ADD COLUMN total_l1_commit_calldata_size INTEGER NOT NULL DEFAULT 0;
+
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+
+ALTER TABLE IF EXISTS batch
+DROP COLUMN total_l1_commit_gas,
+DROP COLUMN total_l1_commit_calldata_size;
+
+-- +goose StatementEnd