add a dump batch task script

Co-authored-by: colinlyguo <colinlyguo@users.noreply.github.com>

.github/workflows/docker.yml

@@ -46,6 +46,7 @@ jobs:
         with:
           context: .
           file: ./build/dockerfiles/event_watcher.Dockerfile
+          platforms: linux/amd64,linux/arm64
           push: true
           tags: |
             ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.REPOSITORY }}:${{ env.IMAGE_TAG }}
@@ -90,6 +91,7 @@ jobs:
         with:
           context: .
           file: ./build/dockerfiles/gas_oracle.Dockerfile
+          platforms: linux/amd64,linux/arm64
           push: true
           tags: |
             ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.REPOSITORY }}:${{ env.IMAGE_TAG }}
@@ -134,6 +136,7 @@ jobs:
         with:
           context: .
           file: ./build/dockerfiles/rollup_relayer.Dockerfile
+          platforms: linux/amd64,linux/arm64
           push: true
           tags: |
             ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.REPOSITORY }}:${{ env.IMAGE_TAG }}
@@ -178,6 +181,7 @@ jobs:
         with:
           context: .
           file: ./build/dockerfiles/bridgehistoryapi-fetcher.Dockerfile
+          platforms: linux/amd64,linux/arm64
           push: true
           tags: |
             ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.REPOSITORY }}:${{ env.IMAGE_TAG }}
@@ -222,6 +226,7 @@ jobs:
         with:
           context: .
           file: ./build/dockerfiles/bridgehistoryapi-api.Dockerfile
+          platforms: linux/amd64,linux/arm64
           push: true
           tags: |
             ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.REPOSITORY }}:${{ env.IMAGE_TAG }}
@@ -266,6 +271,7 @@ jobs:
         with:
           context: .
           file: ./build/dockerfiles/coordinator-api.Dockerfile
+          platforms: linux/amd64,linux/arm64
           push: true
           tags: |
             ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.REPOSITORY }}:${{ env.IMAGE_TAG }}
@@ -310,6 +316,7 @@ jobs:
         with:
           context: .
           file: ./build/dockerfiles/coordinator-cron.Dockerfile
+          platforms: linux/amd64,linux/arm64
           push: true
           tags: |
             ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.REPOSITORY }}:${{ env.IMAGE_TAG }}

common/testcontainers/testcontainers.go

@@ -61,8 +61,11 @@ func (t *TestcontainerApps) StartL1GethContainer() error {
 	req := testcontainers.ContainerRequest{
 		Image:        "scroll_l1geth",
 		ExposedPorts: []string{"8546/tcp", "8545/tcp"},
-		WaitingFor:   wait.ForHTTP("/").WithPort("8545").WithStartupTimeout(100 * time.Second),
-		Cmd:          []string{"--log.debug", "ANY"},
+		WaitingFor: wait.ForAll(
+			wait.ForListeningPort("8546").WithStartupTimeout(100*time.Second),
+			wait.ForListeningPort("8545").WithStartupTimeout(100*time.Second),
+		),
+		Cmd: []string{"--log.debug", "ANY"},
 	}
 	genericContainerReq := testcontainers.GenericContainerRequest{
 		ContainerRequest: req,
@@ -85,7 +88,10 @@ func (t *TestcontainerApps) StartL2GethContainer() error {
 	req := testcontainers.ContainerRequest{
 		Image:        "scroll_l2geth",
 		ExposedPorts: []string{"8546/tcp", "8545/tcp"},
-		WaitingFor:   wait.ForHTTP("/").WithPort("8545").WithStartupTimeout(100 * time.Second),
+		WaitingFor: wait.ForAll(
+			wait.ForListeningPort("8546").WithStartupTimeout(100*time.Second),
+			wait.ForListeningPort("8545").WithStartupTimeout(100*time.Second),
+		),
 	}
 	genericContainerReq := testcontainers.GenericContainerRequest{
 		ContainerRequest: req,

common/version/version.go

@@ -5,7 +5,7 @@ import (
 	"runtime/debug"
 )
 
-var tag = "v4.3.92"
+var tag = "v4.3.93"
 
 var commit = func() string {
 	if info, ok := debug.ReadBuildInfo(); ok {

coordinator/README.md

@@ -38,7 +38,7 @@ make lint
 
 ## Configure
 
-The coordinator behavior can be configured using [`config.json`](config.json). Check the code comments under `ProverManager` in [`config/config.go`](config/config.go) for more details.
+The coordinator behavior can be configured using [`conf/config.json`](conf/config.json). Check the code comments under `ProverManager` in [`internal/config/config.go`](internal/config/config.go) for more details.
 
 
 ## Start

coordinator/internal/orm/batch.go

@@ -75,6 +75,18 @@ func (*Batch) TableName() string {
 	return "batch"
 }
 
+// GetBatchHashByIndex retrieves the hash of the batch given its index.
+// used for the script.
+func (o *Batch) GetBatchHashByIndex(ctx context.Context, index uint64) (string, error) {
+	db := o.db.WithContext(ctx)
+	var batch Batch
+	err := db.Select("hash").Where("index = ?", index).First(&batch).Error
+	if err != nil {
+		return "", fmt.Errorf("Batch.GetBatchHashByIndex error: %w, index: %v", err, index)
+	}
+	return batch.Hash, nil
+}
+
 // GetUnassignedBatch retrieves unassigned batch based on the specified limit.
 // The returned batch are sorted in ascending order by their index.
 func (o *Batch) GetUnassignedBatch(ctx context.Context, startChunkIndex, endChunkIndex uint64, maxActiveAttempts, maxTotalAttempts uint8) (*Batch, error) {

coordinator/internal/script/main.go

@@ -0,0 +1,121 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/scroll-tech/go-ethereum/common"
+	"github.com/scroll-tech/go-ethereum/log"
+	"gorm.io/gorm"
+
+	"scroll-tech/common/database"
+	"scroll-tech/common/types/message"
+	"scroll-tech/coordinator/internal/orm"
+)
+
+func main() {
+	glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.LogfmtFormat()))
+	glogger.Verbosity(log.LvlInfo)
+	log.Root().SetHandler(glogger)
+
+	if len(os.Args) < 2 {
+		log.Crit("no batch index range provided")
+		return
+	}
+
+	indexRange := os.Args[1]
+	indices := strings.Split(indexRange, "-")
+	if len(indices) != 2 {
+		log.Crit("invalid batch index range format. Use start-end", "providedRange", indexRange)
+		return
+	}
+
+	startIndex, err := strconv.Atoi(indices[0])
+	endIndex, err2 := strconv.Atoi(indices[1])
+	if err != nil || err2 != nil || startIndex > endIndex {
+		log.Crit("invalid batch index range", "start", indices[0], "end", indices[1], "err", err, "err2", err2)
+		return
+	}
+
+	db, err := database.InitDB(&database.Config{
+		DriverName: "postgres",
+		DSN:        os.Getenv("DB_DSN"),
+		MaxOpenNum: 200,
+		MaxIdleNum: 20,
+	})
+	if err != nil {
+		log.Crit("failed to init db", "err", err)
+	}
+	defer func() {
+		if deferErr := database.CloseDB(db); deferErr != nil {
+			log.Error("failed to close db", "err", err)
+		}
+	}()
+
+	for i := startIndex; i <= endIndex; i++ {
+		batchIndex := uint64(i)
+		resultBytes, err := getBatchTask(db, batchIndex)
+		if err != nil {
+			log.Crit("failed to get batch task", "batchIndex", batchIndex, "err", err)
+			continue
+		}
+
+		outputFilename := fmt.Sprintf("batch_task_%d.json", batchIndex)
+		if err = os.WriteFile(outputFilename, resultBytes, 0644); err != nil {
+			log.Crit("failed to write output file", "filename", outputFilename, "err", err)
+		}
+	}
+}
+
+func getBatchTask(db *gorm.DB, batchIndex uint64) ([]byte, error) {
+	batchHash, err := orm.NewBatch(db).GetBatchHashByIndex(context.Background(), batchIndex)
+	if err != nil {
+		err = fmt.Errorf("failed to get batch hash by index: %d err: %w ", batchIndex, err)
+		return nil, err
+	}
+
+	chunks, err := orm.NewChunk(db).GetChunksByBatchHash(context.Background(), batchHash)
+	if err != nil {
+		err = fmt.Errorf("failed to get chunk proofs for batch task id: %s err: %w ", batchHash, err)
+		return nil, err
+	}
+
+	var chunkProofs []*message.ChunkProof
+	var chunkInfos []*message.ChunkInfo
+	for _, chunk := range chunks {
+		var proof message.ChunkProof
+		if encodeErr := json.Unmarshal(chunk.Proof, &proof); encodeErr != nil {
+			return nil, fmt.Errorf("Chunk.GetProofsByBatchHash unmarshal proof error: %w, batch hash: %v, chunk hash: %v", encodeErr, batchHash, chunk.Hash)
+		}
+		chunkProofs = append(chunkProofs, &proof)
+
+		chunkInfo := message.ChunkInfo{
+			ChainID:       534351, // sepolia
+			PrevStateRoot: common.HexToHash(chunk.ParentChunkStateRoot),
+			PostStateRoot: common.HexToHash(chunk.StateRoot),
+			WithdrawRoot:  common.HexToHash(chunk.WithdrawRoot),
+			DataHash:      common.HexToHash(chunk.Hash),
+			IsPadding:     false,
+		}
+		if proof.ChunkInfo != nil {
+			chunkInfo.TxBytes = proof.ChunkInfo.TxBytes
+		}
+		chunkInfos = append(chunkInfos, &chunkInfo)
+	}
+
+	taskDetail := message.BatchTaskDetail{
+		ChunkInfos:  chunkInfos,
+		ChunkProofs: chunkProofs,
+	}
+
+	chunkProofsBytes, err := json.MarshalIndent(taskDetail, "", "    ")
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal chunk proofs, taskID:%s err:%w", batchHash, err)
+	}
+
+	return chunkProofsBytes, nil
+}

rollup/Makefile

@@ -24,7 +24,7 @@ rollup_relayer: ## Builds the rollup_relayer bin
 test:
 	go test -v -race -coverprofile=coverage.txt -covermode=atomic -p 1 $(PWD)/...
 
-lint: ## Lint the files - used for CI
+lint: mock_abi ## Lint the files - used for CI
 	GOBIN=$(PWD)/build/bin go run ../build/lint.go
 
 clean: ## Empty out the bin folder

rollup/internal/config/l2.go

@@ -42,7 +42,4 @@ type BatchProposerConfig struct {
 	MaxL1CommitCalldataSizePerBatch uint64  `json:"max_l1_commit_calldata_size_per_batch"`
 	BatchTimeoutSec                 uint64  `json:"batch_timeout_sec"`
 	GasCostIncreaseMultiplier       float64 `json:"gas_cost_increase_multiplier"`
-
-	EnableTestEnvSamplingFeature bool   `json:"enable_test_env_sampling_feature,omitempty"`
-	SamplingPercentage           uint64 `json:"sampling_percentage,omitempty"`
 }

rollup/internal/config/relayer.go

@@ -64,9 +64,6 @@ type RelayerConfig struct {
 	EnableTestEnvBypassFeatures bool `json:"enable_test_env_bypass_features"`
 	// The timeout in seconds for finalizing a batch without proof, only used when EnableTestEnvBypassFeatures is true.
 	FinalizeBatchWithoutProofTimeoutSec uint64 `json:"finalize_batch_without_proof_timeout_sec"`
-
-	EnableTestEnvSamplingFeature bool   `json:"enable_test_env_sampling_feature,omitempty"`
-	SamplingPercentage           uint64 `json:"sampling_percentage,omitempty"`
 }
 
 // GasOracleConfig The config for updating gas price oracle.
@@ -131,10 +128,6 @@ func (r *RelayerConfig) UnmarshalJSON(input []byte) error {
 		return fmt.Errorf("error converting and checking finalize sender private key: %w", err)
 	}
 
-	if r.SamplingPercentage == 0 {
-		r.SamplingPercentage = 100
-	}
-
 	return nil
 }
 

rollup/internal/controller/relayer/l2_relayer.go

@@ -464,9 +464,8 @@ func (r *Layer2Relayer) ProcessCommittedBatches() {
 	case types.ProvingTaskVerified:
 		log.Info("Start to roll up zk proof", "hash", batch.Hash)
 		r.metrics.rollupL2RelayerProcessCommittedBatchesFinalizedTotal.Inc()
-		skipProof := r.cfg.EnableTestEnvSamplingFeature && ((batch.Index % 100) >= r.cfg.SamplingPercentage)
-		if err := r.finalizeBatch(batch, !skipProof); err != nil {
-			log.Error("Failed to finalize batch", "index", batch.Index, "hash", batch.Hash, "withProof", !skipProof, "err", err)
+		if err := r.finalizeBatch(batch, true); err != nil {
+			log.Error("Failed to finalize batch with proof", "index", batch.Index, "hash", batch.Hash, "err", err)
 		}
 
 	case types.ProvingTaskFailed:
@@ -579,7 +578,7 @@ func (r *Layer2Relayer) finalizeBatch(dbBatch *orm.Batch, withProof bool) error
 		return err
 	}
 
-	log.Info("finalizeBatch in layer1", "with proof", withProof, "index", dbBatch.Index, "batch hash", dbBatch.Hash, "tx hash", txHash)
+	log.Info("finalizeBatch in layer1", "with proof", withProof, "index", dbBatch.Index, "batch hash", dbBatch.Hash, "tx hash", txHash.String())
 
 	// record and sync with db, @todo handle db error
 	if err := r.batchOrm.UpdateFinalizeTxHashAndRollupStatus(r.ctx, dbBatch.Hash, txHash.String(), types.RollupFinalizing); err != nil {
@@ -587,22 +586,22 @@ func (r *Layer2Relayer) finalizeBatch(dbBatch *orm.Batch, withProof bool) error
 		return err
 	}
 
-	// // Updating the proving status when finalizing without proof, thus the coordinator could omit this task.
-	// // it isn't a necessary step, so don't put in a transaction with UpdateFinalizeTxHashAndRollupStatus
-	// if !withProof {
-	// 	txErr := r.db.Transaction(func(tx *gorm.DB) error {
-	// 		if updateErr := r.batchOrm.UpdateProvingStatus(r.ctx, dbBatch.Hash, types.ProvingTaskVerified); updateErr != nil {
-	// 			return updateErr
-	// 		}
-	// 		if updateErr := r.chunkOrm.UpdateProvingStatusByBatchHash(r.ctx, dbBatch.Hash, types.ProvingTaskVerified); updateErr != nil {
-	// 			return updateErr
-	// 		}
-	// 		return nil
-	// 	})
-	// 	if txErr != nil {
-	// 		log.Error("Updating chunk and batch proving status when finalizing without proof failure", "batchHash", dbBatch.Hash, "err", txErr)
-	// 	}
-	// }
+	// Updating the proving status when finalizing without proof, thus the coordinator could omit this task.
+	// it isn't a necessary step, so don't put in a transaction with UpdateFinalizeTxHashAndRollupStatus
+	if !withProof {
+		txErr := r.db.Transaction(func(tx *gorm.DB) error {
+			if updateErr := r.batchOrm.UpdateProvingStatus(r.ctx, dbBatch.Hash, types.ProvingTaskVerified); updateErr != nil {
+				return updateErr
+			}
+			if updateErr := r.chunkOrm.UpdateProvingStatusByBatchHash(r.ctx, dbBatch.Hash, types.ProvingTaskVerified); updateErr != nil {
+				return updateErr
+			}
+			return nil
+		})
+		if txErr != nil {
+			log.Error("Updating chunk and batch proving status when finalizing without proof failure", "batchHash", dbBatch.Hash, "err", txErr)
+		}
+	}
 
 	r.metrics.rollupL2RelayerProcessCommittedBatchesFinalizedSuccessTotal.Inc()
 	return nil

rollup/internal/controller/watcher/batch_proposer.go

@@ -14,7 +14,6 @@ import (
 	"gorm.io/gorm"
 
 	"scroll-tech/common/forks"
-	"scroll-tech/common/types"
 	"scroll-tech/common/types/encoding"
 
 	"scroll-tech/rollup/internal/config"
@@ -38,7 +37,6 @@ type BatchProposer struct {
 	gasCostIncreaseMultiplier       float64
 	forkMap                         map[uint64]bool
 
-	cfg      *config.BatchProposerConfig
 	chainCfg *params.ChainConfig
 
 	batchProposerCircleTotal           prometheus.Counter
@@ -76,7 +74,6 @@ func NewBatchProposer(ctx context.Context, cfg *config.BatchProposerConfig, chai
 		batchTimeoutSec:                 cfg.BatchTimeoutSec,
 		gasCostIncreaseMultiplier:       cfg.GasCostIncreaseMultiplier,
 		forkMap:                         forkMap,
-		cfg:                             cfg,
 		chainCfg:                        chainCfg,
 
 		batchProposerCircleTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
@@ -147,27 +144,6 @@ func (p *BatchProposer) updateDBBatchInfo(batch *encoding.Batch, codecVersion en
 			log.Warn("BatchProposer.UpdateBatchHashInRange update the chunk's batch hash failure", "hash", batch.Hash, "error", dbErr)
 			return dbErr
 		}
-
-		skipProof := false
-		if p.cfg.EnableTestEnvSamplingFeature && ((batch.Index % 100) >= p.cfg.SamplingPercentage) {
-			skipProof = true
-		}
-		if skipProof {
-			dbErr = p.batchOrm.UpdateProvingStatus(p.ctx, batch.Hash, types.ProvingTaskVerified, dbTX)
-			if dbErr != nil {
-				log.Warn("BatchProposer.updateBatchInfoInDB update batch proving_status failure",
-					"batch hash", batch.Hash, "error", dbErr)
-				return dbErr
-			}
-			dbErr = p.chunkOrm.UpdateProvingStatusInRange(p.ctx, batch.StartChunkIndex, batch.EndChunkIndex, types.ProvingTaskVerified, dbTX)
-			if dbErr != nil {
-				log.Warn("BatchProposer.updateBatchInfoInDB update chunk proving_status failure",
-					"start chunk index", batch.StartChunkIndex, "end chunk index", batch.EndChunkIndex,
-					"batch hash", batch.Hash, "error", dbErr)
-				return dbErr
-			}
-		}
-
 		return nil
 	})
 	if err != nil {

rollup/internal/orm/chunk.go

@@ -306,28 +306,3 @@ func (o *Chunk) UpdateBatchHashInRange(ctx context.Context, startIndex uint64, e
 	}
 	return nil
 }
-func (o *Chunk) UpdateProvingStatusInRange(ctx context.Context, startIndex uint64, endIndex uint64, status types.ProvingStatus, dbTX ...*gorm.DB) error {
-	db := o.db
-	if len(dbTX) > 0 && dbTX[0] != nil {
-		db = dbTX[0]
-	}
-	db = db.WithContext(ctx)
-	db = db.Model(&Chunk{})
-	db = db.Where("index >= ? AND index <= ?", startIndex, endIndex)
-
-	updateFields := make(map[string]interface{})
-	updateFields["proving_status"] = int(status)
-	switch status {
-	case types.ProvingTaskAssigned:
-		updateFields["prover_assigned_at"] = time.Now()
-	case types.ProvingTaskUnassigned:
-		updateFields["prover_assigned_at"] = nil
-	case types.ProvingTaskVerified:
-		updateFields["proved_at"] = time.Now()
-	}
-
-	if err := db.Updates(updateFields).Error; err != nil {
-		return fmt.Errorf("Chunk.UpdateProvingStatusInRange error: %w, start index: %v, end index: %v, status: %v", err, startIndex, endIndex, status.String())
-	}
-	return nil
-}