fix(coordinator): recover prover_task when the related object is verified. (#1466)

Co-authored-by: amoylan2 <amoylan2@users.noreply.github.com>

Makefile

@@ -6,7 +6,7 @@ help: ## Display this help message
 	@grep -h \
 		-E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \
 		awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
-update:
+update: ## Update dependencies
 	go work sync
 	cd $(PWD)/bridge-history-api/ && go get -u github.com/scroll-tech/go-ethereum@${L2GETH_TAG} && go mod tidy
 	cd $(PWD)/common/ && go get -u github.com/scroll-tech/go-ethereum@${L2GETH_TAG}&& go mod tidy
@@ -15,14 +15,14 @@ update:
 	cd $(PWD)/rollup/ && go get -u github.com/scroll-tech/go-ethereum@${L2GETH_TAG} && go mod tidy
 	cd $(PWD)/tests/integration-test/ && go get -u github.com/scroll-tech/go-ethereum@${L2GETH_TAG} && go mod tidy
 
-lint: ## The code's format and security checks.
+lint: ## The code's format and security checks
 	make -C rollup lint
 	make -C common lint
 	make -C coordinator lint
 	make -C database lint
 	make -C bridge-history-api lint
 
-fmt: ## format the code
+fmt: ## Format the code
 	go work sync
 	cd $(PWD)/bridge-history-api/ && go mod tidy
 	cd $(PWD)/common/ && go mod tidy
@@ -38,7 +38,7 @@ fmt: ## format the code
 	goimports -local $(PWD)/rollup/ -w .
 	goimports -local $(PWD)/tests/integration-test/ -w .
 
-dev_docker: ## build docker images for development/testing usages
+dev_docker: ## Build docker images for development/testing usages
 	docker pull postgres
 	docker build -t scroll_l1geth ./common/testcontainers/docker/l1geth/
 	docker build -t scroll_l2geth ./common/testcontainers/docker/l2geth/

README.md

@@ -16,10 +16,9 @@
 ├── <a href="./common/">common</a>: Common libraries and types
 ├── <a href="./coordinator/">coordinator</a>: Prover coordinator service that dispatches proving tasks to provers
 ├── <a href="./database">database</a>: Database client and schema definition
-├── <a href="./src">l2geth</a>: Scroll execution node
 ├── <a href="./prover">prover</a>: Prover client that runs proof generation for zkEVM circuit and aggregation circuit
 ├── <a href="./rollup">rollup</a>: Rollup-related services
-├── <a href="./rpc-gateway">rpc-gateway</a>: RPC gateway external repo
+├── <a href="./scroll-contracts">scroll-contracts</a>: solidity code for Scroll L1 bridge and rollup contracts and L2 bridge and pre-deployed contracts.
 └── <a href="./tests">tests</a>: Integration tests
 </pre>
 

bridge-history-api/internal/utils/utils.go

@@ -66,32 +66,55 @@ func ComputeMessageHash(
 	return common.BytesToHash(crypto.Keccak256(data))
 }
 
-type commitBatchArgs struct {
-	Version                uint8
-	ParentBatchHeader      []byte
-	Chunks                 [][]byte
-	SkippedL1MessageBitmap []byte
-}
-
 // GetBatchRangeFromCalldata find the block range from calldata, both inclusive.
-func GetBatchRangeFromCalldata(calldata []byte) (uint64, uint64, error) {
-	method := backendabi.IScrollChainABI.Methods["commitBatch"]
-	values, err := method.Inputs.Unpack(calldata[4:])
-	if err != nil {
-		// special case: import genesis batch
-		method = backendabi.IScrollChainABI.Methods["importGenesisBatch"]
-		_, err2 := method.Inputs.Unpack(calldata[4:])
-		if err2 == nil {
-			// genesis batch
-			return 0, 0, nil
-		}
-		// none of "commitBatch" and "importGenesisBatch" match, give up
-		return 0, 0, err
+func GetBatchRangeFromCalldata(txData []byte) (uint64, uint64, error) {
+	const methodIDLength = 4
+	if len(txData) < methodIDLength {
+		return 0, 0, fmt.Errorf("transaction data is too short, length of tx data: %v, minimum length required: %v", len(txData), methodIDLength)
 	}
-	args := commitBatchArgs{}
-	err = method.Inputs.Copy(&args, values)
+	method, err := backendabi.IScrollChainABI.MethodById(txData[:methodIDLength])
 	if err != nil {
-		return 0, 0, err
+		return 0, 0, fmt.Errorf("failed to get method by ID, ID: %v, err: %w", txData[:methodIDLength], err)
+	}
+	values, err := method.Inputs.Unpack(txData[methodIDLength:])
+	if err != nil {
+		return 0, 0, fmt.Errorf("failed to unpack transaction data using ABI, tx data: %v, err: %w", txData, err)
+	}
+
+	var chunks [][]byte
+
+	if method.Name == "importGenesisBatch" {
+		return 0, 0, nil
+	} else if method.Name == "commitBatch" {
+		type commitBatchArgs struct {
+			Version                uint8
+			ParentBatchHeader      []byte
+			Chunks                 [][]byte
+			SkippedL1MessageBitmap []byte
+		}
+
+		var args commitBatchArgs
+		if err = method.Inputs.Copy(&args, values); err != nil {
+			return 0, 0, fmt.Errorf("failed to decode calldata into commitBatch args, values: %+v, err: %w", values, err)
+		}
+
+		chunks = args.Chunks
+
+	} else if method.Name == "commitBatchWithBlobProof" {
+		type commitBatchWithBlobProofArgs struct {
+			Version                uint8
+			ParentBatchHeader      []byte
+			Chunks                 [][]byte
+			SkippedL1MessageBitmap []byte
+			BlobDataProof          []byte
+		}
+
+		var args commitBatchWithBlobProofArgs
+		if err = method.Inputs.Copy(&args, values); err != nil {
+			return 0, 0, fmt.Errorf("failed to decode calldata into commitBatchWithBlobProofArgs args, values: %+v, err: %w", values, err)
+		}
+
+		chunks = args.Chunks
 	}
 
 	var startBlock uint64
@@ -100,14 +123,14 @@ func GetBatchRangeFromCalldata(calldata []byte) (uint64, uint64, error) {
 	// decode blocks from chunk and assume that there's no empty chunk
 	// |   1 byte   | 60 bytes | ... | 60 bytes |
 	// | num blocks |  block 1 | ... |  block n |
-	if len(args.Chunks) == 0 {
+	if len(chunks) == 0 {
 		return 0, 0, errors.New("invalid chunks")
 	}
-	chunk := args.Chunks[0]
+	chunk := chunks[0]
 	block := chunk[1:61] // first block in chunk
 	startBlock = binary.BigEndian.Uint64(block[0:8])
 
-	chunk = args.Chunks[len(args.Chunks)-1]
+	chunk = chunks[len(chunks)-1]
 	lastBlockIndex := int(chunk[0]) - 1
 	block = chunk[1+lastBlockIndex*60 : 1+lastBlockIndex*60+60] // last block in chunk
 	finishBlock = binary.BigEndian.Uint64(block[0:8])

build/dockerfiles/gas_oracle.Dockerfile

@@ -21,6 +21,8 @@ RUN --mount=target=. \
 # Pull gas_oracle into a second stage deploy ubuntu container
 FROM ubuntu:20.04
 
+RUN apt update && apt install ca-certificates -y
+
 ENV CGO_LDFLAGS="-ldl"
 
 COPY --from=builder /bin/gas_oracle /bin/

build/dockerfiles/rollup_relayer.Dockerfile

@@ -21,6 +21,8 @@ RUN --mount=target=. \
 # Pull rollup_relayer into a second stage deploy ubuntu container
 FROM ubuntu:20.04
 
+RUN apt update && apt install ca-certificates -y
+
 ENV CGO_LDFLAGS="-ldl"
 
 COPY --from=builder /bin/rollup_relayer /bin/

common/types/db.go

@@ -109,6 +109,10 @@ const (
 	ProverTaskFailureTypeVerifiedFailed
 	// ProverTaskFailureTypeServerError collect occur error
 	ProverTaskFailureTypeServerError
+	// ProverTaskFailureTypeObjectAlreadyVerified object(batch/chunk) already verified, may exists in test env when ENABLE_TEST_ENV_BYPASS_FEATURES is true
+	ProverTaskFailureTypeObjectAlreadyVerified
+	// ProverTaskFailureTypeReassignedByAdmin reassigned by admin, this value is used in admin-system and defined here for clarity
+	ProverTaskFailureTypeReassignedByAdmin
 )
 
 func (r ProverTaskFailureType) String() string {
@@ -123,6 +127,10 @@ func (r ProverTaskFailureType) String() string {
 		return "prover task failure verified failed"
 	case ProverTaskFailureTypeServerError:
 		return "prover task failure server exception"
+	case ProverTaskFailureTypeObjectAlreadyVerified:
+		return "prover task failure object already verified"
+	case ProverTaskFailureTypeReassignedByAdmin:
+		return "prover task failure reassigned by admin"
 	default:
 		return fmt.Sprintf("illegal prover task failure type (%d)", int32(r))
 	}

common/version/version.go

@@ -5,7 +5,7 @@ import (
 	"runtime/debug"
 )
 
-var tag = "v4.4.26"
+var tag = "v4.4.35"
 
 var commit = func() string {
 	if info, ok := debug.ReadBuildInfo(); ok {

coordinator/internal/logic/provertask/batch_prover_task.go

@@ -3,7 +3,6 @@ package provertask
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"math"
 	"time"
@@ -191,7 +190,7 @@ func (bp *BatchProverTask) doAssignTaskWithinChunkRange(ctx *gin.Context, taskCt
 func (bp *BatchProverTask) getChunkRangeByName(ctx *gin.Context, hardForkName string) (*chunkIndexRange, error) {
 	hardForkNumber, err := bp.getHardForkNumberByName(hardForkName)
 	if err != nil {
-		// log.Error("batch assign failure because of the hard fork name don't exist", "fork name", hardForkName)
+		log.Error("batch assign failure because of the hard fork name don't exist", "fork name", hardForkName)
 		return nil, err
 	}
 
@@ -247,7 +246,10 @@ func (bp *BatchProverTask) assignWithTwoCircuits(ctx *gin.Context, taskCtx *prov
 	for i := 0; i < 2; i++ {
 		hardForkNames[i] = bp.reverseVkMap[getTaskParameter.VKs[i]]
 		chunkRanges[i], err = bp.getChunkRangeByName(ctx, hardForkNames[i])
-		if err == nil && chunkRanges[i] != nil {
+		if err != nil {
+			return nil, err
+		}
+		if chunkRanges[i] != nil {
 			if chunkRange == nil {
 				chunkRange = chunkRanges[i]
 			} else {
@@ -256,8 +258,7 @@ func (bp *BatchProverTask) assignWithTwoCircuits(ctx *gin.Context, taskCtx *prov
 		}
 	}
 	if chunkRange == nil {
-		log.Error("chunkRange empty")
-		return nil, errors.New("chunkRange empty")
+		return nil, nil
 	}
 	var hardForkName string
 	getHardForkName := func(batch *orm.Batch) (string, error) {

coordinator/internal/logic/provertask/chunk_prover_task.go

@@ -3,7 +3,6 @@ package provertask
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"time"
 
@@ -182,31 +181,21 @@ func (cp *ChunkProverTask) assignWithTwoCircuits(ctx *gin.Context, taskCtx *prov
 		blockRanges   [2]*blockRange
 		err           error
 	)
-	var blockRange *blockRange
 	for i := 0; i < 2; i++ {
 		hardForkNames[i] = cp.reverseVkMap[getTaskParameter.VKs[i]]
 		blockRanges[i], err = cp.getBlockRangeByName(hardForkNames[i])
-		if err == nil && blockRanges[i] != nil {
-			if blockRange == nil {
-				blockRange = blockRanges[i]
-			} else {
-				var err2 error
-				blockRange, err2 = blockRange.merge(*blockRanges[i])
-				if err2 != nil {
-					return nil, err2
-				}
-			}
+		if err != nil {
+			return nil, err
 		}
 	}
-	if blockRange == nil {
-		log.Error("blockRange empty")
-		return nil, errors.New("blockRange empty")
+	blockRange, err := blockRanges[0].merge(*blockRanges[1])
+	if err != nil {
+		return nil, err
 	}
-
 	var hardForkName string
 	getHardForkName := func(chunk *orm.Chunk) (string, error) {
 		for i := 0; i < 2; i++ {
-			if blockRanges[i] != nil && blockRanges[i].contains(chunk.StartBlockNumber, chunk.EndBlockNumber) {
+			if blockRanges[i].contains(chunk.StartBlockNumber, chunk.EndBlockNumber) {
 				hardForkName = hardForkNames[i]
 				break
 			}
@@ -246,7 +235,7 @@ func (r *blockRange) contains(start, end uint64) bool {
 func (cp *ChunkProverTask) getBlockRangeByName(hardForkName string) (*blockRange, error) {
 	hardForkNumber, err := cp.getHardForkNumberByName(hardForkName)
 	if err != nil {
-		// log.Error("chunk assign failure because of the hard fork name don't exist", "fork name", hardForkName)
+		log.Error("chunk assign failure because of the hard fork name don't exist", "fork name", hardForkName)
 		return nil, err
 	}
 

coordinator/internal/logic/submitproof/proof_receiver.go

@@ -284,6 +284,8 @@ func (m *ProofReceiverLogic) validator(ctx context.Context, proverTask *orm.Prov
 
 	// if the batch/chunk have proved and verifier success, need skip this submit proof
 	if m.checkIsTaskSuccess(ctx, proofMsg.ID, proofMsg.Type) {
+		m.proofRecover(ctx, proverTask, types.ProverTaskFailureTypeObjectAlreadyVerified, proofMsg)
+
 		m.validateFailureProverTaskHaveVerifier.Inc()
 		log.Info("the prove task have proved and verifier success, skip this submit proof", "hash", proofMsg.ID,
 			"taskType", proverTask.TaskType, "proverName", proverTask.ProverName, "proverPublicKey", pk, "forkName", forkName)

go.work.sum

@@ -511,6 +511,8 @@ github.com/scroll-tech/da-codec v0.0.0-20240515170552-2e5286688fce/go.mod h1:1wW
 github.com/scroll-tech/da-codec v0.0.0-20240605080813-32bfc9fccde7/go.mod h1:1wWYii0OPwd5kw+xrz0PFgS420xNadrNF1x/ELJT+TM=
 github.com/scroll-tech/da-codec v0.0.0-20240708144942-a554fe417a39 h1:1gg+HPuirILVVVugZczq7ZQtPdCC/a38pwaFBqFtt1o=
 github.com/scroll-tech/da-codec v0.0.0-20240708144942-a554fe417a39/go.mod h1:D6XEESeNVJkQJlv3eK+FyR+ufPkgVQbJzERylQi53Bs=
+github.com/scroll-tech/da-codec v0.0.0-20240711065717-10bc0bed2be3 h1:QDiMz4RVjNigz4sHh1ieMe4nl/ca+MrXxyaVf0IKePk=
+github.com/scroll-tech/da-codec v0.0.0-20240711065717-10bc0bed2be3/go.mod h1:D6XEESeNVJkQJlv3eK+FyR+ufPkgVQbJzERylQi53Bs=
 github.com/scroll-tech/go-ethereum v1.10.14-0.20221202061207-804e7edc23ba/go.mod h1:jurIpDQ0hqtp9//xxeWzr8X9KMP/+TYn+vz3K1wZrv0=
 github.com/scroll-tech/go-ethereum v1.10.14-0.20221213034543-78c1f57fcfea/go.mod h1:jurIpDQ0hqtp9//xxeWzr8X9KMP/+TYn+vz3K1wZrv0=
 github.com/scroll-tech/go-ethereum v1.10.14-0.20221221073256-5ca70bf3a257/go.mod h1:jurIpDQ0hqtp9//xxeWzr8X9KMP/+TYn+vz3K1wZrv0=

prover/Cargo.lock

@@ -59,7 +59,7 @@ dependencies = [
 [[package]]
 name = "aggregator"
 version = "0.11.0"
-source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.11.4#38a68e22d3d8449bd39a50c22da55b9e741de453"
+source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.11.5#6ea8fb3fad4d8a8bfe873e18e2f881ad1c807ded"
 dependencies = [
  "ark-std 0.3.0",
  "bitstream-io",
@@ -661,7 +661,7 @@ dependencies = [
 [[package]]
 name = "bus-mapping"
 version = "0.11.0"
-source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.11.4#38a68e22d3d8449bd39a50c22da55b9e741de453"
+source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.11.5#6ea8fb3fad4d8a8bfe873e18e2f881ad1c807ded"
 dependencies = [
  "eth-types 0.11.0",
  "ethers-core 2.0.7 (git+https://github.com/scroll-tech/ethers-rs.git?branch=v2.0.7)",
@@ -1391,7 +1391,7 @@ dependencies = [
 [[package]]
 name = "eth-types"
 version = "0.11.0"
-source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.11.4#38a68e22d3d8449bd39a50c22da55b9e741de453"
+source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.11.5#6ea8fb3fad4d8a8bfe873e18e2f881ad1c807ded"
 dependencies = [
  "base64 0.13.1",
  "ethers-core 2.0.7 (git+https://github.com/scroll-tech/ethers-rs.git?branch=v2.0.7)",
@@ -1625,7 +1625,7 @@ dependencies = [
 [[package]]
 name = "external-tracer"
 version = "0.11.0"
-source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.11.4#38a68e22d3d8449bd39a50c22da55b9e741de453"
+source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.11.5#6ea8fb3fad4d8a8bfe873e18e2f881ad1c807ded"
 dependencies = [
  "eth-types 0.11.0",
  "geth-utils 0.11.0",
@@ -1863,7 +1863,7 @@ dependencies = [
 [[package]]
 name = "gadgets"
 version = "0.11.0"
-source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.11.4#38a68e22d3d8449bd39a50c22da55b9e741de453"
+source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.11.5#6ea8fb3fad4d8a8bfe873e18e2f881ad1c807ded"
 dependencies = [
  "eth-types 0.11.0",
  "halo2_proofs",
@@ -1896,7 +1896,7 @@ dependencies = [
 [[package]]
 name = "geth-utils"
 version = "0.11.0"
-source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.11.4#38a68e22d3d8449bd39a50c22da55b9e741de453"
+source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.11.5#6ea8fb3fad4d8a8bfe873e18e2f881ad1c807ded"
 dependencies = [
  "env_logger 0.10.2",
  "gobuild",
@@ -2795,7 +2795,7 @@ dependencies = [
 [[package]]
 name = "mock"
 version = "0.11.0"
-source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.11.4#38a68e22d3d8449bd39a50c22da55b9e741de453"
+source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.11.5#6ea8fb3fad4d8a8bfe873e18e2f881ad1c807ded"
 dependencies = [
  "eth-types 0.11.0",
  "ethers-core 2.0.7 (git+https://github.com/scroll-tech/ethers-rs.git?branch=v2.0.7)",
@@ -2825,7 +2825,7 @@ dependencies = [
 [[package]]
 name = "mpt-zktrie"
 version = "0.11.0"
-source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.11.4#38a68e22d3d8449bd39a50c22da55b9e741de453"
+source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.11.5#6ea8fb3fad4d8a8bfe873e18e2f881ad1c807ded"
 dependencies = [
  "eth-types 0.11.0",
  "halo2curves",
@@ -3451,7 +3451,7 @@ dependencies = [
 [[package]]
 name = "prover"
 version = "0.11.0"
-source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.11.4#38a68e22d3d8449bd39a50c22da55b9e741de453"
+source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.11.5#6ea8fb3fad4d8a8bfe873e18e2f881ad1c807ded"
 dependencies = [
  "aggregator 0.11.0",
  "anyhow",
@@ -5595,7 +5595,7 @@ dependencies = [
 [[package]]
 name = "zkevm-circuits"
 version = "0.11.0"
-source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.11.4#38a68e22d3d8449bd39a50c22da55b9e741de453"
+source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.11.5#6ea8fb3fad4d8a8bfe873e18e2f881ad1c807ded"
 dependencies = [
  "array-init",
  "bus-mapping 0.11.0",

prover/Cargo.toml

@@ -30,7 +30,7 @@ ethers-providers = { git = "https://github.com/scroll-tech/ethers-rs.git", branc
 halo2_proofs = { git = "https://github.com/scroll-tech/halo2.git", branch = "v1.1" }
 snark-verifier-sdk = { git = "https://github.com/scroll-tech/snark-verifier", branch = "develop", default-features = false, features = ["loader_halo2", "loader_evm", "halo2-pse"] }
 prover = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "v0.10", default-features = false, features = ["parallel_syn", "scroll", "shanghai"] }
-prover_next = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.11.4", package = "prover", default-features = false, features = ["parallel_syn", "scroll"] }
+prover_next = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.11.5", package = "prover", default-features = false, features = ["parallel_syn", "scroll"] }
 base64 = "0.13.1"
 reqwest = { version = "0.12.4", features = ["gzip"] }
 reqwest-middleware = "0.3"

rollup/cmd/rollup_relayer/app/app.go

@@ -106,9 +106,9 @@ func action(ctx *cli.Context) error {
 		l2watcher.TryFetchRunningMissingBlocks(number)
 	})
 
-	go utils.Loop(subCtx, 2*time.Second, chunkProposer.TryProposeChunk)
+	go utils.Loop(subCtx, time.Duration(cfg.L2Config.ChunkProposerConfig.ProposeIntervalMilliseconds)*time.Millisecond, chunkProposer.TryProposeChunk)
 
-	go utils.Loop(subCtx, 10*time.Second, batchProposer.TryProposeBatch)
+	go utils.Loop(subCtx, time.Duration(cfg.L2Config.BatchProposerConfig.ProposeIntervalMilliseconds)*time.Millisecond, batchProposer.TryProposeBatch)
 
 	go utils.Loop(subCtx, 2*time.Second, l2relayer.ProcessPendingBatches)
 

rollup/conf/config.json

@@ -61,6 +61,7 @@
       "l1_commit_gas_limit_multiplier": 1.2
     },
     "chunk_proposer_config": {
+      "propose_interval_milliseconds": 100,
       "max_block_num_per_chunk": 100,
       "max_tx_num_per_chunk": 100,
       "max_l1_commit_gas_per_chunk": 11234567,
@@ -71,6 +72,7 @@
       "max_uncompressed_batch_bytes_size": 634880
     },
     "batch_proposer_config": {
+      "propose_interval_milliseconds": 1000,
       "max_l1_commit_gas_per_batch": 11234567,
       "max_l1_commit_calldata_size_per_batch": 112345,
       "batch_timeout_sec": 300,

rollup/go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/go-resty/resty/v2 v2.7.0
 	github.com/holiman/uint256 v1.2.4
 	github.com/prometheus/client_golang v1.16.0
-	github.com/scroll-tech/da-codec v0.0.0-20240710085339-459342e9a542
+	github.com/scroll-tech/da-codec v0.0.0-20240712125636-d7e76c3f54b5
 	github.com/scroll-tech/go-ethereum v1.10.14-0.20240607130425-e2becce6a1a4
 	github.com/smartystreets/goconvey v1.8.0
 	github.com/stretchr/testify v1.9.0

rollup/go.sum

@@ -236,8 +236,8 @@ github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/scroll-tech/da-codec v0.0.0-20240710085339-459342e9a542 h1:if1EeBvoNf7eLzGJYpTt0aJ6Jvw5n274TUDD+9uj9sA=
-github.com/scroll-tech/da-codec v0.0.0-20240710085339-459342e9a542/go.mod h1:D6XEESeNVJkQJlv3eK+FyR+ufPkgVQbJzERylQi53Bs=
+github.com/scroll-tech/da-codec v0.0.0-20240712125636-d7e76c3f54b5 h1:mdgFgYSKbB7JbUPEvqKdXxXlzc3uRwD+dlNA4GsFSoo=
+github.com/scroll-tech/da-codec v0.0.0-20240712125636-d7e76c3f54b5/go.mod h1:D6XEESeNVJkQJlv3eK+FyR+ufPkgVQbJzERylQi53Bs=
 github.com/scroll-tech/go-ethereum v1.10.14-0.20240607130425-e2becce6a1a4 h1:gheWXra3HdZsz6q+w4LrXy8ybHOO6/t6Kb/V64bR5wE=
 github.com/scroll-tech/go-ethereum v1.10.14-0.20240607130425-e2becce6a1a4/go.mod h1:byf/mZ8jLYUCnUePTicjJWn+RvKdxDn7buS6glTnMwQ=
 github.com/scroll-tech/zktrie v0.8.4 h1:UagmnZ4Z3ITCk+aUq9NQZJNAwnWl4gSxsLb2Nl7IgRE=

rollup/internal/config/l2.go

@@ -26,6 +26,7 @@ type L2Config struct {
 
 // ChunkProposerConfig loads chunk_proposer configuration items.
 type ChunkProposerConfig struct {
+	ProposeIntervalMilliseconds     uint64  `json:"propose_interval_milliseconds"`
 	MaxBlockNumPerChunk             uint64  `json:"max_block_num_per_chunk"`
 	MaxTxNumPerChunk                uint64  `json:"max_tx_num_per_chunk"`
 	MaxL1CommitGasPerChunk          uint64  `json:"max_l1_commit_gas_per_chunk"`
@@ -38,6 +39,7 @@ type ChunkProposerConfig struct {
 
 // BatchProposerConfig loads batch_proposer configuration items.
 type BatchProposerConfig struct {
+	ProposeIntervalMilliseconds     uint64  `json:"propose_interval_milliseconds"`
 	MaxL1CommitGasPerBatch          uint64  `json:"max_l1_commit_gas_per_batch"`
 	MaxL1CommitCalldataSizePerBatch uint64  `json:"max_l1_commit_calldata_size_per_batch"`
 	BatchTimeoutSec                 uint64  `json:"batch_timeout_sec"`

rollup/internal/controller/watcher/batch_proposer.go

@@ -154,9 +154,49 @@ func (p *BatchProposer) TryProposeBatch() {
 	}
 }
 
-func (p *BatchProposer) updateDBBatchInfo(batch *encoding.Batch, codecVersion encoding.CodecVersion, metrics utils.BatchMetrics) error {
+func (p *BatchProposer) updateDBBatchInfo(batch *encoding.Batch, codecVersion encoding.CodecVersion, metrics *utils.BatchMetrics) error {
+	compatibilityBreachOccurred := false
+
+	for {
+		compatible, err := utils.CheckBatchCompressedDataCompatibility(batch, codecVersion)
+		if err != nil {
+			log.Error("Failed to check batch compressed data compatibility", "batch index", batch.Index, "codecVersion", codecVersion, "err", err)
+			return err
+		}
+
+		if compatible {
+			break
+		}
+
+		if len(batch.Chunks) == 1 {
+			log.Error("Cannot truncate batch with only 1 chunk for compatibility", "start block number", batch.Chunks[0].Blocks[0].Header.Number.Uint64(),
+				"end block number", batch.Chunks[0].Blocks[len(batch.Chunks[0].Blocks)-1].Header.Number.Uint64())
+			return errors.New("cannot truncate batch with only 1 chunk for compatibility")
+		}
+
+		compatibilityBreachOccurred = true
+		batch.Chunks = batch.Chunks[:len(batch.Chunks)-1]
+
+		log.Info("Batch not compatible with compressed data, removing last chunk", "batch index", batch.Index, "truncated chunk length", len(batch.Chunks))
+	}
+
+	if compatibilityBreachOccurred {
+		p.compressedDataCompatibilityBreachTotal.Inc()
+
+		// recalculate batch metrics after truncation
+		var calcErr error
+		metrics, calcErr = utils.CalculateBatchMetrics(batch, codecVersion)
+		if calcErr != nil {
+			return fmt.Errorf("failed to calculate batch metrics, batch index: %v, error: %w", batch.Index, calcErr)
+		}
+
+		p.recordTimerBatchMetrics(metrics)
+		p.recordAllBatchMetrics(metrics)
+	}
+
+	p.proposeBatchUpdateInfoTotal.Inc()
 	err := p.db.Transaction(func(dbTX *gorm.DB) error {
-		dbBatch, dbErr := p.batchOrm.InsertBatch(p.ctx, batch, codecVersion, metrics, dbTX)
+		dbBatch, dbErr := p.batchOrm.InsertBatch(p.ctx, batch, codecVersion, *metrics, dbTX)
 		if dbErr != nil {
 			log.Warn("BatchProposer.updateBatchInfoInDB insert batch failure", "index", batch.Index, "parent hash", batch.ParentBatchHash.Hex(), "error", dbErr)
 			return dbErr
@@ -239,18 +279,6 @@ func (p *BatchProposer) proposeBatch() error {
 	for i, chunk := range daChunks {
 		batch.Chunks = append(batch.Chunks, chunk)
 		metrics, calcErr := utils.CalculateBatchMetrics(&batch, codecVersion)
-
-		if errors.Is(calcErr, &encoding.CompressedDataCompatibilityError{}) {
-			if i == 0 {
-				// The first chunk fails compressed data compatibility check, manual fix is needed.
-				return fmt.Errorf("the first chunk fails compressed data compatibility check; start block number: %v, end block number: %v", dbChunks[0].StartBlockNumber, dbChunks[0].EndBlockNumber)
-			}
-			log.Warn("breaking limit condition in proposing a new batch due to a compressed data compatibility breach", "start chunk index", dbChunks[0].Index, "end chunk index", dbChunks[len(dbChunks)-1].Index)
-			batch.Chunks = batch.Chunks[:len(batch.Chunks)-1]
-			p.compressedDataCompatibilityBreachTotal.Inc()
-			return p.updateDBBatchInfo(&batch, codecVersion, *metrics)
-		}
-
 		if calcErr != nil {
 			return fmt.Errorf("failed to calculate batch metrics: %w", calcErr)
 		}
@@ -285,7 +313,7 @@ func (p *BatchProposer) proposeBatch() error {
 			}
 
 			p.recordAllBatchMetrics(metrics)
-			return p.updateDBBatchInfo(&batch, codecVersion, *metrics)
+			return p.updateDBBatchInfo(&batch, codecVersion, metrics)
 		}
 	}
 
@@ -303,7 +331,7 @@ func (p *BatchProposer) proposeBatch() error {
 
 		p.batchFirstBlockTimeoutReached.Inc()
 		p.recordAllBatchMetrics(metrics)
-		return p.updateDBBatchInfo(&batch, codecVersion, *metrics)
+		return p.updateDBBatchInfo(&batch, codecVersion, metrics)
 	}
 
 	log.Debug("pending chunks do not reach one of the constraints or contain a timeout block")

rollup/internal/controller/watcher/chunk_proposer.go

@@ -168,14 +168,52 @@ func (p *ChunkProposer) TryProposeChunk() {
 	}
 }
 
-func (p *ChunkProposer) updateDBChunkInfo(chunk *encoding.Chunk, codecVersion encoding.CodecVersion, metrics utils.ChunkMetrics) error {
+func (p *ChunkProposer) updateDBChunkInfo(chunk *encoding.Chunk, codecVersion encoding.CodecVersion, metrics *utils.ChunkMetrics) error {
 	if chunk == nil {
 		return nil
 	}
 
+	compatibilityBreachOccurred := false
+
+	for {
+		compatible, err := utils.CheckChunkCompressedDataCompatibility(chunk, codecVersion)
+		if err != nil {
+			log.Error("Failed to check chunk compressed data compatibility", "start block number", chunk.Blocks[0].Header.Number, "codecVersion", codecVersion, "err", err)
+			return err
+		}
+
+		if compatible {
+			break
+		}
+
+		if len(chunk.Blocks) == 1 {
+			log.Error("Cannot truncate chunk with only 1 block for compatibility", "block number", chunk.Blocks[0].Header.Number)
+			return errors.New("cannot truncate chunk with only 1 block for compatibility")
+		}
+
+		compatibilityBreachOccurred = true
+		chunk.Blocks = chunk.Blocks[:len(chunk.Blocks)-1]
+
+		log.Info("Chunk not compatible with compressed data, removing last block", "start block number", chunk.Blocks[0].Header.Number, "truncated block length", len(chunk.Blocks))
+	}
+
+	if compatibilityBreachOccurred {
+		p.compressedDataCompatibilityBreachTotal.Inc()
+
+		// recalculate chunk metrics after truncation
+		var calcErr error
+		metrics, calcErr = utils.CalculateChunkMetrics(chunk, codecVersion)
+		if calcErr != nil {
+			return fmt.Errorf("failed to calculate chunk metrics, start block number: %v, error: %w", chunk.Blocks[0].Header.Number, calcErr)
+		}
+
+		p.recordTimerChunkMetrics(metrics)
+		p.recordAllChunkMetrics(metrics)
+	}
+
 	p.proposeChunkUpdateInfoTotal.Inc()
 	err := p.db.Transaction(func(dbTX *gorm.DB) error {
-		dbChunk, err := p.chunkOrm.InsertChunk(p.ctx, chunk, codecVersion, metrics, dbTX)
+		dbChunk, err := p.chunkOrm.InsertChunk(p.ctx, chunk, codecVersion, *metrics, dbTX)
 		if err != nil {
 			log.Warn("ChunkProposer.InsertChunk failed", "err", err)
 			return err
@@ -234,7 +272,7 @@ func (p *ChunkProposer) proposeChunk() error {
 			return fmt.Errorf("failed to calculate chunk metrics: %w", calcErr)
 		}
 		p.recordTimerChunkMetrics(metrics)
-		return p.updateDBChunkInfo(&chunk, codecVersion, *metrics)
+		return p.updateDBChunkInfo(&chunk, codecVersion, metrics)
 	}
 
 	var chunk encoding.Chunk
@@ -242,17 +280,6 @@ func (p *ChunkProposer) proposeChunk() error {
 		chunk.Blocks = append(chunk.Blocks, block)
 
 		metrics, calcErr := utils.CalculateChunkMetrics(&chunk, codecVersion)
-		if errors.Is(calcErr, &encoding.CompressedDataCompatibilityError{}) {
-			if i == 0 {
-				// The first block fails compressed data compatibility check, manual fix is needed.
-				return fmt.Errorf("the first block fails compressed data compatibility check; block number: %v", block.Header.Number)
-			}
-			log.Warn("breaking limit condition in proposing a new chunk due to a compressed data compatibility breach", "start block number", chunk.Blocks[0].Header.Number, "block count", len(chunk.Blocks))
-			chunk.Blocks = chunk.Blocks[:len(chunk.Blocks)-1]
-			p.compressedDataCompatibilityBreachTotal.Inc()
-			return p.updateDBChunkInfo(&chunk, codecVersion, *metrics)
-		}
-
 		if calcErr != nil {
 			return fmt.Errorf("failed to calculate chunk metrics: %w", calcErr)
 		}
@@ -295,7 +322,7 @@ func (p *ChunkProposer) proposeChunk() error {
 			}
 
 			p.recordAllChunkMetrics(metrics)
-			return p.updateDBChunkInfo(&chunk, codecVersion, *metrics)
+			return p.updateDBChunkInfo(&chunk, codecVersion, metrics)
 		}
 	}
 
@@ -315,7 +342,7 @@ func (p *ChunkProposer) proposeChunk() error {
 
 		p.chunkFirstBlockTimeoutReached.Inc()
 		p.recordAllChunkMetrics(metrics)
-		return p.updateDBChunkInfo(&chunk, codecVersion, *metrics)
+		return p.updateDBChunkInfo(&chunk, codecVersion, metrics)
 	}
 
 	log.Debug("pending blocks do not reach one of the constraints or contain a timeout block")

rollup/internal/utils/utils.go

@@ -1,7 +1,6 @@
 package utils
 
 import (
-	"errors"
 	"fmt"
 	"time"
 
@@ -9,6 +8,7 @@ import (
 	"github.com/scroll-tech/da-codec/encoding/codecv0"
 	"github.com/scroll-tech/da-codec/encoding/codecv1"
 	"github.com/scroll-tech/da-codec/encoding/codecv2"
+	"github.com/scroll-tech/da-codec/encoding/codecv3"
 	"github.com/scroll-tech/go-ethereum/common"
 )
 
@@ -92,11 +92,7 @@ func CalculateChunkMetrics(chunk *encoding.Chunk, codecVersion encoding.CodecVer
 		metrics.L1CommitUncompressedBatchBytesSize, metrics.L1CommitBlobSize, err = codecv2.EstimateChunkL1CommitBatchSizeAndBlobSize(chunk)
 		metrics.EstimateBlobSizeTime = time.Since(start)
 		if err != nil {
-			if errors.Is(err, &encoding.CompressedDataCompatibilityError{}) {
-				return nil, err
-			} else {
-				return nil, fmt.Errorf("failed to estimate codecv2 chunk L1 commit batch size and blob size: %w", err)
-			}
+			return nil, fmt.Errorf("failed to estimate codecv2 chunk L1 commit batch size and blob size: %w", err)
 		}
 		return metrics, nil
 	default:
@@ -104,6 +100,34 @@ func CalculateChunkMetrics(chunk *encoding.Chunk, codecVersion encoding.CodecVer
 	}
 }
 
+// CheckChunkCompressedDataCompatibility checks compressed data compatibility of a batch built by a single chunk.
+func CheckChunkCompressedDataCompatibility(chunk *encoding.Chunk, codecVersion encoding.CodecVersion) (bool, error) {
+	switch codecVersion {
+	case encoding.CodecV0, encoding.CodecV1:
+		return true, nil
+	case encoding.CodecV2:
+		return codecv2.CheckChunkCompressedDataCompatibility(chunk)
+	case encoding.CodecV3:
+		return codecv3.CheckChunkCompressedDataCompatibility(chunk)
+	default:
+		return false, fmt.Errorf("unsupported codec version: %v", codecVersion)
+	}
+}
+
+// CheckBatchCompressedDataCompatibility checks compressed data compatibility of a batch built by a single chunk.
+func CheckBatchCompressedDataCompatibility(batch *encoding.Batch, codecVersion encoding.CodecVersion) (bool, error) {
+	switch codecVersion {
+	case encoding.CodecV0, encoding.CodecV1:
+		return true, nil
+	case encoding.CodecV2:
+		return codecv2.CheckBatchCompressedDataCompatibility(batch)
+	case encoding.CodecV3:
+		return codecv3.CheckBatchCompressedDataCompatibility(batch)
+	default:
+		return false, fmt.Errorf("unsupported codec version: %v", codecVersion)
+	}
+}
+
 // BatchMetrics indicates the metrics for proposing a batch.
 type BatchMetrics struct {
 	// common metrics
@@ -176,11 +200,7 @@ func CalculateBatchMetrics(batch *encoding.Batch, codecVersion encoding.CodecVer
 		metrics.L1CommitUncompressedBatchBytesSize, metrics.L1CommitBlobSize, err = codecv2.EstimateBatchL1CommitBatchSizeAndBlobSize(batch)
 		metrics.EstimateBlobSizeTime = time.Since(start)
 		if err != nil {
-			if errors.Is(err, &encoding.CompressedDataCompatibilityError{}) {
-				return nil, err
-			} else {
-				return nil, fmt.Errorf("failed to estimate codecv2 batch L1 commit batch size and blob size: %w", err)
-			}
+			return nil, fmt.Errorf("failed to estimate codecv2 batch L1 commit batch size and blob size: %w", err)
 		}
 		return metrics, nil
 	default: