feat: turn libscroll_zstd dynamic lib into static (#1358)

Co-authored-by: colinlyguo <colinlyguo@users.noreply.github.com>

.github/workflows/docker.yml

@@ -46,7 +46,7 @@ jobs:
         with:
           context: .
           file: ./build/dockerfiles/event_watcher.Dockerfile
-          platforms: linux/amd64
+          platforms: linux/amd64,linux/arm64
           push: true
           tags: |
             ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.REPOSITORY }}:${{ env.IMAGE_TAG }}
@@ -91,7 +91,7 @@ jobs:
         with:
           context: .
           file: ./build/dockerfiles/gas_oracle.Dockerfile
-          platforms: linux/amd64
+          platforms: linux/amd64,linux/arm64
           push: true
           tags: |
             ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.REPOSITORY }}:${{ env.IMAGE_TAG }}
@@ -136,7 +136,7 @@ jobs:
         with:
           context: .
           file: ./build/dockerfiles/rollup_relayer.Dockerfile
-          platforms: linux/amd64
+          platforms: linux/amd64,linux/arm64
           push: true
           tags: |
             ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.REPOSITORY }}:${{ env.IMAGE_TAG }}

.github/workflows/rollup.yml

@@ -105,7 +105,7 @@ jobs:
     - name: Test rollup packages
       working-directory: 'rollup'
       run: |
-        ./run_test.sh
+        make test
     - name: Upload coverage reports to Codecov
       uses: codecov/codecov-action@v3
       env:

README.md

@@ -46,18 +46,7 @@ make dev_docker
 Run the tests using the following commands:
 
 ```bash
-export LIBSCROLL_ZSTD_VERSION=v0.1.0-rc0-ubuntu20.04
-export SCROLL_LIB_PATH=/scroll/lib
-
-sudo mkdir -p $SCROLL_LIB_PATH
-
-sudo wget -O $SCROLL_LIB_PATH/libscroll_zstd.so https://github.com/scroll-tech/da-codec/releases/download/$LIBSCROLL_ZSTD_VERSION/libscroll_zstd.so
-
-export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SCROLL_LIB_PATH
-export CGO_LDFLAGS="-L$SCROLL_LIB_PATH -Wl,-rpath,$SCROLL_LIB_PATH"
-
 go test -v -race -covermode=atomic scroll-tech/rollup/...
-
 go test -tags="mock_verifier" -v -race -covermode=atomic scroll-tech/coordinator/...
 go test -v -race -covermode=atomic scroll-tech/database/...
 go test -v -race -covermode=atomic scroll-tech/common/...

build/dockerfiles/event_watcher.Dockerfile

@@ -1,6 +1,3 @@
-ARG LIBSCROLL_ZSTD_VERSION=v0.1.0-rc0-ubuntu20.04
-ARG SCROLL_LIB_PATH=/scroll/lib
-
 # Download Go dependencies
 FROM scrolltech/go-rust-builder:go-1.21-rust-nightly-2023-12-03 as base
 
@@ -18,36 +15,14 @@ RUN go mod download -x
 # Build event_watcher
 FROM base as builder
 
-ARG LIBSCROLL_ZSTD_VERSION
-ARG SCROLL_LIB_PATH
-
-RUN mkdir -p $SCROLL_LIB_PATH
-
-RUN apt-get -qq update && apt-get -qq install -y wget
-
-RUN wget -O $SCROLL_LIB_PATH/libscroll_zstd.so https://github.com/scroll-tech/da-codec/releases/download/$LIBSCROLL_ZSTD_VERSION/libscroll_zstd.so
-
-ENV LD_LIBRARY_PATH=$SCROLL_LIB_PATH
-ENV CGO_LDFLAGS="-L$SCROLL_LIB_PATH -Wl,-rpath,$SCROLL_LIB_PATH"
-
 RUN --mount=target=. \
     --mount=type=cache,target=/root/.cache/go-build \
-    cd /src/rollup/cmd/event_watcher/ && go build -v -p 4 -o /bin/event_watcher
+    cd /src/rollup/cmd/event_watcher/ && CGO_LDFLAGS="-ldl" go build -v -p 4 -o /bin/event_watcher
 
 # Pull event_watcher into a second stage deploy alpine container
 FROM ubuntu:20.04
 
-ARG LIBSCROLL_ZSTD_VERSION
-ARG SCROLL_LIB_PATH
-
-RUN mkdir -p $SCROLL_LIB_PATH
-
-RUN apt-get -qq update && apt-get -qq install -y wget
-
-RUN wget -O $SCROLL_LIB_PATH/libscroll_zstd.so https://github.com/scroll-tech/da-codec/releases/download/$LIBSCROLL_ZSTD_VERSION/libscroll_zstd.so
-
-ENV LD_LIBRARY_PATH=$SCROLL_LIB_PATH
-ENV CGO_LDFLAGS="-L$SCROLL_LIB_PATH -Wl,-rpath,$SCROLL_LIB_PATH"
+ENV CGO_LDFLAGS="-ldl"
 
 COPY --from=builder /bin/event_watcher /bin/
 WORKDIR /app

build/dockerfiles/gas_oracle.Dockerfile

@@ -1,6 +1,3 @@
-ARG LIBSCROLL_ZSTD_VERSION=v0.1.0-rc0-ubuntu20.04
-ARG SCROLL_LIB_PATH=/scroll/lib
-
 # Download Go dependencies
 FROM scrolltech/go-rust-builder:go-1.21-rust-nightly-2023-12-03 as base
 
@@ -18,36 +15,14 @@ RUN go mod download -x
 # Build gas_oracle
 FROM base as builder
 
-ARG LIBSCROLL_ZSTD_VERSION
-ARG SCROLL_LIB_PATH
-
-RUN mkdir -p $SCROLL_LIB_PATH
-
-RUN apt-get -qq update && apt-get -qq install -y wget
-
-RUN wget -O $SCROLL_LIB_PATH/libscroll_zstd.so https://github.com/scroll-tech/da-codec/releases/download/$LIBSCROLL_ZSTD_VERSION/libscroll_zstd.so
-
-ENV LD_LIBRARY_PATH=$SCROLL_LIB_PATH
-ENV CGO_LDFLAGS="-L$SCROLL_LIB_PATH -Wl,-rpath,$SCROLL_LIB_PATH"
-
 RUN --mount=target=. \
     --mount=type=cache,target=/root/.cache/go-build \
-    cd /src/rollup/cmd/gas_oracle/ && go build -v -p 4 -o /bin/gas_oracle
+    cd /src/rollup/cmd/gas_oracle/ && CGO_LDFLAGS="-ldl" go build -v -p 4 -o /bin/gas_oracle
 
 # Pull gas_oracle into a second stage deploy alpine container
 FROM ubuntu:20.04
 
-ARG LIBSCROLL_ZSTD_VERSION
-ARG SCROLL_LIB_PATH
-
-RUN mkdir -p $SCROLL_LIB_PATH
-
-RUN apt-get -qq update && apt-get -qq install -y wget
-
-RUN wget -O $SCROLL_LIB_PATH/libscroll_zstd.so https://github.com/scroll-tech/da-codec/releases/download/$LIBSCROLL_ZSTD_VERSION/libscroll_zstd.so
-
-ENV LD_LIBRARY_PATH=$SCROLL_LIB_PATH
-ENV CGO_LDFLAGS="-L$SCROLL_LIB_PATH -Wl,-rpath,$SCROLL_LIB_PATH"
+ENV CGO_LDFLAGS="-ldl"
 
 COPY --from=builder /bin/gas_oracle /bin/
 WORKDIR /app

build/dockerfiles/rollup_relayer.Dockerfile

@@ -1,6 +1,3 @@
-ARG LIBSCROLL_ZSTD_VERSION=v0.1.0-rc0-ubuntu20.04
-ARG SCROLL_LIB_PATH=/scroll/lib
-
 # Download Go dependencies
 FROM scrolltech/go-rust-builder:go-1.21-rust-nightly-2023-12-03 as base
 
@@ -18,36 +15,14 @@ RUN go mod download -x
 # Build rollup_relayer
 FROM base as builder
 
-ARG LIBSCROLL_ZSTD_VERSION
-ARG SCROLL_LIB_PATH
-
-RUN mkdir -p $SCROLL_LIB_PATH
-
-RUN apt-get -qq update && apt-get -qq install -y wget
-
-RUN wget -O $SCROLL_LIB_PATH/libscroll_zstd.so https://github.com/scroll-tech/da-codec/releases/download/$LIBSCROLL_ZSTD_VERSION/libscroll_zstd.so
-
-ENV LD_LIBRARY_PATH=$SCROLL_LIB_PATH
-ENV CGO_LDFLAGS="-L$SCROLL_LIB_PATH -Wl,-rpath,$SCROLL_LIB_PATH"
-
 RUN --mount=target=. \
     --mount=type=cache,target=/root/.cache/go-build \
-    cd /src/rollup/cmd/rollup_relayer/ && go build -v -p 4 -o /bin/rollup_relayer
+    cd /src/rollup/cmd/rollup_relayer/ && CGO_LDFLAGS="-ldl" go build -v -p 4 -o /bin/rollup_relayer
 
 # Pull rollup_relayer into a second stage deploy alpine container
 FROM ubuntu:20.04
 
-ARG LIBSCROLL_ZSTD_VERSION
-ARG SCROLL_LIB_PATH
-
-RUN mkdir -p $SCROLL_LIB_PATH
-
-RUN apt-get -qq update && apt-get -qq install -y wget
-
-RUN wget -O $SCROLL_LIB_PATH/libscroll_zstd.so https://github.com/scroll-tech/da-codec/releases/download/$LIBSCROLL_ZSTD_VERSION/libscroll_zstd.so
-
-ENV LD_LIBRARY_PATH=$SCROLL_LIB_PATH
-ENV CGO_LDFLAGS="-L$SCROLL_LIB_PATH -Wl,-rpath,$SCROLL_LIB_PATH"
+ENV CGO_LDFLAGS="-ldl"
 
 COPY --from=builder /bin/rollup_relayer /bin/
 WORKDIR /app

common/version/version.go

@@ -5,7 +5,7 @@ import (
 	"runtime/debug"
 )
 
-var tag = "v4.4.9"
+var tag = "v4.4.11"
 
 var commit = func() string {
 	if info, ok := debug.ReadBuildInfo(); ok {

coordinator/internal/logic/provertask/batch_prover_task.go

@@ -42,6 +42,7 @@ func NewBatchProverTask(cfg *config.Config, chainCfg *params.ChainConfig, db *go
 	bp := &BatchProverTask{
 		BaseProverTask: BaseProverTask{
 			vkMap:              vkMap,
+			reverseVkMap:       reverseMap(vkMap),
 			db:                 db,
 			cfg:                cfg,
 			nameForkMap:        nameForkMap,
@@ -64,48 +65,31 @@ func NewBatchProverTask(cfg *config.Config, chainCfg *params.ChainConfig, db *go
 	return bp
 }
 
-// Assign load and assign batch tasks
-func (bp *BatchProverTask) Assign(ctx *gin.Context, getTaskParameter *coordinatorType.GetTaskParameter) (*coordinatorType.GetTaskSchema, error) {
-	taskCtx, err := bp.checkParameter(ctx, getTaskParameter)
-	if err != nil || taskCtx == nil {
-		return nil, fmt.Errorf("check prover task parameter failed, error:%w", err)
-	}
+type chunkIndexRange struct {
+	start uint64
+	end   uint64
+}
 
-	hardForkNumber, err := bp.getHardForkNumberByName(taskCtx.HardForkName)
-	if err != nil {
-		log.Error("batch assign failure because of the hard fork name don't exist", "fork name", taskCtx.HardForkName)
-		return nil, err
+func (r *chunkIndexRange) merge(o chunkIndexRange) *chunkIndexRange {
+	var start, end = r.start, r.end
+	if o.start < r.start {
+		start = o.start
 	}
+	if o.end > r.end {
+		end = o.end
+	}
+	return &chunkIndexRange{start, end}
+}
 
-	// if the hard fork number set, rollup relayer must generate the chunk from hard fork number,
-	// so the hard fork chunk's start_block_number must be ForkBlockNumber
-	var startChunkIndex uint64 = 0
-	var endChunkIndex uint64 = math.MaxInt64
-	fromBlockNum, toBlockNum := forks.BlockRange(hardForkNumber, bp.forkHeights)
-	if fromBlockNum != 0 {
-		startChunk, chunkErr := bp.chunkOrm.GetChunkByStartBlockNumber(ctx.Copy(), fromBlockNum)
-		if chunkErr != nil {
-			log.Error("failed to get fork start chunk index", "forkName", taskCtx.HardForkName, "fromBlockNumber", fromBlockNum, "err", chunkErr)
-			return nil, ErrCoordinatorInternalFailure
-		}
-		if startChunk == nil {
-			return nil, nil
-		}
-		startChunkIndex = startChunk.Index
-	}
-	if toBlockNum != math.MaxInt64 {
-		toChunk, chunkErr := bp.chunkOrm.GetChunkByStartBlockNumber(ctx.Copy(), toBlockNum)
-		if chunkErr != nil {
-			log.Error("failed to get fork end chunk index", "forkName", taskCtx.HardForkName, "toBlockNumber", toBlockNum, "err", chunkErr)
-			return nil, ErrCoordinatorInternalFailure
-		}
-		if toChunk != nil {
-			// toChunk being nil only indicates that we haven't yet reached the fork boundary
-			// don't need change the endChunkIndex of math.MaxInt64
-			endChunkIndex = toChunk.Index
-		}
-	}
+func (r *chunkIndexRange) contains(start, end uint64) bool {
+	return r.start <= start && r.end > end
+}
 
+type getHardForkNameByBatchFunc func(*orm.Batch) (string, error)
+
+func (bp *BatchProverTask) doAssignTaskWithinChunkRange(ctx *gin.Context, taskCtx *proverTaskContext,
+	chunkRange *chunkIndexRange, getTaskParameter *coordinatorType.GetTaskParameter, getHardForkName getHardForkNameByBatchFunc) (*coordinatorType.GetTaskSchema, error) {
+	startChunkIndex, endChunkIndex := chunkRange.start, chunkRange.end
 	maxActiveAttempts := bp.cfg.ProverManager.ProversPerSession
 	maxTotalAttempts := bp.cfg.ProverManager.SessionAttempts
 	var batchTask *orm.Batch
@@ -154,13 +138,25 @@ func (bp *BatchProverTask) Assign(ctx *gin.Context, getTaskParameter *coordinato
 	}
 
 	log.Info("start batch proof generation session", "id", batchTask.Hash, "public key", taskCtx.PublicKey, "prover name", taskCtx.ProverName)
+	var (
+		proverVersion = taskCtx.ProverVersion
+		hardForkName  = taskCtx.HardForkName
+	)
+	var err error
+	if getHardForkName != nil {
+		hardForkName, err = getHardForkName(batchTask)
+		if err != nil {
+			log.Error("failed to get hard fork name by batch", "error", err.Error())
+			return nil, ErrCoordinatorInternalFailure
+		}
+	}
 
 	proverTask := orm.ProverTask{
 		TaskID:          batchTask.Hash,
 		ProverPublicKey: taskCtx.PublicKey,
 		TaskType:        int16(message.ProofTypeBatch),
 		ProverName:      taskCtx.ProverName,
-		ProverVersion:   taskCtx.ProverVersion,
+		ProverVersion:   proverVersion,
 		ProvingStatus:   int16(types.ProverAssigned),
 		FailureType:     int16(types.ProverTaskFailureTypeUndefined),
 		// here why need use UTC time. see scroll/common/databased/db.go
@@ -181,7 +177,7 @@ func (bp *BatchProverTask) Assign(ctx *gin.Context, getTaskParameter *coordinato
 		return nil, ErrCoordinatorInternalFailure
 	}
 
-	bp.batchTaskGetTaskTotal.WithLabelValues(taskCtx.HardForkName).Inc()
+	bp.batchTaskGetTaskTotal.WithLabelValues(hardForkName).Inc()
 	bp.batchTaskGetTaskProver.With(prometheus.Labels{
 		coordinatorType.LabelProverName:      proverTask.ProverName,
 		coordinatorType.LabelProverPublicKey: proverTask.ProverPublicKey,
@@ -191,6 +187,107 @@ func (bp *BatchProverTask) Assign(ctx *gin.Context, getTaskParameter *coordinato
 	return taskMsg, nil
 }
 
+func (bp *BatchProverTask) getChunkRangeByName(ctx *gin.Context, hardForkName string) (*chunkIndexRange, error) {
+	hardForkNumber, err := bp.getHardForkNumberByName(hardForkName)
+	if err != nil {
+		log.Error("batch assign failure because of the hard fork name don't exist", "fork name", hardForkName)
+		return nil, err
+	}
+
+	// if the hard fork number set, rollup relayer must generate the chunk from hard fork number,
+	// so the hard fork chunk's start_block_number must be ForkBlockNumber
+	var startChunkIndex uint64 = 0
+	var endChunkIndex uint64 = math.MaxInt64
+	fromBlockNum, toBlockNum := forks.BlockRange(hardForkNumber, bp.forkHeights)
+	if fromBlockNum != 0 {
+		startChunk, chunkErr := bp.chunkOrm.GetChunkByStartBlockNumber(ctx.Copy(), fromBlockNum)
+		if chunkErr != nil {
+			log.Error("failed to get fork start chunk index", "forkName", hardForkName, "fromBlockNumber", fromBlockNum, "err", chunkErr)
+			return nil, ErrCoordinatorInternalFailure
+		}
+		if startChunk == nil {
+			return nil, nil
+		}
+		startChunkIndex = startChunk.Index
+	}
+	if toBlockNum != math.MaxInt64 {
+		toChunk, chunkErr := bp.chunkOrm.GetChunkByStartBlockNumber(ctx.Copy(), toBlockNum)
+		if chunkErr != nil {
+			log.Error("failed to get fork end chunk index", "forkName", hardForkName, "toBlockNumber", toBlockNum, "err", chunkErr)
+			return nil, ErrCoordinatorInternalFailure
+		}
+		if toChunk != nil {
+			// toChunk being nil only indicates that we haven't yet reached the fork boundary
+			// don't need change the endChunkIndex of math.MaxInt64
+			endChunkIndex = toChunk.Index
+		}
+	}
+	return &chunkIndexRange{startChunkIndex, endChunkIndex}, nil
+}
+
+func (bp *BatchProverTask) assignWithSingleCircuit(ctx *gin.Context, taskCtx *proverTaskContext, getTaskParameter *coordinatorType.GetTaskParameter) (*coordinatorType.GetTaskSchema, error) {
+	chunkRange, err := bp.getChunkRangeByName(ctx, taskCtx.HardForkName)
+	if err != nil {
+		return nil, err
+	}
+	if chunkRange == nil {
+		return nil, nil
+	}
+	return bp.doAssignTaskWithinChunkRange(ctx, taskCtx, chunkRange, getTaskParameter, nil)
+}
+
+func (bp *BatchProverTask) assignWithTwoCircuits(ctx *gin.Context, taskCtx *proverTaskContext, getTaskParameter *coordinatorType.GetTaskParameter) (*coordinatorType.GetTaskSchema, error) {
+	var (
+		hardForkNames [2]string
+		chunkRanges   [2]*chunkIndexRange
+		err           error
+	)
+	for i := 0; i < 2; i++ {
+		hardForkNames[i] = bp.reverseVkMap[getTaskParameter.VKs[i]]
+		chunkRanges[i], err = bp.getChunkRangeByName(ctx, hardForkNames[i])
+		if err != nil {
+			return nil, err
+		}
+		if chunkRanges[i] == nil {
+			return nil, nil
+		}
+	}
+	chunkRange := chunkRanges[0].merge(*chunkRanges[1])
+	var hardForkName string
+	getHardForkName := func(batch *orm.Batch) (string, error) {
+		for i := 0; i < 2; i++ {
+			if chunkRanges[i].contains(batch.StartChunkIndex, batch.EndChunkIndex) {
+				hardForkName = hardForkNames[i]
+				break
+			}
+		}
+		if hardForkName == "" {
+			log.Warn("get batch not belongs to any hard fork name", "batch id", batch.Index)
+			return "", fmt.Errorf("get batch not belongs to any hard fork name, batch id: %d", batch.Index)
+		}
+		return hardForkName, nil
+	}
+	schema, err := bp.doAssignTaskWithinChunkRange(ctx, taskCtx, chunkRange, getTaskParameter, getHardForkName)
+	if schema != nil && err == nil {
+		schema.HardForkName = hardForkName
+		return schema, nil
+	}
+	return schema, err
+}
+
+// Assign load and assign batch tasks
+func (bp *BatchProverTask) Assign(ctx *gin.Context, getTaskParameter *coordinatorType.GetTaskParameter) (*coordinatorType.GetTaskSchema, error) {
+	taskCtx, err := bp.checkParameter(ctx, getTaskParameter)
+	if err != nil || taskCtx == nil {
+		return nil, fmt.Errorf("check prover task parameter failed, error:%w", err)
+	}
+
+	if len(getTaskParameter.VKs) > 0 {
+		return bp.assignWithTwoCircuits(ctx, taskCtx, getTaskParameter)
+	}
+	return bp.assignWithSingleCircuit(ctx, taskCtx, getTaskParameter)
+}
+
 func (bp *BatchProverTask) formatProverTask(ctx context.Context, task *orm.ProverTask) (*coordinatorType.GetTaskSchema, error) {
 	// get chunk from db
 	chunks, err := bp.chunkOrm.GetChunksByBatchHash(ctx, task.TaskID)

coordinator/internal/logic/provertask/chunk_prover_task.go

@@ -39,6 +39,7 @@ func NewChunkProverTask(cfg *config.Config, chainCfg *params.ChainConfig, db *go
 	cp := &ChunkProverTask{
 		BaseProverTask: BaseProverTask{
 			vkMap:              vkMap,
+			reverseVkMap:       reverseMap(vkMap),
 			db:                 db,
 			cfg:                cfg,
 			nameForkMap:        nameForkMap,
@@ -61,20 +62,11 @@ func NewChunkProverTask(cfg *config.Config, chainCfg *params.ChainConfig, db *go
 	return cp
 }
 
-// Assign the chunk proof which need to prove
-func (cp *ChunkProverTask) Assign(ctx *gin.Context, getTaskParameter *coordinatorType.GetTaskParameter) (*coordinatorType.GetTaskSchema, error) {
-	taskCtx, err := cp.checkParameter(ctx, getTaskParameter)
-	if err != nil || taskCtx == nil {
-		return nil, fmt.Errorf("check prover task parameter failed, error:%w", err)
-	}
+type getHardForkNameByChunkFunc func(*orm.Chunk) (string, error)
 
-	hardForkNumber, err := cp.getHardForkNumberByName(taskCtx.HardForkName)
-	if err != nil {
-		log.Error("chunk assign failure because of the hard fork name don't exist", "fork name", taskCtx.HardForkName)
-		return nil, err
-	}
-
-	fromBlockNum, toBlockNum := forks.BlockRange(hardForkNumber, cp.forkHeights)
+func (cp *ChunkProverTask) doAssignTaskWithinBlockRange(ctx *gin.Context, taskCtx *proverTaskContext,
+	blockRange *blockRange, getTaskParameter *coordinatorType.GetTaskParameter, getHardForkName getHardForkNameByChunkFunc) (*coordinatorType.GetTaskSchema, error) {
+	fromBlockNum, toBlockNum := blockRange.from, blockRange.to
 	if toBlockNum > getTaskParameter.ProverHeight {
 		toBlockNum = getTaskParameter.ProverHeight + 1
 	}
@@ -127,13 +119,25 @@ func (cp *ChunkProverTask) Assign(ctx *gin.Context, getTaskParameter *coordinato
 	}
 
 	log.Info("start chunk generation session", "id", chunkTask.Hash, "public key", taskCtx.PublicKey, "prover name", taskCtx.ProverName)
+	var (
+		proverVersion = taskCtx.ProverVersion
+		hardForkName  = taskCtx.HardForkName
+		err           error
+	)
+	if getHardForkName != nil {
+		hardForkName, err = getHardForkName(chunkTask)
+		if err != nil {
+			log.Error("failed to get hard fork name by chunk", "error", err.Error())
+			return nil, ErrCoordinatorInternalFailure
+		}
+	}
 
 	proverTask := orm.ProverTask{
 		TaskID:          chunkTask.Hash,
 		ProverPublicKey: taskCtx.PublicKey,
 		TaskType:        int16(message.ProofTypeChunk),
 		ProverName:      taskCtx.ProverName,
-		ProverVersion:   taskCtx.ProverVersion,
+		ProverVersion:   proverVersion,
 		ProvingStatus:   int16(types.ProverAssigned),
 		FailureType:     int16(types.ProverTaskFailureTypeUndefined),
 		// here why need use UTC time. see scroll/common/databased/db.go
@@ -153,7 +157,7 @@ func (cp *ChunkProverTask) Assign(ctx *gin.Context, getTaskParameter *coordinato
 		return nil, ErrCoordinatorInternalFailure
 	}
 
-	cp.chunkTaskGetTaskTotal.WithLabelValues(taskCtx.HardForkName).Inc()
+	cp.chunkTaskGetTaskTotal.WithLabelValues(hardForkName).Inc()
 	cp.chunkTaskGetTaskProver.With(prometheus.Labels{
 		coordinatorType.LabelProverName:      proverTask.ProverName,
 		coordinatorType.LabelProverPublicKey: proverTask.ProverPublicKey,
@@ -163,6 +167,95 @@ func (cp *ChunkProverTask) Assign(ctx *gin.Context, getTaskParameter *coordinato
 	return taskMsg, nil
 }
 
+func (cp *ChunkProverTask) assignWithSingleCircuit(ctx *gin.Context, taskCtx *proverTaskContext, getTaskParameter *coordinatorType.GetTaskParameter) (*coordinatorType.GetTaskSchema, error) {
+	blockRange, err := cp.getBlockRangeByName(taskCtx.HardForkName)
+	if err != nil {
+		return nil, err
+	}
+	return cp.doAssignTaskWithinBlockRange(ctx, taskCtx, blockRange, getTaskParameter, nil)
+}
+
+func (cp *ChunkProverTask) assignWithTwoCircuits(ctx *gin.Context, taskCtx *proverTaskContext, getTaskParameter *coordinatorType.GetTaskParameter) (*coordinatorType.GetTaskSchema, error) {
+	var (
+		hardForkNames [2]string
+		blockRanges   [2]*blockRange
+		err           error
+	)
+	for i := 0; i < 2; i++ {
+		hardForkNames[i] = cp.reverseVkMap[getTaskParameter.VKs[i]]
+		blockRanges[i], err = cp.getBlockRangeByName(hardForkNames[i])
+		if err != nil {
+			return nil, err
+		}
+	}
+	blockRange, err := blockRanges[0].merge(*blockRanges[1])
+	if err != nil {
+		return nil, err
+	}
+	var hardForkName string
+	getHardForkName := func(chunk *orm.Chunk) (string, error) {
+		for i := 0; i < 2; i++ {
+			if blockRanges[i].contains(chunk.StartBlockNumber, chunk.EndBlockNumber) {
+				hardForkName = hardForkNames[i]
+				break
+			}
+		}
+		if hardForkName == "" {
+			log.Warn("get chunk not belongs to any hard fork name", "chunk id", chunk.Index)
+			return "", fmt.Errorf("get chunk not belongs to any hard fork name, chunk id: %d", chunk.Index)
+		}
+		return hardForkName, nil
+	}
+	schema, err := cp.doAssignTaskWithinBlockRange(ctx, taskCtx, blockRange, getTaskParameter, getHardForkName)
+	if schema != nil && err == nil {
+		schema.HardForkName = hardForkName
+		return schema, nil
+	}
+	return schema, err
+}
+
+type blockRange struct {
+	from uint64
+	to   uint64
+}
+
+func (r *blockRange) merge(o blockRange) (*blockRange, error) {
+	if r.from == o.to {
+		return &blockRange{o.from, r.to}, nil
+	} else if r.to == o.from {
+		return &blockRange{r.from, o.to}, nil
+	}
+	return nil, fmt.Errorf("two ranges are not adjacent")
+}
+
+func (r *blockRange) contains(start, end uint64) bool {
+	return r.from <= start && r.to > end
+}
+
+func (cp *ChunkProverTask) getBlockRangeByName(hardForkName string) (*blockRange, error) {
+	hardForkNumber, err := cp.getHardForkNumberByName(hardForkName)
+	if err != nil {
+		log.Error("chunk assign failure because of the hard fork name don't exist", "fork name", hardForkName)
+		return nil, err
+	}
+
+	fromBlockNum, toBlockNum := forks.BlockRange(hardForkNumber, cp.forkHeights)
+	return &blockRange{fromBlockNum, toBlockNum}, nil
+}
+
+// Assign the chunk proof which need to prove
+func (cp *ChunkProverTask) Assign(ctx *gin.Context, getTaskParameter *coordinatorType.GetTaskParameter) (*coordinatorType.GetTaskSchema, error) {
+	taskCtx, err := cp.checkParameter(ctx, getTaskParameter)
+	if err != nil || taskCtx == nil {
+		return nil, fmt.Errorf("check prover task parameter failed, error:%w", err)
+	}
+
+	if len(getTaskParameter.VKs) > 0 {
+		return cp.assignWithTwoCircuits(ctx, taskCtx, getTaskParameter)
+	}
+	return cp.assignWithSingleCircuit(ctx, taskCtx, getTaskParameter)
+}
+
 func (cp *ChunkProverTask) formatProverTask(ctx context.Context, task *orm.ProverTask) (*coordinatorType.GetTaskSchema, error) {
 	// Get block hashes.
 	blockHashes, dbErr := cp.blockOrm.GetL2BlockHashesByChunkHash(ctx, task.TaskID)

coordinator/internal/logic/provertask/prover_task.go

@@ -29,14 +29,25 @@ type ProverTask interface {
 	Assign(ctx *gin.Context, getTaskParameter *coordinatorType.GetTaskParameter) (*coordinatorType.GetTaskSchema, error)
 }
 
+func reverseMap(input map[string]string) map[string]string {
+	output := make(map[string]string, len(input))
+	for k, v := range input {
+		output[v] = k
+	}
+	return output
+}
+
 // BaseProverTask a base prover task which contain series functions
 type BaseProverTask struct {
 	cfg *config.Config
 	db  *gorm.DB
 
-	vkMap       map[string]string
-	nameForkMap map[string]uint64
-	forkHeights []uint64
+	// key is hardForkName, value is vk
+	vkMap map[string]string
+	// key is vk, value is hardForkName
+	reverseVkMap map[string]string
+	nameForkMap  map[string]uint64
+	forkHeights  []uint64
 
 	batchOrm           *orm.Batch
 	chunkOrm           *orm.Chunk
@@ -74,30 +85,42 @@ func (b *BaseProverTask) checkParameter(ctx *gin.Context, getTaskParameter *coor
 	}
 	ptc.ProverVersion = proverVersion.(string)
 
-	hardForkName, hardForkNameExist := ctx.Get(coordinatorType.HardForkName)
-	if !hardForkNameExist {
-		return nil, fmt.Errorf("get hard fork name from context failed")
-	}
-	ptc.HardForkName = hardForkName.(string)
-
 	if !version.CheckScrollRepoVersion(proverVersion.(string), b.cfg.ProverManager.MinProverVersion) {
 		return nil, fmt.Errorf("incompatible prover version. please upgrade your prover, minimum allowed version: %s, actual version: %s", b.cfg.ProverManager.MinProverVersion, proverVersion.(string))
 	}
 
-	vk, vkExist := b.vkMap[ptc.HardForkName]
-	if !vkExist {
-		return nil, fmt.Errorf("can't get vk for hard fork:%s, vkMap:%v", ptc.HardForkName, b.vkMap)
-	}
-
-	// if the prover has a different vk
-	if getTaskParameter.VK != vk {
-		log.Error("vk inconsistency", "prover vk", getTaskParameter.VK, "vk", vk, "hardForkName", ptc.HardForkName)
-		// if the prover reports a different prover version
-		if !version.CheckScrollProverVersion(proverVersion.(string)) {
-			return nil, fmt.Errorf("incompatible prover version. please upgrade your prover, expect version: %s, actual version: %s", version.Version, proverVersion.(string))
+	// signals that the prover is multi-circuits version
+	if len(getTaskParameter.VKs) > 0 {
+		if len(getTaskParameter.VKs) != 2 {
+			return nil, fmt.Errorf("parameter vks length must be 2")
+		}
+		for _, vk := range getTaskParameter.VKs {
+			if _, exists := b.reverseVkMap[vk]; !exists {
+				return nil, fmt.Errorf("incompatible vk. vk %s is invalid", vk)
+			}
+		}
+	} else {
+		hardForkName, hardForkNameExist := ctx.Get(coordinatorType.HardForkName)
+		if !hardForkNameExist {
+			return nil, fmt.Errorf("get hard fork name from context failed")
+		}
+		ptc.HardForkName = hardForkName.(string)
+
+		vk, vkExist := b.vkMap[ptc.HardForkName]
+		if !vkExist {
+			return nil, fmt.Errorf("can't get vk for hard fork:%s, vkMap:%v", ptc.HardForkName, b.vkMap)
+		}
+
+		// if the prover has a different vk
+		if getTaskParameter.VK != vk {
+			log.Error("vk inconsistency", "prover vk", getTaskParameter.VK, "vk", vk, "hardForkName", ptc.HardForkName)
+			// if the prover reports a different prover version
+			if !version.CheckScrollProverVersion(proverVersion.(string)) {
+				return nil, fmt.Errorf("incompatible prover version. please upgrade your prover, expect version: %s, actual version: %s", version.Version, proverVersion.(string))
+			}
+			// if the prover reports a same prover version
+			return nil, fmt.Errorf("incompatible vk. please check your params files or config files")
 		}
-		// if the prover reports a same prover version
-		return nil, fmt.Errorf("incompatible vk. please check your params files or config files")
 	}
 
 	isBlocked, err := b.proverBlockListOrm.IsPublicKeyBlocked(ctx.Copy(), publicKey.(string))

coordinator/internal/logic/submitproof/proof_receiver.go

@@ -134,7 +134,12 @@ func (m *ProofReceiverLogic) HandleZkProof(ctx *gin.Context, proofMsg *message.P
 	if len(pv) == 0 {
 		return fmt.Errorf("get ProverVersion from context failed")
 	}
-	hardForkName := ctx.GetString(coordinatorType.HardForkName)
+	// use hard_fork_name from parameter first
+	// if prover support multi hard_forks, the real hard_fork_name is not set to the gin context
+	hardForkName := proofParameter.HardForkName
+	if hardForkName == "" {
+		hardForkName = ctx.GetString(coordinatorType.HardForkName)
+	}
 
 	var proverTask *orm.ProverTask
 	var err error

coordinator/internal/logic/verifier/verifier.go

@@ -164,8 +164,8 @@ func (v *Verifier) loadEmbedVK() error {
 		return err
 	}
 
-	v.BatchVKMap["shanghai"] = base64.StdEncoding.EncodeToString(batchVKBytes)
-	v.ChunkVKMap["shanghai"] = base64.StdEncoding.EncodeToString(chunkVkBytes)
+	v.BatchVKMap["bernoulli"] = base64.StdEncoding.EncodeToString(batchVKBytes)
+	v.ChunkVKMap["bernoulli"] = base64.StdEncoding.EncodeToString(chunkVkBytes)
 	v.BatchVKMap[""] = base64.StdEncoding.EncodeToString(batchVKBytes)
 	v.ChunkVKMap[""] = base64.StdEncoding.EncodeToString(chunkVkBytes)
 	return nil

coordinator/internal/types/get_task.go

@@ -2,15 +2,17 @@ package types
 
 // GetTaskParameter for ProverTasks request parameter
 type GetTaskParameter struct {
-	ProverHeight uint64 `form:"prover_height" json:"prover_height"`
-	TaskType     int    `form:"task_type" json:"task_type"`
-	VK           string `form:"vk" json:"vk"`
+	ProverHeight uint64   `form:"prover_height" json:"prover_height"`
+	TaskType     int      `form:"task_type" json:"task_type"`
+	VK           string   `form:"vk" json:"vk"`   // will be deprecated after all go_prover offline
+	VKs          []string `form:"vks" json:"vks"` // for rust_prover that supporting multi-circuits
 }
 
 // GetTaskSchema the schema data return to prover for get prover task
 type GetTaskSchema struct {
-	UUID     string `json:"uuid"`
-	TaskID   string `json:"task_id"`
-	TaskType int    `json:"task_type"`
-	TaskData string `json:"task_data"`
+	UUID         string `json:"uuid"`
+	TaskID       string `json:"task_id"`
+	TaskType     int    `json:"task_type"`
+	TaskData     string `json:"task_data"`
+	HardForkName string `json:"hard_fork_name"`
 }

coordinator/internal/types/submit_proof.go

@@ -3,11 +3,12 @@ package types
 // SubmitProofParameter the SubmitProof api request parameter
 type SubmitProofParameter struct {
 	// TODO when prover have upgrade, need change this field to required
-	UUID        string `form:"uuid" json:"uuid"`
-	TaskID      string `form:"task_id" json:"task_id" binding:"required"`
-	TaskType    int    `form:"task_type" json:"task_type" binding:"required"`
-	Status      int    `form:"status" json:"status"`
-	Proof       string `form:"proof" json:"proof"`
-	FailureType int    `form:"failure_type" json:"failure_type"`
-	FailureMsg  string `form:"failure_msg" json:"failure_msg"`
+	UUID         string `form:"uuid" json:"uuid"`
+	TaskID       string `form:"task_id" json:"task_id" binding:"required"`
+	TaskType     int    `form:"task_type" json:"task_type" binding:"required"`
+	Status       int    `form:"status" json:"status"`
+	Proof        string `form:"proof" json:"proof"`
+	FailureType  int    `form:"failure_type" json:"failure_type"`
+	FailureMsg   string `form:"failure_msg" json:"failure_msg"`
+	HardForkName string `form:"hard_fork_name" json:"hard_fork_name"`
 }

go.work.sum

@@ -378,8 +378,8 @@ github.com/sclevine/agouti v3.0.0+incompatible h1:8IBJS6PWz3uTlMP3YBIR5f+KAldcGu
 github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
 github.com/scroll-tech/da-codec v0.0.0-20240515170552-2e5286688fce h1:SN43TBs7VaJt9q737eWWqGz0OCg4v+PtUn3RbJcG1o0=
 github.com/scroll-tech/da-codec v0.0.0-20240515170552-2e5286688fce/go.mod h1:1wWYii0OPwd5kw+xrz0PFgS420xNadrNF1x/ELJT+TM=
-github.com/scroll-tech/da-codec v0.0.0-20240527110002-d7bcb621a61a h1:5uWQHo/+cXexQGeSrywtXM2z29zRFctmux2vXs3JLrM=
-github.com/scroll-tech/da-codec v0.0.0-20240527110002-d7bcb621a61a/go.mod h1:1wWYii0OPwd5kw+xrz0PFgS420xNadrNF1x/ELJT+TM=
+github.com/scroll-tech/da-codec v0.0.0-20240604103336-17c1ba7dd66d h1:ywij6xtaviuNbpdeOrimT0C5S560Lz2E1kl7AUYXCbE=
+github.com/scroll-tech/da-codec v0.0.0-20240604103336-17c1ba7dd66d/go.mod h1:1wWYii0OPwd5kw+xrz0PFgS420xNadrNF1x/ELJT+TM=
 github.com/scroll-tech/go-ethereum v1.10.14-0.20221202061207-804e7edc23ba/go.mod h1:jurIpDQ0hqtp9//xxeWzr8X9KMP/+TYn+vz3K1wZrv0=
 github.com/scroll-tech/go-ethereum v1.10.14-0.20221213034543-78c1f57fcfea/go.mod h1:jurIpDQ0hqtp9//xxeWzr8X9KMP/+TYn+vz3K1wZrv0=
 github.com/scroll-tech/go-ethereum v1.10.14-0.20221221073256-5ca70bf3a257/go.mod h1:jurIpDQ0hqtp9//xxeWzr8X9KMP/+TYn+vz3K1wZrv0=

rollup/Makefile

@@ -2,28 +2,14 @@
 
 IMAGE_VERSION=latest
 REPO_ROOT_DIR=./..
-LIBSCROLL_ZSTD_VERSION=v0.1.0-rc0-ubuntu20.04
-SCROLL_LIB_PATH=/scroll/lib
 
 mock_abi:
 	cd .. && solc --evm-version cancun --bin --abi --optimize --overwrite -o ./build/bin ./rollup/mock_bridge/MockBridge.sol
 	cd .. && go run github.com/scroll-tech/go-ethereum/cmd/abigen --bin=./build/bin/MockBridge.bin --abi=./build/bin/MockBridge.abi --pkg=mock_bridge --out=./rollup/mock_bridge/MockBridge.go
 
-libzstd:
-	sudo mkdir -p $(SCROLL_LIB_PATH)/
-	sudo wget -O $(SCROLL_LIB_PATH)/libscroll_zstd.so https://github.com/scroll-tech/da-codec/releases/download/v0.1.0-rc0-ubuntu20.04/libscroll_zstd.so
-
-rollup_bins: libzstd ## Builds the Rollup bins.
-	export LD_LIBRARY_PATH=$$LD_LIBRARY_PATH:$(SCROLL_LIB_PATH) && \
-	export CGO_LDFLAGS="-L$(SCROLL_LIB_PATH) -Wl,-rpath,$(SCROLL_LIB_PATH)" && \
+rollup_bins: ## Builds the Rollup bins.
 	go build -o $(PWD)/build/bin/event_watcher ./cmd/event_watcher/
-
-	export LD_LIBRARY_PATH=$$LD_LIBRARY_PATH:$(SCROLL_LIB_PATH) && \
-	export CGO_LDFLAGS="-L$(SCROLL_LIB_PATH) -Wl,-rpath,$(SCROLL_LIB_PATH)" && \
 	go build -o $(PWD)/build/bin/gas_oracle ./cmd/gas_oracle/
-
-	export LD_LIBRARY_PATH=$$LD_LIBRARY_PATH:$(SCROLL_LIB_PATH) && \
-	export CGO_LDFLAGS="-L$(SCROLL_LIB_PATH) -Wl,-rpath,$(SCROLL_LIB_PATH)" && \
 	go build -o $(PWD)/build/bin/rollup_relayer ./cmd/rollup_relayer/
 
 event_watcher: ## Builds the event_watcher bin

rollup/go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/go-resty/resty/v2 v2.7.0
 	github.com/holiman/uint256 v1.2.4
 	github.com/prometheus/client_golang v1.16.0
-	github.com/scroll-tech/da-codec v0.0.0-20240527110002-d7bcb621a61a
+	github.com/scroll-tech/da-codec v0.0.0-20240604103336-17c1ba7dd66d
 	github.com/scroll-tech/go-ethereum v1.10.14-0.20240426041101-a860446ebaea
 	github.com/smartystreets/goconvey v1.8.0
 	github.com/stretchr/testify v1.9.0

rollup/go.sum

@@ -236,8 +236,8 @@ github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/scroll-tech/da-codec v0.0.0-20240527110002-d7bcb621a61a h1:5uWQHo/+cXexQGeSrywtXM2z29zRFctmux2vXs3JLrM=
-github.com/scroll-tech/da-codec v0.0.0-20240527110002-d7bcb621a61a/go.mod h1:1wWYii0OPwd5kw+xrz0PFgS420xNadrNF1x/ELJT+TM=
+github.com/scroll-tech/da-codec v0.0.0-20240604103336-17c1ba7dd66d h1:ywij6xtaviuNbpdeOrimT0C5S560Lz2E1kl7AUYXCbE=
+github.com/scroll-tech/da-codec v0.0.0-20240604103336-17c1ba7dd66d/go.mod h1:1wWYii0OPwd5kw+xrz0PFgS420xNadrNF1x/ELJT+TM=
 github.com/scroll-tech/go-ethereum v1.10.14-0.20240426041101-a860446ebaea h1:CH1WXWrpEpLaP3N+bFs2a1xdE0+lRm1AuJQb5YvE6Ls=
 github.com/scroll-tech/go-ethereum v1.10.14-0.20240426041101-a860446ebaea/go.mod h1:i4VBgWoaW/y0D8MmQb7hSOulyw1dKhuiSFAbznwivCA=
 github.com/scroll-tech/zktrie v0.8.2 h1:UMuIfA+jdgWMLmTgTL64Emo+zzMOdcnH0+eYdDcshxQ=

rollup/internal/controller/relayer/l1_relayer_test.go

@@ -32,7 +32,7 @@ func setupL1RelayerDB(t *testing.T) *gorm.DB {
 	return db
 }
 
-// testCreateNewRelayer test create new relayer instance and stop
+// testCreateNewL1Relayer test create new relayer instance and stop
 func testCreateNewL1Relayer(t *testing.T) {
 	db := setupL1RelayerDB(t)
 	defer database.CloseDB(db)

rollup/run_test.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-# Download .so files
-export LIBSCROLL_ZSTD_VERSION=v0.1.0-rc0-ubuntu20.04
-export SCROLL_LIB_PATH=/scroll/lib
-
-sudo mkdir -p $SCROLL_LIB_PATH
-
-sudo wget -O $SCROLL_LIB_PATH/libscroll_zstd.so https://github.com/scroll-tech/da-codec/releases/download/$LIBSCROLL_ZSTD_VERSION/libscroll_zstd.so
-
-# Set the environment variable
-export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SCROLL_LIB_PATH
-export CGO_LDFLAGS="-L$SCROLL_LIB_PATH -Wl,-rpath,$SCROLL_LIB_PATH"
-
-# Run module tests
-go test -v -race -gcflags="-l" -ldflags="-s=false" -coverprofile=coverage.txt -covermode=atomic ./...