feat(coordinator): split the coordinator cron to single process (#995)

Co-authored-by: maskpp <maskpp266@gmail.com>
Co-authored-by: HAOYUatHZ <37070449+HAOYUatHZ@users.noreply.github.com>
Co-authored-by: HAOYUatHZ <HAOYUatHZ@users.noreply.github.com>

.github/workflows/docker.yml

@@ -111,7 +111,7 @@ jobs:
         tags: scrolltech/bridgehistoryapi-server:${{github.ref_name}}
         # cache-from: type=gha,scope=${{ github.workflow }}
         # cache-to: type=gha,scope=${{ github.workflow }}
-  coordinator:
+  coordinator-api:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout code
@@ -127,8 +127,29 @@ jobs:
       uses: docker/build-push-action@v2
       with:
         context: .
-        file: ./build/dockerfiles/coordinator.Dockerfile
+        file: ./build/dockerfiles/coordinator-api.Dockerfile
         push: true
-        tags: scrolltech/coordinator:${{github.ref_name}}
+        tags: scrolltech/coordinator-api:${{github.ref_name}}
+        # cache-from: type=gha,scope=${{ github.workflow }}
+        # cache-to: type=gha,scope=${{ github.workflow }}
+  coordinator-cron:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+    - name: Login to Docker Hub
+      uses: docker/login-action@v2
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+    - name: Build and push coordinator docker
+      uses: docker/build-push-action@v2
+      with:
+        context: .
+        file: ./build/dockerfiles/coordinator-cron.Dockerfile
+        push: true
+        tags: scrolltech/coordinator-cron:${{github.ref_name}}
         # cache-from: type=gha,scope=${{ github.workflow }}
         # cache-to: type=gha,scope=${{ github.workflow }}

build/dockerfiles/coordinator-api.Dockerfile

@@ -36,7 +36,7 @@ COPY . .
 RUN cp -r ./common/libzkp/interface ./coordinator/internal/logic/verifier/lib
 COPY --from=zkp-builder /app/target/release/libzkp.so ./coordinator/internal/logic/verifier/lib/
 COPY --from=zkp-builder /app/target/release/libzktrie.so ./coordinator/internal/logic/verifier/lib/
-RUN cd ./coordinator && make coordinator_skip_libzkp && mv ./build/bin/coordinator /bin/coordinator && mv internal/logic/verifier/lib /bin/
+RUN cd ./coordinator && make coordinator_api_skip_libzkp && mv ./build/bin/coordinator_api /bin/coordinator_api && mv internal/logic/verifier/lib /bin/
 
 # Pull coordinator into a second stage deploy alpine container
 FROM ubuntu:20.04
@@ -44,7 +44,7 @@ ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/src/coordinator/internal/logic/verifier/li
 # ENV CHAIN_ID=534353
 RUN mkdir -p /src/coordinator/internal/logic/verifier/lib
 COPY --from=builder /bin/lib /src/coordinator/internal/logic/verifier/lib
-COPY --from=builder /bin/coordinator /bin/
-RUN /bin/coordinator --version
+COPY --from=builder /bin/coordinator_api /bin/
+RUN /bin/coordinator_api --version
 
-ENTRYPOINT ["/bin/coordinator"]
+ENTRYPOINT ["/bin/coordinator_api"]

build/dockerfiles/coordinator-cron.Dockerfile

@@ -0,0 +1,25 @@
+# Download Go dependencies
+FROM scrolltech/go-alpine-builder:1.19 as base
+
+WORKDIR /src
+COPY go.work* ./
+COPY ./rollup/go.* ./rollup/
+COPY ./common/go.* ./common/
+COPY ./coordinator/go.* ./coordinator/
+COPY ./database/go.* ./database/
+COPY ./prover/go.* ./prover/
+COPY ./tests/integration-test/go.* ./tests/integration-test/
+COPY ./bridge-history-api/go.* ./bridge-history-api/
+RUN go mod download -x
+
+# Build coordinator
+FROM base as builder
+RUN --mount=target=. \
+    --mount=type=cache,target=/root/.cache/go-build \
+    cd /src/coordinator/cmd/cron/ && go build -v -p 4 -o /bin/coordinator_cron
+
+# Pull coordinator into a second stage deploy alpine container
+FROM alpine:latest
+COPY --from=builder /bin/coordinator_cron /bin/
+
+ENTRYPOINT ["coordinator_cron"]

build/dockerfiles/coordinator-cron.Dockerfile.dockerignore

@@ -0,0 +1,6 @@
+assets/
+contracts/
+docs/
+l2geth/
+rpc-gateway/
+*target/*

common/utils/simulation.go

@@ -24,8 +24,10 @@ var (
 	// DBCliApp the name of mock database app.
 	DBCliApp MockAppName = "db_cli-test"
 
-	// CoordinatorApp the name of mock coordinator app.
-	CoordinatorApp MockAppName = "coordinator-test"
+	// CoordinatorAPIApp the name of mock coordinator app.
+	CoordinatorAPIApp MockAppName = "coordinator-api-test"
+	// CoordinatorCronApp the name of mock coordinator cron app.
+	CoordinatorCronApp MockAppName = "coordinator-cron-test"
 
 	// ChunkProverApp the name of mock chunk prover app.
 	ChunkProverApp MockAppName = "chunkProver-test"

common/version/version.go

@@ -5,7 +5,7 @@ import (
 	"runtime/debug"
 )
 
-var tag = "v4.3.37"
+var tag = "v4.3.38"
 
 var commit = func() string {
 	if info, ok := debug.ReadBuildInfo(); ok {

coordinator/Makefile

@@ -1,6 +1,5 @@
 .PHONY: lint docker clean coordinator coordinator_skip_libzkp mock_coordinator
 
-IMAGE_NAME=coordinator
 IMAGE_VERSION=latest
 REPO_ROOT_DIR=./..
 
@@ -22,14 +21,20 @@ libzkp:
 	rm -rf ./internal/logic/verifier/lib && cp -r ../common/libzkp/interface ./internal/logic/verifier/lib
 	find ../common | grep libzktrie.so | xargs -I{} cp {} ./internal/logic/verifier/lib
 
-coordinator: libzkp ## Builds the Coordinator instance.
-	go build -ldflags "-X scroll-tech/common/version.ZkVersion=${ZK_VERSION}" -o $(PWD)/build/bin/coordinator ./cmd
+coordinator_api: libzkp ## Builds the Coordinator api instance.
+	go build -ldflags "-X scroll-tech/common/version.ZkVersion=${ZK_VERSION}" -o $(PWD)/build/bin/coordinator_api ./cmd/api
 
-coordinator_skip_libzkp:
-	go build -ldflags "-X scroll-tech/common/version.ZkVersion=${ZK_VERSION}" -o $(PWD)/build/bin/coordinator ./cmd
+coordinator_cron:
+	go build -ldflags "-X scroll-tech/common/version.ZkVersion=${ZK_VERSION}" -o $(PWD)/build/bin/coordinator_cron ./cmd/cron
 
-mock_coordinator: ## Builds the mocked Coordinator instance.
-	go build -tags="mock_prover mock_verifier" -o $(PWD)/build/bin/coordinator ./cmd
+coordinator_api_skip_libzkp:
+	go build -ldflags "-X scroll-tech/common/version.ZkVersion=${ZK_VERSION}" -o $(PWD)/build/bin/coordinator_api ./cmd/api
+
+mock_coordinator_api: ## Builds the mocked Coordinator instance.
+	go build -tags="mock_prover mock_verifier" -o $(PWD)/build/bin/coordinator_api ./cmd/api
+
+mock_coordinator_cron: ## Builds the mocked Coordinator instance.
+	go build -tags="mock_prover mock_verifier" -o $(PWD)/build/bin/coordinator_cron ./cmd/cron
 
 test-verifier: libzkp
 	go test -tags ffi -timeout 0 -v ./internal/logic/verifier
@@ -45,7 +50,9 @@ clean: ## Empty out the bin folder
 	@rm -rf build/bin
 
 docker:
-	DOCKER_BUILDKIT=1 docker build -t scrolltech/${IMAGE_NAME}:${IMAGE_VERSION} ${REPO_ROOT_DIR}/ -f ${REPO_ROOT_DIR}/build/dockerfiles/coordinator.Dockerfile
+	DOCKER_BUILDKIT=1 docker build -t scrolltech/coordinator-api:${IMAGE_VERSION} ${REPO_ROOT_DIR}/ -f ${REPO_ROOT_DIR}/build/dockerfiles/coordinator-api.Dockerfile
+	DOCKER_BUILDKIT=1 docker build -t scrolltech/coordinator-cron:${IMAGE_VERSION} ${REPO_ROOT_DIR}/ -f ${REPO_ROOT_DIR}/build/dockerfiles/coordinator-cron.Dockerfile
 
 docker_push:
-	docker push scrolltech/${IMAGE_NAME}:${IMAGE_VERSION}
+	docker push scrolltech/coordinator-api:${IMAGE_VERSION}
+	docker push scrolltech/coordinator-cron:${IMAGE_VERSION}

coordinator/README.md

@@ -12,7 +12,8 @@ See [monorepo prerequisites](../README.md#prerequisites).
 
 ```bash
 make clean
-make coordinator
+make coordinator_api
+make coordinator_cron
 ```
 The built coordinator binary is in the `build/bin` directory.
 
@@ -44,13 +45,15 @@ The coordinator behavior can be configured using [`config.json`](config.json). C
 
 * Using default ports and config.json:
 ```bash
-./build/bin/coordinator --http
+./build/bin/coordinator_api --http
+./build/bin/coordinator_cron 
 ```
 
 * Using manually specified ports and config.json:
 ```bash
-./build/bin/coordinator --config ./config.json --http --http.addr localhost --http.port 8390
+./build/bin/coordinator_api --config ./config.json --http --http.addr localhost --http.port 8390
+./build/bin/coordinator_cron --config ./config.json 
 ```
 
-* For other flags, refer to [`cmd/app/flags.go`](cmd/app/flags.go).
+* For other flags, refer to [`cmd/api/app/flags.go`](cmd/api/app/flags.go).
 

coordinator/cmd/api/app/app.go

@@ -22,7 +22,6 @@ import (
 
 	"scroll-tech/coordinator/internal/config"
 	"scroll-tech/coordinator/internal/controller/api"
-	"scroll-tech/coordinator/internal/controller/cron"
 	"scroll-tech/coordinator/internal/route"
 )
 
@@ -41,7 +40,7 @@ func init() {
 		return utils.LogSetup(ctx)
 	}
 	// Register `coordinator-test` app for integration-test.
-	utils.RegisterSimulation(app, utils.CoordinatorApp)
+	utils.RegisterSimulation(app, utils.CoordinatorAPIApp)
 }
 
 func action(ctx *cli.Context) error {
@@ -51,28 +50,23 @@ func action(ctx *cli.Context) error {
 		log.Crit("failed to load config file", "config file", cfgFile, "error", err)
 	}
 
-	subCtx, cancel := context.WithCancel(ctx.Context)
 	db, err := database.InitDB(cfg.DB)
 	if err != nil {
 		log.Crit("failed to init db connection", "err", err)
 	}
-
-	registry := prometheus.DefaultRegisterer
-	observability.Server(ctx, db)
-
-	proofCollector := cron.NewCollector(subCtx, db, cfg, registry)
 	defer func() {
-		proofCollector.Stop()
-		cancel()
 		if err = database.CloseDB(db); err != nil {
 			log.Error("can not close db connection", "error", err)
 		}
 	}()
 
+	registry := prometheus.DefaultRegisterer
+	observability.Server(ctx, db)
+
 	apiSrv := apiServer(ctx, cfg, db, registry)
 
 	log.Info(
-		"coordinator start successfully",
+		"Start coordinator api successfully.",
 		"version", version.Version,
 	)
 

coordinator/cmd/api/app/app_test.go

@@ -10,7 +10,7 @@ import (
 )
 
 func TestRunCoordinator(t *testing.T) {
-	coordinator := cmd.NewCmd("coordinator-test", "--version")
+	coordinator := cmd.NewCmd("coordinator-api-test", "--version")
 	defer coordinator.WaitExit()
 
 	// wait result

coordinator/cmd/api/app/flags.go

@@ -8,10 +8,6 @@ var (
 		&httpEnabledFlag,
 		&httpListenAddrFlag,
 		&httpPortFlag,
-		// ws flags
-		&wsEnabledFlag,
-		&wsListenAddrFlag,
-		&wsPortFlag,
 	}
 	// httpEnabledFlag enable rpc server.
 	httpEnabledFlag = cli.BoolFlag{
@@ -31,19 +27,4 @@ var (
 		Usage: "HTTP-RPC server listening port",
 		Value: 8390,
 	}
-	wsEnabledFlag = cli.BoolFlag{
-		Name:  "ws",
-		Usage: "Enable the WS-RPC server",
-	}
-	wsListenAddrFlag = cli.StringFlag{
-		Name:  "ws.addr",
-		Usage: "WS-RPC server listening interface",
-		Value: "localhost",
-	}
-	// websocket port
-	wsPortFlag = cli.IntFlag{
-		Name:  "ws.port",
-		Usage: "WS-RPC server listening port",
-		Value: 8391,
-	}
 )

coordinator/cmd/api/app/mock_app.go

@@ -55,8 +55,8 @@ func NewCoordinatorApp(base *docker.App, file string) *CoordinatorApp {
 
 // RunApp run coordinator-test child process by multi parameters.
 func (c *CoordinatorApp) RunApp(t *testing.T, args ...string) {
-	c.AppAPI = cmd.NewCmd(string(utils.CoordinatorApp), append(c.args, args...)...)
-	c.AppAPI.RunApp(func() bool { return c.AppAPI.WaitResult(t, time.Second*20, "Start coordinator successfully") })
+	c.AppAPI = cmd.NewCmd(string(utils.CoordinatorAPIApp), append(c.args, args...)...)
+	c.AppAPI.RunApp(func() bool { return c.AppAPI.WaitResult(t, time.Second*20, "Start coordinator api successfully") })
 }
 
 // Free stop and release coordinator-test.

coordinator/cmd/api/main.go

@@ -0,0 +1,7 @@
+package main
+
+import "scroll-tech/coordinator/cmd/api/app"
+
+func main() {
+	app.Run()
+}

coordinator/cmd/cron/app/app.go

@@ -0,0 +1,87 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/signal"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/scroll-tech/go-ethereum/log"
+	"github.com/urfave/cli/v2"
+
+	"scroll-tech/common/database"
+	"scroll-tech/common/observability"
+	"scroll-tech/common/utils"
+	"scroll-tech/common/version"
+
+	"scroll-tech/coordinator/internal/config"
+	"scroll-tech/coordinator/internal/controller/cron"
+)
+
+var app *cli.App
+
+func init() {
+	// Set up coordinator app info.
+	app = cli.NewApp()
+	app.Action = action
+	app.Name = "coordinator cron"
+	app.Usage = "The Scroll L2 Coordinator cron"
+	app.Version = version.Version
+	app.Flags = append(app.Flags, utils.CommonFlags...)
+	app.Before = func(ctx *cli.Context) error {
+		return utils.LogSetup(ctx)
+	}
+	// Register `coordinator-cron-test` app for integration-cron-test.
+	utils.RegisterSimulation(app, utils.CoordinatorCronApp)
+}
+
+func action(ctx *cli.Context) error {
+	cfgFile := ctx.String(utils.ConfigFileFlag.Name)
+	cfg, err := config.NewConfig(cfgFile)
+	if err != nil {
+		log.Crit("failed to load config file", "config file", cfgFile, "error", err)
+	}
+
+	subCtx, cancel := context.WithCancel(ctx.Context)
+	db, err := database.InitDB(cfg.DB)
+	if err != nil {
+		log.Crit("failed to init db connection", "err", err)
+	}
+
+	registry := prometheus.DefaultRegisterer
+	observability.Server(ctx, db)
+
+	proofCollector := cron.NewCollector(subCtx, db, cfg, registry)
+	defer func() {
+		proofCollector.Stop()
+		cancel()
+		if err = database.CloseDB(db); err != nil {
+			log.Error("can not close db connection", "error", err)
+		}
+	}()
+
+	log.Info(
+		"coordinator cron start successfully",
+		"version", version.Version,
+	)
+
+	// Catch CTRL-C to ensure a graceful shutdown.
+	interrupt := make(chan os.Signal, 1)
+	signal.Notify(interrupt, os.Interrupt)
+
+	// Wait until the interrupt signal is received from an OS signal.
+	<-interrupt
+
+	log.Info("coordinator cron exiting success")
+	return nil
+}
+
+// Run coordinator.
+func Run() {
+	// RunApp the coordinator.
+	if err := app.Run(os.Args); err != nil {
+		_, _ = fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}

coordinator/cmd/cron/app/app_test.go

@@ -0,0 +1,19 @@
+package app
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"scroll-tech/common/cmd"
+	"scroll-tech/common/version"
+)
+
+func TestRunCoordinatorCron(t *testing.T) {
+	coordinator := cmd.NewCmd("coordinator-cron-test", "--version")
+	defer coordinator.WaitExit()
+
+	// wait result
+	coordinator.ExpectWithTimeout(t, true, time.Second*3, fmt.Sprintf("coordinator cron version %s", version.Version))
+	coordinator.RunApp(nil)
+}

coordinator/cmd/cron/main.go

@@ -0,0 +1,7 @@
+package main
+
+import "scroll-tech/coordinator/cmd/cron/app"
+
+func main() {
+	app.Run()
+}

coordinator/cmd/main.go

@@ -1,7 +0,0 @@
-package main
-
-import "scroll-tech/coordinator/cmd/app"
-
-func main() {
-	app.Run()
-}

coordinator/internal/controller/cron/collect_proof.go

@@ -76,7 +76,7 @@ func NewCollector(ctx context.Context, db *gorm.DB, cfg *config.Config, reg prom
 	go c.checkBatchAllChunkReady()
 	go c.cleanupChallenge()
 
-	log.Info("Start coordinator successfully.")
+	log.Info("Start coordinator cron successfully.")
 
 	return c
 }

tests/integration-test/integration_test.go

@@ -17,7 +17,7 @@ import (
 
 	"scroll-tech/database/migrate"
 
-	capp "scroll-tech/coordinator/cmd/app"
+	capp "scroll-tech/coordinator/cmd/api/app"
 
 	"scroll-tech/common/database"
 	"scroll-tech/common/docker"