fix(rollup-relayer): better estimate commit batch gas limit (#982)

Co-authored-by: colinlyguo <colinlyguo@users.noreply.github.com>

.github/pull_request_template.md

@@ -20,7 +20,7 @@ Your PR title must follow [conventional commits](https://www.conventionalcommits
 
 ### Deployment tag versioning
 
-Has `tag` in `common/version.go` been updated?
+Has `tag` in `common/version.go` been updated or have you added `bump-version` label to this PR?
 
 - [ ] No, this PR doesn't involve a new deployment, git tag, docker image tag
 - [ ] Yes

.github/workflows/bump_version.yml

@@ -1,20 +1,24 @@
-name: Bump Version
+name: Bump version
 
 on:
   pull_request:
-    branches: [develop]
+    branches: [ develop ]
     types:
       - opened
       - reopened
       - synchronize
       - ready_for_review
+      - labeled
 
 jobs:
   try-to-bump:
+    if: contains(github.event.pull_request.labels.*.name, 'bump-version')
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
       - name: check diff
         id: check_diff
         run: |
@@ -49,7 +53,11 @@ jobs:
       - name: bump version in common/version/version.go
         if: steps.check_diff.outputs.result == 'bump'
         run: node .github/scripts/bump_version_dot_go.mjs
+
+        # Commits made by this Action do not trigger new Workflow runs
       - uses: stefanzweifel/git-auto-commit-action@3ea6ae190baf489ba007f7c92608f33ce20ef04a
         if: steps.check_diff.outputs.result == 'bump'
         with:
+          skip_fetch: true # already did fetch in check diff
+          file_pattern: "common/version/version.go"
           commit_message: "chore: auto version bump [bot]"

.github/workflows/common.yml

@@ -9,6 +9,7 @@ on:
       - alpha
     paths:
       - 'common/**'
+      - '!common/version/version.go'
       - '.github/workflows/common.yml'
   pull_request:
     types:
@@ -18,6 +19,7 @@ on:
       - ready_for_review
     paths:
       - 'common/**'
+      - '!common/version/version.go'
       - '.github/workflows/common.yml'
 
 jobs:
@@ -88,8 +90,6 @@ jobs:
         version: '0.8.16'
     - name: Install Geth Tools
       uses: gacts/install-geth-tools@v1
-      with:
-        version: 1.10.19
     - name: Build prerequisites
       run: |
         make dev_docker

.github/workflows/coordinator.yml

@@ -10,6 +10,7 @@ on:
     paths:
       - 'coordinator/**'
       - 'common/**'
+      - '!common/version/version.go'
       - 'database/**'
       - '.github/workflows/coordinator.yml'
   pull_request:
@@ -21,6 +22,7 @@ on:
     paths:
       - 'coordinator/**'
       - 'common/**'
+      - '!common/version/version.go'
       - 'database/**'
       - '.github/workflows/coordinator.yml'
 
@@ -104,8 +106,6 @@ jobs:
         version: '0.8.16'
     - name: Install Geth Tools
       uses: gacts/install-geth-tools@v1
-      with:
-        version: 1.10.19
     - name: Build prerequisites
       run: |
         make dev_docker

.github/workflows/database.yml

@@ -10,6 +10,7 @@ on:
     paths:
       - 'database/**'
       - 'common/**'
+      - '!common/version/version.go'
       - '.github/workflows/database.yml'
   pull_request:
     types:
@@ -20,6 +21,7 @@ on:
     paths:
       - 'database/**'
       - 'common/**'
+      - '!common/version/version.go'
       - '.github/workflows/database.yml'
 
 jobs:
@@ -81,8 +83,6 @@ jobs:
         version: '0.8.16'
     - name: Install Geth Tools
       uses: gacts/install-geth-tools@v1
-      with:
-        version: 1.10.19
     - name: Build prerequisites
       run: |
         make dev_docker

.github/workflows/docker.yml

@@ -48,27 +48,6 @@ jobs:
         tags: scrolltech/gas-oracle:${{github.ref_name}}
         # cache-from: type=gha,scope=${{ github.workflow }}
         # cache-to: type=gha,scope=${{ github.workflow }}
-  msg_relayer:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v2
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
-    - name: Login to Docker Hub
-      uses: docker/login-action@v2
-      with:
-        username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - name: Build and push msg_relayer docker
-      uses: docker/build-push-action@v2
-      with:
-        context: .
-        file: ./build/dockerfiles/msg_relayer.Dockerfile
-        push: true
-        tags: scrolltech/msg-relayer:${{github.ref_name}}
-        # cache-from: type=gha,scope=${{ github.workflow }}
-        # cache-to: type=gha,scope=${{ github.workflow }}
   rollup_relayer:
     runs-on: ubuntu-latest
     steps:
@@ -153,24 +132,3 @@ jobs:
         tags: scrolltech/coordinator:${{github.ref_name}}
         # cache-from: type=gha,scope=${{ github.workflow }}
         # cache-to: type=gha,scope=${{ github.workflow }}
-  prover-stats-api:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v2
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
-    - name: Login to Docker Hub
-      uses: docker/login-action@v2
-      with:
-        username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - name: Build and push prover-stats-api docker
-      uses: docker/build-push-action@v2
-      with:
-        context: .
-        file: ./build/dockerfiles/prover-stats-api.Dockerfile
-        push: true
-        tags: scrolltech/prover-stats-api:${{github.ref_name}}
-        # cache-from: type=gha,scope=${{ github.workflow }}
-        # cache-to: type=gha,scope=${{ github.workflow }}

.github/workflows/integration.yml

@@ -33,12 +33,10 @@ jobs:
         version: '0.8.16'
     - name: Install Geth Tools
       uses: gacts/install-geth-tools@v1
-      with:
-        version: 1.10.19
     - name: Build prerequisites
       run: |
         make dev_docker
-        make -C bridge mock_abi
+        make -C rollup mock_abi
         make -C common/bytecode all
     - name: Run integration tests
       run: |

.github/workflows/prover_stats_api.yml

@@ -1,80 +0,0 @@
-name: ProverStatsAPI
-
-on:
-  push:
-    branches:
-      - main
-      - staging
-      - develop
-      - alpha
-    paths:
-      - 'prover-stats-api/**'
-      - '.github/workflows/prover_stats_api.yml'
-  pull_request:
-    types:
-      - opened
-      - reopened
-      - synchronize
-      - ready_for_review
-    paths:
-      - 'prover-stats-api/**'
-      - '.github/workflows/prover_stats_api.yml'
-
-defaults:
-  run:
-    working-directory: 'prover-stats-api'
-
-jobs:
-  check:
-     if: github.event.pull_request.draft == false
-     runs-on: ubuntu-latest
-     steps:
-     - name: Install Go
-       uses: actions/setup-go@v2
-       with:
-         go-version: 1.19.x
-     - name: Checkout code
-       uses: actions/checkout@v2
-     - name: Lint
-       run: |
-         rm -rf $HOME/.cache/golangci-lint
-         make lint
-  test:
-    if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest
-    steps:
-      - name: Install Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.19.x
-      - name: Checkout code
-        uses: actions/checkout@v2
-      - name: Test
-        run: |
-          make test
-      - name: Upload coverage reports to Codecov
-        uses: codecov/codecov-action@v3
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-        with:
-          flags: prover-stats-api
-  goimports-lint:
-    if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest
-    steps:
-      - name: Install Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.19.x
-      - name: Checkout code
-        uses: actions/checkout@v2
-      - name: Install goimports
-        run: go install golang.org/x/tools/cmd/goimports
-      - run: goimports -local scroll-tech/prover-stats-api/ -w .
-      - run: go mod tidy
-      # If there are any diffs from goimports or go mod tidy, fail.
-      - name: Verify no changes from goimports and go mod tidy
-        run: |
-          if [ -n "$(git status --porcelain)" ]; then
-            exit 1
-          fi

.github/workflows/rollup.yml

@@ -1,4 +1,4 @@
-name: Bridge
+name: Rollup
 
 on:
   push:
@@ -8,10 +8,11 @@ on:
       - develop
       - alpha
     paths:
-      - 'bridge/**'
+      - 'rollup/**'
       - 'common/**'
+      - '!common/version/version.go'
       - 'database/**'
-      - '.github/workflows/bridge.yml'
+      - '.github/workflows/rollup.yml'
   pull_request:
     types:
       - opened
@@ -19,10 +20,11 @@ on:
       - synchronize
       - ready_for_review
     paths:
-      - 'bridge/**'
+      - 'rollup/**'
       - 'common/**'
+      - '!common/version/version.go'
       - 'database/**'
-      - '.github/workflows/bridge.yml'
+      - '.github/workflows/rollup.yml'
 
 jobs:
   check:
@@ -43,10 +45,8 @@ jobs:
         version: '0.8.16'
     - name: Install Geth Tools
       uses: gacts/install-geth-tools@v1
-      with:
-        version: 1.10.19
     - name: Lint
-      working-directory: 'bridge'
+      working-directory: 'rollup'
       run: |
         rm -rf $HOME/.cache/golangci-lint
         make mock_abi
@@ -64,14 +64,14 @@ jobs:
     - name: Install goimports
       run: go install golang.org/x/tools/cmd/goimports
     - name: Run goimports lint
-      run: goimports -local scroll-tech/bridge/ -w .
-      working-directory: 'bridge'
+      run: goimports -local scroll-tech/rollup/ -w .
+      working-directory: 'rollup'
     - name: Run go mod tidy
       run: go mod tidy
-      working-directory: 'bridge'
+      working-directory: 'rollup'
     # If there are any diffs from goimports or go mod tidy, fail.
     - name: Verify no changes from goimports and go mod tidy
-      working-directory: 'bridge'
+      working-directory: 'rollup'
       run: |
         if [ -n "$(git status --porcelain)" ]; then
           exit 1
@@ -94,18 +94,16 @@ jobs:
         version: '0.8.16'
     - name: Install Geth Tools
       uses: gacts/install-geth-tools@v1
-      with:
-        version: 1.10.19
     - name: Build prerequisites
       run: |
         make dev_docker
-        make -C bridge mock_abi
-    - name: Build bridge binaries
-      working-directory: 'bridge'
+        make -C rollup mock_abi
+    - name: Build rollup binaries
+      working-directory: 'rollup'
       run: |
-        make bridge_bins
-    - name: Test bridge packages
-      working-directory: 'bridge'
+        make rollup_bins
+    - name: Test rollup packages
+      working-directory: 'rollup'
       run: |
         go test -v -race -gcflags="-l" -ldflags="-s=false" -coverprofile=coverage.txt -covermode=atomic ./...
     - name: Upload coverage reports to Codecov
@@ -113,7 +111,7 @@ jobs:
       env:
         CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
       with:
-        flags: bridge
+        flags: rollup
   # docker-build:
   #   if: github.event.pull_request.draft == false
   #   runs-on: ubuntu-latest

.gitignore

@@ -1,9 +1,22 @@
-.idea
+# Asset files
 assets/params*
 assets/seed
-coverage.txt
+
+# Built binaries
 build/bin
+
+coverage.txt
 *.integration.txt
 
+# Visual Studio Code
+.vscode
+
+# IntelliJ
+.idea
+
+# MacOS
+.DS_Store
+
 # misc
 sftp-config.json
+*~

CODE_OF_CONDUCT.md

@@ -0,0 +1,128 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+contributor@scroll.io.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.

CONTRIBUTING.md

@@ -0,0 +1,42 @@
+## Contributing
+
+[fork]: /fork
+[pr]: /compare
+[style]: https://standardjs.com/
+[code-of-conduct]: CODE_OF_CONDUCT.md
+
+Hi there! We're thrilled that you'd like to contribute to this project. Your help is essential for keeping it great.
+
+Please note that this project is released with a [Contributor Code of Conduct][code-of-conduct]. By participating in this project you agree to abide by its terms.
+
+## Contribute to Scroll
+
+Did you know there are many ways of contributing to Scroll? If you are looking to contribute to by adding Scroll to existing Dev Tools or by doing integrations please go to the [Contribute to Scroll](https://github.com/scroll-tech/contribute-to-scroll) repo instead. If you are looking to contribute to Scroll's Halo2 zkEVM circuits please refer to the [zkEVM circuits](https://github.com/scroll-tech/zkevm-circuits) repo. This repository covers the Scroll infrastructure and smart contracts, if you want to contribute to these areas continue reading this document.
+
+## Issues and PRs
+
+If you have suggestions for how this project could be improved, or want to report a bug, open an issue! We'd love all and any contributions. If you have questions, too, we'd love to hear them.
+
+We'd also love PRs. If you're thinking of a large PR, we advise opening up an issue first to talk about it, though! Look at the links below if you're not sure how to open a PR.
+
+## Submitting a pull request
+
+1. [Fork][fork] and clone the repository.
+1. Create a new branch: `git checkout -b my-branch-name`.
+1. Make your change, add tests, and make sure the tests still pass.
+1. Push to your fork and [submit a pull request][pr].
+1. Pat yourself on the back and wait for your pull request to be reviewed and merged.
+
+Here are a few things you can do that will increase the likelihood of your pull request being accepted:
+
+- Write and update tests.
+- Keep your changes as focused as possible. If there are multiple changes you would like to make that are not dependent upon each other, consider submitting them as separate pull requests.
+- Write a [good commit message](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).
+
+Work in Progress pull requests are also welcome to get feedback early on, or if there is something that blocked you.
+
+## Resources
+
+- [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/)
+- [Using Pull Requests](https://help.github.com/articles/about-pull-requests/)
+- [GitHub Help](https://help.github.com)

Makefile

@@ -1,6 +1,6 @@
 .PHONY: check update dev_docker build_test_docker run_test_docker clean
 
-L2GETH_TAG=scroll-v4.3.34
+L2GETH_TAG=scroll-v4.3.55
 
 help: ## Display this help message
 	@grep -h \
@@ -8,7 +8,7 @@ help: ## Display this help message
 		awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
 lint: ## The code's format and security checks.
-	make -C bridge lint
+	make -C rollup lint
 	make -C common lint
 	make -C coordinator lint
 	make -C database lint
@@ -17,7 +17,7 @@ lint: ## The code's format and security checks.
 
 update: ## update dependencies
 	go work sync
-	cd $(PWD)/bridge/ && go get -u github.com/scroll-tech/go-ethereum@${L2GETH_TAG} && go mod tidy
+	cd $(PWD)/rollup/ && go get -u github.com/scroll-tech/go-ethereum@${L2GETH_TAG} && go mod tidy
 	cd $(PWD)/bridge-history-api/ && go get -u github.com/ethereum/go-ethereum@latest && go mod tidy
 	cd $(PWD)/common/ && go get -u github.com/scroll-tech/go-ethereum@${L2GETH_TAG} && go mod tidy
 	cd $(PWD)/coordinator/ && go get -u github.com/scroll-tech/go-ethereum@${L2GETH_TAG} && go mod tidy
@@ -25,7 +25,7 @@ update: ## update dependencies
 	cd $(PWD)/prover/ && go get -u github.com/scroll-tech/go-ethereum@${L2GETH_TAG} && go mod tidy
 	cd $(PWD)/prover-stats-api/ && go get -u github.com/scroll-tech/go-ethereum@${L2GETH_TAG} && go mod tidy
 	cd $(PWD)/tests/integration-test/ && go get -u github.com/scroll-tech/go-ethereum@${L2GETH_TAG} && go mod tidy
-	goimports -local $(PWD)/bridge/ -w .
+	goimports -local $(PWD)/rollup/ -w .
 	goimports -local $(PWD)/bridge-history-api/ -w .
 	goimports -local $(PWD)/common/ -w .
 	goimports -local $(PWD)/coordinator/ -w .

README.md

@@ -1,7 +1,33 @@
 # Scroll Monorepo
 
+[![rollup](https://github.com/scroll-tech/scroll/actions/workflows/rollup.yml/badge.svg)](https://github.com/scroll-tech/scroll/actions/workflows/rollup.yml)
+[![contracts](https://github.com/scroll-tech/scroll/actions/workflows/contracts.yml/badge.svg)](https://github.com/scroll-tech/scroll/actions/workflows/contracts.yml)
+[![bridge-history](https://github.com/scroll-tech/scroll/actions/workflows/bridge_history_api.yml/badge.svg)](https://github.com/scroll-tech/scroll/actions/workflows/bridge_history_api.yml)
+[![coordinator](https://github.com/scroll-tech/scroll/actions/workflows/coordinator.yml/badge.svg)](https://github.com/scroll-tech/scroll/actions/workflows/coordinator.yml)
+[![prover](https://github.com/scroll-tech/scroll/actions/workflows/prover.yml/badge.svg)](https://github.com/scroll-tech/scroll/actions/workflows/prover.yml)
+[![integration](https://github.com/scroll-tech/scroll/actions/workflows/integration.yml/badge.svg)](https://github.com/scroll-tech/scroll/actions/workflows/integration.yml)
 [![codecov](https://codecov.io/gh/scroll-tech/scroll/branch/develop/graph/badge.svg?token=VJVHNQWGGW)](https://codecov.io/gh/scroll-tech/scroll)
 
+<a href="https://scroll.io">Scroll</a> is a zkRollup Layer 2 dedicated to enhance Ethereum scalability through a bytecode-equivalent [zkEVM](https://github.com/scroll-tech/zkevm-circuits) circuit. This monorepo encompasses essential infrastructure components of the Scroll protocol. It contains the L1 and L2 contracts, the rollup node, the prover client, and the prover coordinator.
+
+## Directory Structure
+
+<pre>
+├── <a href="./bridge-history-api/">bridge-history-api</a>: Bridge history service that collects deposit and withdraw events from both L1 and L2 chain and generates withdrawal proofs
+├── <a href="./common/">common</a>: Common libraries and types
+├── <a href="./coordinator/">coordinator</a>: Prover coordinator service that dispatches proving tasks to provers
+├── <a href="./database">database</a>: Database client and schema definition
+├── <a href="./src">l2geth</a>: Scroll execution node
+├── <a href="./prover">prover</a>: Prover client that runs proof generation for zkEVM circuit and aggregation circuit
+├── <a href="./rollup">rollup</a>: Rollup-related services
+├── <a href="./rpc-gateway">rpc-gateway</a>: RPC gateway external repo
+└── <a href="./tests">tests</a>: Integration tests
+</pre>
+
+## Contributing
+
+We welcome community contributions to this repository. Before you submit any issues or PRs, please read the [Code of Conduct](CODE_OF_CONDUCT.md) and the [Contribution Guideline](CONTRIBUTING.md).
+
 ## Prerequisites
 + Go 1.19
 + Rust (for version, see [rust-toolchain](./common/libzkp/impl/rust-toolchain))
@@ -15,14 +41,14 @@ docker pull postgres
 make dev_docker
 ```
 
-## Testing Bridge & Coordinator
+## Testing Rollup & Coordinator
 
 ### For Non-Apple Silicon (M1/M2) Macs
 
 Run the tests using the following commands:
 
 ```bash
-go test -v -race -covermode=atomic scroll-tech/bridge/...
+go test -v -race -covermode=atomic scroll-tech/rollup/...
 go test -tags="mock_verifier" -v -race -covermode=atomic scroll-tech/coordinator/...
 go test -v -race -covermode=atomic scroll-tech/database/...
 go test -v -race -covermode=atomic scroll-tech/common/...
@@ -55,7 +81,7 @@ This command runs a Docker container named `scroll_test_container` from the `scr
 Once the Docker container is running, execute the tests using the following commands:
 
 ```bash
-go test -v -race -covermode=atomic scroll-tech/bridge/...
+go test -v -race -covermode=atomic scroll-tech/rollup/...
 go test -tags="mock_verifier" -v -race -covermode=atomic scroll-tech/coordinator/...
 go test -v -race -covermode=atomic scroll-tech/database/...
 go test -v -race -covermode=atomic scroll-tech/common/...
@@ -63,6 +89,10 @@ go test -v -race -covermode=atomic scroll-tech/common/...
 
 ## Testing Contracts
 
-You can find the unit tests in [`<REPO_DIR>/contracts/src/test/`](/contracts/src/test/), and integration tests in [`<REPO_DIR>/contracts/integration-test/`](/contracts/integration-test/).
+You can find the unit tests in [`contracts/src/test/`](/contracts/src/test/), and integration tests in [`contracts/integration-test/`](/contracts/integration-test/).
 
-For more details on contracts, see [`/contracts`](/contracts).
+See [`contracts`](/contracts) for more details on the contracts.
+
+## License
+
+Scroll Monorepo is licensed under the [MIT](./LICENSE) license.

bridge-history-api/README.md

@@ -1 +1,82 @@
 # bridge-history-api
+
+This directory contains the `bridge-history-api` service that provides REST APIs to query txs interact with Scroll official bridge contracts
+
+## Instructions
+The bridge-history-api contains three distinct components
+
+### bridgehistoryapi-db-cli
+
+Provide init, show version, rollback, check status services of DB
+```
+    cd ./bridge-history-api
+    make bridgehistoryapi-db-cli
+    ./build/bin/bridgehistoryapi-db-cli [command]
+```
+
+### bridgehistoryapi-cross-msg-fetcher
+
+Fetch the transactions from both l1 and l2
+```
+    cd ./bridge-history-api
+    make bridgehistoryapi-cross-msg-fetcher
+    ./build/bin/bridgehistoryapi-cross-msg-fetcher
+```
+
+### bridgehistoryapi-server
+
+provides REST APIs. Please refer to the API details below.
+```
+    cd ./bridge-history-api
+    make bridgehistoryapi-server
+    ./build/bin/bridgehistoryapi-server
+```
+
+## APIs provided by bridgehistoryapi-server
+
+assume `bridgehistoryapi-server` listening on `https://localhost:8080`
+can change this port thru modify `config.json`
+
+1. `/txs`
+```
+// @Summary    	 get all txs under given address
+// @Accept       plain
+// @Produce      plain
+// @Param        address query string true "wallet address"
+// @Param        page_size query int true "page size"
+// @Param        page query int true "page"
+// @Success      200
+// @Router       /api/txs [get]
+```
+
+2. `/txsbyhashes`
+```
+// @Summary    	 get txs by given tx hashes
+// @Accept       plain
+// @Produce      plain
+// @Param        hashes  query  string array  true  "array of hashes list"
+// @Success      200  
+// @Router       /api/txsbyhashes [post]
+```
+
+3. `/claimable`
+```
+// @Summary    	 get all claimable txs under given address
+// @Accept       plain
+// @Produce      plain
+// @Param        address query string true "wallet address"
+// @Param        page_size query int true "page size"
+// @Param        page query int true "page"
+// @Success      200
+// @Router       /api/claimable [get]
+```
+
+4. `/withdraw_root`
+```
+// @Summary    	 get withdraw_root of given batch index
+// @Accept       plain
+// @Produce      plain
+// @Param        batch_index  query string  true  "batch_index"
+// @Success      200
+// @Router       /api/withdraw_root [get]
+```

bridge-history-api/abi/backend_abi.go

@@ -89,9 +89,6 @@ var (
 	// L2FailedRelayedMessageEventSignature = keccak256("FailedRelayedMessage(bytes32)")
 	L2FailedRelayedMessageEventSignature common.Hash
 
-	// L2ImportBlockEventSignature = keccak256("ImportBlock(bytes32,uint256,uint256,uint256,bytes32)")
-	L2ImportBlockEventSignature common.Hash
-
 	// L2AppendMessageEventSignature = keccak256("AppendMessage(uint256,bytes32)")
 	L2AppendMessageEventSignature common.Hash
 )
@@ -153,8 +150,6 @@ func init() {
 	L2RelayedMessageEventSignature = L2ScrollMessengerABI.Events["RelayedMessage"].ID
 	L2FailedRelayedMessageEventSignature = L2ScrollMessengerABI.Events["FailedRelayedMessage"].ID
 
-	L2ImportBlockEventSignature = L1BlockContainerABI.Events["ImportBlock"].ID
-
 	L2AppendMessageEventSignature = L2MessageQueueABI.Events["AppendMessage"].ID
 
 }

bridge-history-api/cmd/backend_server/app/app.go

@@ -7,11 +7,13 @@ import (
 
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/gin-gonic/gin"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/urfave/cli/v2"
 
 	"bridge-history-api/config"
 	"bridge-history-api/internal/controller"
 	"bridge-history-api/internal/route"
+	"bridge-history-api/observability"
 	"bridge-history-api/utils"
 )
 
@@ -54,13 +56,18 @@ func action(ctx *cli.Context) error {
 
 	router := gin.Default()
 	controller.InitController(db)
-	route.Route(router, cfg)
+
+	registry := prometheus.DefaultRegisterer
+	route.Route(router, cfg, registry)
 
 	go func() {
 		if runServerErr := router.Run(fmt.Sprintf(":%s", port)); runServerErr != nil {
 			log.Crit("run http server failure", "error", runServerErr)
 		}
 	}()
+
+	observability.Server(ctx, db)
+
 	// Catch CTRL-C to ensure a graceful shutdown.
 	interrupt := make(chan os.Signal, 1)
 	signal.Notify(interrupt, os.Interrupt)

bridge-history-api/go.mod

@@ -3,15 +3,20 @@ module bridge-history-api
 go 1.19
 
 require (
+	github.com/bits-and-blooms/bitset v1.7.0
 	github.com/ethereum/go-ethereum v1.12.2
 	github.com/gin-contrib/cors v1.4.0
+	github.com/gin-contrib/pprof v1.4.0
 	github.com/gin-gonic/gin v1.9.1
 	github.com/mattn/go-colorable v0.1.13
 	github.com/mattn/go-isatty v0.0.19
 	github.com/modern-go/reflect2 v1.0.2
+	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pressly/goose/v3 v3.7.0
+	github.com/prometheus/client_golang v1.14.0
 	github.com/stretchr/testify v1.8.3
 	github.com/urfave/cli/v2 v2.25.7
+	golang.org/x/sync v0.3.0
 	gorm.io/driver/postgres v1.5.0
 	gorm.io/gorm v1.25.2
 )
@@ -20,7 +25,6 @@ require (
 	github.com/DataDog/zstd v1.5.2 // indirect
 	github.com/VictoriaMetrics/fastcache v1.6.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/bits-and-blooms/bitset v1.7.0 // indirect
 	github.com/btcsuite/btcd v0.20.1-beta // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect
 	github.com/bytedance/sonic v1.9.2 // indirect
@@ -94,7 +98,6 @@ require (
 	github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_golang v1.14.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.39.0 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect
@@ -116,8 +119,7 @@ require (
 	golang.org/x/arch v0.4.0 // indirect
 	golang.org/x/crypto v0.12.0 // indirect
 	golang.org/x/exp v0.0.0-20230810033253-352e893a4cad // indirect
-	golang.org/x/net v0.12.0 // indirect
-	golang.org/x/sync v0.3.0 // indirect
+	golang.org/x/net v0.14.0 // indirect
 	golang.org/x/sys v0.11.0 // indirect
 	golang.org/x/text v0.12.0 // indirect
 	golang.org/x/time v0.3.0 // indirect

bridge-history-api/go.sum

@@ -119,6 +119,8 @@ github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnR
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/cors v1.4.0 h1:oJ6gwtUl3lqV0WEIwM/LxPF1QZ5qe2lGWdY2+bz7y0g=
 github.com/gin-contrib/cors v1.4.0/go.mod h1:bs9pNM0x/UsmHPBWT2xZz9ROh8xYjYkiURUfmBoMlcs=
+github.com/gin-contrib/pprof v1.4.0 h1:XxiBSf5jWZ5i16lNOPbMTVdgHBdhfGRD5PZ1LWazzvg=
+github.com/gin-contrib/pprof v1.4.0/go.mod h1:RrehPJasUVBPK6yTUwOl8/NP6i0vbUgmxtis+Z5KE90=
 github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
@@ -362,6 +364,8 @@ github.com/onsi/gomega v1.27.1 h1:rfztXRbg6nv/5f+Raen9RcGoSecHIFgBBLQK3Wdj754=
 github.com/onsi/gomega v1.27.1/go.mod h1:aHX5xOykVYzWOV4WqQy0sy8BQptgukenXpCXfadcIAw=
 github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo=
 github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
@@ -531,8 +535,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
 golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
-golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
+golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
+golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

bridge-history-api/internal/controller/batch_controller.go

@@ -24,14 +24,14 @@ func NewBatchController(db *gorm.DB) *BatchController {
 func (b *BatchController) GetWithdrawRootByBatchIndex(ctx *gin.Context) {
 	var req types.QueryByBatchIndexRequest
 	if err := ctx.ShouldBind(&req); err != nil {
-		types.RenderJSON(ctx, types.ErrParameterInvalidNo, err, nil)
+		types.RenderFailure(ctx, types.ErrParameterInvalidNo, err)
 		return
 	}
 	result, err := b.batchLogic.GetWithdrawRootByBatchIndex(ctx, req.BatchIndex)
 	if err != nil {
-		types.RenderJSON(ctx, types.ErrGetWithdrawRootByBatchIndexFailure, err, nil)
+		types.RenderFailure(ctx, types.ErrGetWithdrawRootByBatchIndexFailure, err)
 		return
 	}
 
-	types.RenderJSON(ctx, types.Success, nil, result)
+	types.RenderSuccess(ctx, result)
 }

bridge-history-api/internal/controller/controller.go

@@ -10,7 +10,8 @@ var (
 	// HistoryCtrler is controller instance
 	HistoryCtrler *HistoryController
 	// BatchCtrler is controller instance
-	BatchCtrler        *BatchController
+	BatchCtrler *BatchController
+
 	initControllerOnce sync.Once
 )
 

bridge-history-api/internal/controller/history_controller.go

@@ -1,23 +1,40 @@
 package controller
 
 import (
+	"errors"
+	"reflect"
+	"time"
+
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/gin-gonic/gin"
+	"github.com/patrickmn/go-cache"
+	"golang.org/x/sync/singleflight"
 	"gorm.io/gorm"
 
 	"bridge-history-api/internal/logic"
 	"bridge-history-api/internal/types"
 )
 
+const (
+	cacheKeyPrefixClaimableTxsByAddr = "claimableTxsByAddr:"
+	cacheKeyPrefixQueryTxsByHash     = "queryTxsByHash:"
+)
+
 // HistoryController contains the query claimable txs service
 type HistoryController struct {
 	historyLogic *logic.HistoryLogic
+	cache        *cache.Cache
+	singleFlight singleflight.Group
+	cacheMetrics *cacheMetrics
 }
 
 // NewHistoryController return HistoryController instance
 func NewHistoryController(db *gorm.DB) *HistoryController {
 	return &HistoryController{
 		historyLogic: logic.NewHistoryLogic(db),
+		cache:        cache.New(30*time.Second, 10*time.Minute),
+		cacheMetrics: initCacheMetrics(),
 	}
 }
 
@@ -25,48 +42,120 @@ func NewHistoryController(db *gorm.DB) *HistoryController {
 func (c *HistoryController) GetAllClaimableTxsByAddr(ctx *gin.Context) {
 	var req types.QueryByAddressRequest
 	if err := ctx.ShouldBind(&req); err != nil {
-		types.RenderJSON(ctx, types.ErrParameterInvalidNo, err, nil)
-		return
-	}
-	offset := (req.Page - 1) * req.PageSize
-	limit := req.PageSize
-	txs, total, err := c.historyLogic.GetClaimableTxsByAddress(ctx, common.HexToAddress(req.Address), offset, limit)
-	if err != nil {
-		types.RenderJSON(ctx, types.ErrGetClaimablesFailure, err, nil)
+		types.RenderFailure(ctx, types.ErrParameterInvalidNo, err)
 		return
 	}
 
-	types.RenderJSON(ctx, types.Success, nil, &types.ResultData{Result: txs, Total: total})
-}
+	cacheKey := cacheKeyPrefixClaimableTxsByAddr + req.Address
+	if cachedData, found := c.cache.Get(cacheKey); found {
+		c.cacheMetrics.cacheHits.WithLabelValues("GetAllClaimableTxsByAddr").Inc()
+		// Log cache hit along with request param.
+		log.Info("cache hit", "request", req)
+		if cachedData == nil {
+			types.RenderSuccess(ctx, &types.ResultData{})
+			return
+		} else if resultData, ok := cachedData.(*types.ResultData); ok {
+			types.RenderSuccess(ctx, resultData)
+			return
+		}
+		// Log error for unexpected type, then fetch data from the database.
+		log.Error("unexpected type in cache", "expected", "*types.ResultData", "got", reflect.TypeOf(cachedData))
+	} else {
+		c.cacheMetrics.cacheMisses.WithLabelValues("GetAllClaimableTxsByAddr").Inc()
+		// Log cache miss along with request param.
+		log.Info("cache miss", "request", req)
+	}
+
+	result, err, _ := c.singleFlight.Do(cacheKey, func() (interface{}, error) {
+		txs, total, err := c.historyLogic.GetClaimableTxsByAddress(ctx, common.HexToAddress(req.Address))
+		if err != nil {
+			return nil, err
+		}
+		resultData := &types.ResultData{Result: txs, Total: total}
+		c.cache.Set(cacheKey, resultData, cache.DefaultExpiration)
+		return resultData, nil
+	})
 
-// GetAllTxsByAddr defines the http get method behavior
-func (c *HistoryController) GetAllTxsByAddr(ctx *gin.Context) {
-	var req types.QueryByAddressRequest
-	if err := ctx.ShouldBind(&req); err != nil {
-		types.RenderJSON(ctx, types.ErrParameterInvalidNo, err, nil)
-		return
-	}
-	offset := (req.Page - 1) * req.PageSize
-	limit := req.PageSize
-	message, total, err := c.historyLogic.GetTxsByAddress(ctx, common.HexToAddress(req.Address), offset, limit)
 	if err != nil {
-		types.RenderJSON(ctx, types.ErrGetTxsByAddrFailure, err, nil)
+		types.RenderFailure(ctx, types.ErrGetClaimablesFailure, err)
 		return
 	}
-	types.RenderJSON(ctx, types.Success, nil, &types.ResultData{Result: message, Total: total})
+
+	if resultData, ok := result.(*types.ResultData); ok {
+		types.RenderSuccess(ctx, resultData)
+	} else {
+		log.Error("unexpected type from singleflight", "expected", "*types.ResultData", "got", reflect.TypeOf(result))
+		types.RenderFailure(ctx, types.ErrGetClaimablesFailure, errors.New("unexpected error"))
+	}
 }
 
 // PostQueryTxsByHash defines the http post method behavior
 func (c *HistoryController) PostQueryTxsByHash(ctx *gin.Context) {
 	var req types.QueryByHashRequest
 	if err := ctx.ShouldBindJSON(&req); err != nil {
-		types.RenderJSON(ctx, types.ErrParameterInvalidNo, err, nil)
+		types.RenderFailure(ctx, types.ErrParameterInvalidNo, err)
 		return
 	}
-	result, err := c.historyLogic.GetTxsByHashes(ctx, req.Txs)
-	if err != nil {
-		types.RenderJSON(ctx, types.ErrGetTxsByHashFailure, err, nil)
+
+	if len(req.Txs) > 10 {
+		types.RenderFailure(ctx, types.ErrParameterInvalidNo, errors.New("the number of hashes in the request exceeds the allowed maximum of 10"))
 		return
 	}
-	types.RenderJSON(ctx, types.Success, nil, &types.ResultData{Result: result, Total: 0})
+	hashesMap := make(map[string]struct{}, len(req.Txs))
+	results := make([]*types.TxHistoryInfo, 0, len(req.Txs))
+	uncachedHashes := make([]string, 0, len(req.Txs))
+	for _, hash := range req.Txs {
+		if _, exists := hashesMap[hash]; exists {
+			// Skip duplicate tx hash values.
+			continue
+		}
+		hashesMap[hash] = struct{}{}
+
+		cacheKey := cacheKeyPrefixQueryTxsByHash + hash
+		if cachedData, found := c.cache.Get(cacheKey); found {
+			c.cacheMetrics.cacheHits.WithLabelValues("PostQueryTxsByHash").Inc()
+			// Log cache hit along with tx hash.
+			log.Info("cache hit", "tx hash", hash)
+			if cachedData == nil {
+				continue
+			} else if txInfo, ok := cachedData.(*types.TxHistoryInfo); ok {
+				results = append(results, txInfo)
+			} else {
+				log.Error("unexpected type in cache", "expected", "*types.TxHistoryInfo", "got", reflect.TypeOf(cachedData))
+				uncachedHashes = append(uncachedHashes, hash)
+			}
+		} else {
+			c.cacheMetrics.cacheMisses.WithLabelValues("PostQueryTxsByHash").Inc()
+			// Log cache miss along with tx hash.
+			log.Info("cache miss", "tx hash", hash)
+			uncachedHashes = append(uncachedHashes, hash)
+		}
+	}
+
+	if len(uncachedHashes) > 0 {
+		dbResults, err := c.historyLogic.GetTxsByHashes(ctx, uncachedHashes)
+		if err != nil {
+			types.RenderFailure(ctx, types.ErrGetTxsByHashFailure, err)
+			return
+		}
+
+		resultMap := make(map[string]*types.TxHistoryInfo)
+		for _, result := range dbResults {
+			results = append(results, result)
+			resultMap[result.Hash] = result
+		}
+
+		for _, hash := range uncachedHashes {
+			cacheKey := cacheKeyPrefixQueryTxsByHash + hash
+			result, found := resultMap[hash]
+			if found {
+				c.cache.Set(cacheKey, result, cache.DefaultExpiration)
+			} else {
+				c.cache.Set(cacheKey, nil, cache.DefaultExpiration)
+			}
+		}
+	}
+
+	resultData := &types.ResultData{Result: results, Total: uint64(len(results))}
+	types.RenderSuccess(ctx, resultData)
 }

bridge-history-api/internal/controller/metrics.go

@@ -0,0 +1,40 @@
+package controller
+
+import (
+	"sync"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+type cacheMetrics struct {
+	cacheHits   *prometheus.CounterVec
+	cacheMisses *prometheus.CounterVec
+}
+
+var (
+	initMetricsOnce sync.Once
+	cm              *cacheMetrics
+)
+
+func initCacheMetrics() *cacheMetrics {
+	initMetricsOnce.Do(func() {
+		cm = &cacheMetrics{
+			cacheHits: promauto.NewCounterVec(
+				prometheus.CounterOpts{
+					Name: "bridge_history_api_cache_hits_total",
+					Help: "The total number of cache hits",
+				},
+				[]string{"api"},
+			),
+			cacheMisses: promauto.NewCounterVec(
+				prometheus.CounterOpts{
+					Name: "bridge_history_api_cache_misses_total",
+					Help: "The total number of cache misses",
+				},
+				[]string{"api"},
+			),
+		}
+	})
+	return cm
+}

bridge-history-api/internal/logic/history_logic.go

@@ -23,66 +23,101 @@ func NewHistoryLogic(db *gorm.DB) *HistoryLogic {
 	return logic
 }
 
-// getCrossTxClaimInfo get UserClaimInfos by address
-func getCrossTxClaimInfo(ctx context.Context, msgHash string, db *gorm.DB) *types.UserClaimInfo {
+// updateL2TxClaimInfo updates UserClaimInfos for each transaction history.
+func updateL2TxClaimInfo(ctx context.Context, txHistories []*types.TxHistoryInfo, db *gorm.DB) {
 	l2SentMsgOrm := orm.NewL2SentMsg(db)
 	rollupOrm := orm.NewRollupBatch(db)
-	l2sentMsg, err := l2SentMsgOrm.GetL2SentMsgByHash(ctx, msgHash)
-	if err != nil || l2sentMsg == nil {
-		log.Debug("getCrossTxClaimInfo failed", "error", err)
-		return &types.UserClaimInfo{}
-	}
-	batch, err := rollupOrm.GetRollupBatchByIndex(ctx, l2sentMsg.BatchIndex)
-	if err != nil {
-		log.Debug("getCrossTxClaimInfo failed", "error", err)
-		return &types.UserClaimInfo{}
-	}
-	return &types.UserClaimInfo{
-		From:       l2sentMsg.Sender,
-		To:         l2sentMsg.Target,
-		Value:      l2sentMsg.Value,
-		Nonce:      strconv.FormatUint(l2sentMsg.Nonce, 10),
-		Message:    l2sentMsg.MsgData,
-		Proof:      "0x" + l2sentMsg.MsgProof,
-		BatchHash:  batch.BatchHash,
-		BatchIndex: strconv.FormatUint(l2sentMsg.BatchIndex, 10),
+
+	var l2MsgHashes []string
+	for _, txHistory := range txHistories {
+		if !txHistory.IsL1 {
+			l2MsgHashes = append(l2MsgHashes, txHistory.MsgHash)
+		}
 	}
 
+	l2sentMsgs, err := l2SentMsgOrm.GetL2SentMsgsByHashes(ctx, l2MsgHashes)
+	if err != nil || len(l2sentMsgs) == 0 {
+		log.Debug("GetL2SentMsgsByHashes failed", "l2 sent msgs", l2sentMsgs, "error", err)
+		return
+	}
+
+	l2MsgMap := make(map[string]*orm.L2SentMsg, len(l2sentMsgs))
+	var batchIndexes []uint64
+	for _, l2sentMsg := range l2sentMsgs {
+		l2MsgMap[l2sentMsg.MsgHash] = l2sentMsg
+		batchIndexes = append(batchIndexes, l2sentMsg.BatchIndex)
+	}
+
+	batches, err := rollupOrm.GetRollupBatchesByIndexes(ctx, batchIndexes)
+	if err != nil {
+		log.Debug("GetRollupBatchesByIndexes failed", "error", err)
+		return
+	}
+
+	batchMap := make(map[uint64]*orm.RollupBatch, len(batches))
+	for _, batch := range batches {
+		batchMap[batch.BatchIndex] = batch
+	}
+
+	for _, txHistory := range txHistories {
+		if txHistory.IsL1 {
+			continue
+		}
+
+		l2sentMsg, foundL2SentMsg := l2MsgMap[txHistory.MsgHash]
+		batch, foundBatch := batchMap[l2sentMsg.BatchIndex]
+		if foundL2SentMsg && foundBatch {
+			txHistory.ClaimInfo = &types.UserClaimInfo{
+				From:       l2sentMsg.Sender,
+				To:         l2sentMsg.Target,
+				Value:      l2sentMsg.Value,
+				Nonce:      strconv.FormatUint(l2sentMsg.Nonce, 10),
+				Message:    l2sentMsg.MsgData,
+				Proof:      "0x" + l2sentMsg.MsgProof,
+				BatchHash:  batch.BatchHash,
+				BatchIndex: strconv.FormatUint(l2sentMsg.BatchIndex, 10),
+			}
+		}
+	}
 }
 
-func updateCrossTxHash(ctx context.Context, msgHash string, txInfo *types.TxHistoryInfo, db *gorm.DB) {
+func updateCrossTxHashes(ctx context.Context, txHistories []*types.TxHistoryInfo, db *gorm.DB) {
+	msgHashes := make([]string, len(txHistories))
+	for i, txHistory := range txHistories {
+		msgHashes[i] = txHistory.MsgHash
+	}
+
 	relayed := orm.NewRelayedMsg(db)
-	relayed, err := relayed.GetRelayedMsgByHash(ctx, msgHash)
-	if err != nil {
-		log.Debug("updateCrossTxHash failed", "error", err)
-		return
-	}
-	if relayed == nil {
-		return
-	}
-	if relayed.Layer1Hash != "" {
-		txInfo.FinalizeTx.Hash = relayed.Layer1Hash
-		txInfo.FinalizeTx.BlockNumber = relayed.Height
-		return
-	}
-	if relayed.Layer2Hash != "" {
-		txInfo.FinalizeTx.Hash = relayed.Layer2Hash
-		txInfo.FinalizeTx.BlockNumber = relayed.Height
+	relayedMsgs, err := relayed.GetRelayedMsgsByHashes(ctx, msgHashes)
+	if err != nil || len(relayedMsgs) == 0 {
+		log.Debug("GetRelayedMsgsByHashes failed", "msg hashes", msgHashes, "relayed msgs", relayedMsgs, "error", err)
 		return
 	}
 
+	relayedMsgMap := make(map[string]*orm.RelayedMsg, len(relayedMsgs))
+	for _, relayedMsg := range relayedMsgs {
+		relayedMsgMap[relayedMsg.MsgHash] = relayedMsg
+	}
+
+	for _, txHistory := range txHistories {
+		if relayedMsg, found := relayedMsgMap[txHistory.MsgHash]; found {
+			txHistory.FinalizeTx.Hash = relayedMsg.Layer1Hash + relayedMsg.Layer2Hash
+			txHistory.FinalizeTx.BlockNumber = relayedMsg.Height
+		}
+	}
+}
+
+func updateCrossTxHashesAndL2TxClaimInfo(ctx context.Context, txHistories []*types.TxHistoryInfo, db *gorm.DB) {
+	updateCrossTxHashes(ctx, txHistories, db)
+	updateL2TxClaimInfo(ctx, txHistories, db)
 }
 
 // GetClaimableTxsByAddress get all claimable txs under given address
-func (h *HistoryLogic) GetClaimableTxsByAddress(ctx context.Context, address common.Address, offset int, limit int) ([]*types.TxHistoryInfo, uint64, error) {
+func (h *HistoryLogic) GetClaimableTxsByAddress(ctx context.Context, address common.Address) ([]*types.TxHistoryInfo, uint64, error) {
 	var txHistories []*types.TxHistoryInfo
 	l2SentMsgOrm := orm.NewL2SentMsg(h.db)
 	l2CrossMsgOrm := orm.NewCrossMsg(h.db)
-	total, err := l2SentMsgOrm.GetClaimableL2SentMsgByAddressTotalNum(ctx, address.Hex())
-	if err != nil || total == 0 {
-		return txHistories, 0, err
-	}
-	results, err := l2SentMsgOrm.GetClaimableL2SentMsgByAddressWithOffset(ctx, address.Hex(), offset, limit)
+	results, err := l2SentMsgOrm.GetClaimableL2SentMsgByAddress(ctx, address.Hex())
 	if err != nil || len(results) == 0 {
 		return txHistories, 0, err
 	}
@@ -102,10 +137,10 @@ func (h *HistoryLogic) GetClaimableTxsByAddress(ctx context.Context, address com
 	for _, result := range results {
 		txInfo := &types.TxHistoryInfo{
 			Hash:        result.TxHash,
+			MsgHash:     result.MsgHash,
 			IsL1:        false,
 			BlockNumber: result.Height,
 			FinalizeTx:  &types.Finalized{},
-			ClaimInfo:   getCrossTxClaimInfo(ctx, result.MsgHash, h.db),
 		}
 		if crossMsg, exist := crossMsgMap[result.MsgHash]; exist {
 			txInfo.Amount = crossMsg.Amount
@@ -117,96 +152,36 @@ func (h *HistoryLogic) GetClaimableTxsByAddress(ctx context.Context, address com
 		}
 		txHistories = append(txHistories, txInfo)
 	}
-	return txHistories, total, err
-}
-
-// GetTxsByAddress get all txs under given address
-func (h *HistoryLogic) GetTxsByAddress(ctx context.Context, address common.Address, offset int, limit int) ([]*types.TxHistoryInfo, uint64, error) {
-	var txHistories []*types.TxHistoryInfo
-	utilOrm := orm.NewCrossMsg(h.db)
-	total, err := utilOrm.GetTotalCrossMsgCountByAddress(ctx, address.String())
-	if err != nil || total == 0 {
-		return txHistories, 0, err
-	}
-	result, err := utilOrm.GetCrossMsgsByAddressWithOffset(ctx, address.String(), offset, limit)
-
-	if err != nil {
-		return nil, 0, err
-	}
-	for _, msg := range result {
-		txHistory := &types.TxHistoryInfo{
-			Hash:           msg.Layer1Hash + msg.Layer2Hash,
-			Amount:         msg.Amount,
-			To:             msg.Target,
-			L1Token:        msg.Layer1Token,
-			L2Token:        msg.Layer2Token,
-			IsL1:           msg.MsgType == int(orm.Layer1Msg),
-			BlockNumber:    msg.Height,
-			BlockTimestamp: msg.Timestamp,
-			CreatedAt:      msg.CreatedAt,
-			FinalizeTx: &types.Finalized{
-				Hash: "",
-			},
-			ClaimInfo: getCrossTxClaimInfo(ctx, msg.MsgHash, h.db),
-		}
-		updateCrossTxHash(ctx, msg.MsgHash, txHistory, h.db)
-		txHistories = append(txHistories, txHistory)
-	}
-	return txHistories, total, nil
+	updateL2TxClaimInfo(ctx, txHistories, h.db)
+	return txHistories, uint64(len(results)), err
 }
 
 // GetTxsByHashes get tx infos under given tx hashes
 func (h *HistoryLogic) GetTxsByHashes(ctx context.Context, hashes []string) ([]*types.TxHistoryInfo, error) {
-	txHistories := make([]*types.TxHistoryInfo, 0)
 	CrossMsgOrm := orm.NewCrossMsg(h.db)
-	for _, hash := range hashes {
-		l1result, err := CrossMsgOrm.GetL1CrossMsgByHash(ctx, common.HexToHash(hash))
-		if err != nil {
-			return nil, err
-		}
-		if l1result != nil {
-			txHistory := &types.TxHistoryInfo{
-				Hash:           l1result.Layer1Hash,
-				Amount:         l1result.Amount,
-				To:             l1result.Target,
-				IsL1:           true,
-				L1Token:        l1result.Layer1Token,
-				L2Token:        l1result.Layer2Token,
-				BlockNumber:    l1result.Height,
-				BlockTimestamp: l1result.Timestamp,
-				CreatedAt:      l1result.CreatedAt,
-				FinalizeTx: &types.Finalized{
-					Hash: "",
-				},
-			}
-			updateCrossTxHash(ctx, l1result.MsgHash, txHistory, h.db)
-			txHistories = append(txHistories, txHistory)
-			continue
-		}
-		l2result, err := CrossMsgOrm.GetL2CrossMsgByHash(ctx, common.HexToHash(hash))
-		if err != nil {
-			return nil, err
-		}
-		if l2result != nil {
-			txHistory := &types.TxHistoryInfo{
-				Hash:           l2result.Layer2Hash,
-				Amount:         l2result.Amount,
-				To:             l2result.Target,
-				IsL1:           false,
-				L1Token:        l2result.Layer1Token,
-				L2Token:        l2result.Layer2Token,
-				BlockNumber:    l2result.Height,
-				BlockTimestamp: l2result.Timestamp,
-				CreatedAt:      l2result.CreatedAt,
-				FinalizeTx: &types.Finalized{
-					Hash: "",
-				},
-				ClaimInfo: getCrossTxClaimInfo(ctx, l2result.MsgHash, h.db),
-			}
-			updateCrossTxHash(ctx, l2result.MsgHash, txHistory, h.db)
-			txHistories = append(txHistories, txHistory)
-			continue
-		}
+	results, err := CrossMsgOrm.GetCrossMsgsByHashes(ctx, hashes)
+	if err != nil {
+		return nil, err
 	}
+
+	var txHistories []*types.TxHistoryInfo
+	for _, result := range results {
+		txHistory := &types.TxHistoryInfo{
+			Hash:           result.Layer1Hash + result.Layer2Hash,
+			MsgHash:        result.MsgHash,
+			Amount:         result.Amount,
+			To:             result.Target,
+			L1Token:        result.Layer1Token,
+			L2Token:        result.Layer2Token,
+			IsL1:           orm.MsgType(result.MsgType) == orm.Layer1Msg,
+			BlockNumber:    result.Height,
+			BlockTimestamp: result.Timestamp,
+			CreatedAt:      result.CreatedAt,
+			FinalizeTx:     &types.Finalized{Hash: ""},
+		}
+		txHistories = append(txHistories, txHistory)
+	}
+
+	updateCrossTxHashesAndL2TxClaimInfo(ctx, txHistories, h.db)
 	return txHistories, nil
 }

bridge-history-api/internal/route/route.go

@@ -5,13 +5,15 @@ import (
 
 	"github.com/gin-contrib/cors"
 	"github.com/gin-gonic/gin"
+	"github.com/prometheus/client_golang/prometheus"
 
 	"bridge-history-api/config"
 	"bridge-history-api/internal/controller"
+	"bridge-history-api/observability"
 )
 
 // Route routes the APIs
-func Route(router *gin.Engine, conf *config.Config) {
+func Route(router *gin.Engine, conf *config.Config, reg prometheus.Registerer) {
 	router.Use(cors.New(cors.Config{
 		AllowOrigins:     []string{"*"},
 		AllowMethods:     []string{"GET", "POST", "PUT", "DELETE"},
@@ -20,9 +22,9 @@ func Route(router *gin.Engine, conf *config.Config) {
 		MaxAge:           12 * time.Hour,
 	}))
 
+	observability.Use(router, "bridge_history_api", reg)
+
 	r := router.Group("api/")
-	r.GET("/txs", controller.HistoryCtrler.GetAllTxsByAddr)
 	r.POST("/txsbyhashes", controller.HistoryCtrler.PostQueryTxsByHash)
 	r.GET("/claimable", controller.HistoryCtrler.GetAllClaimableTxsByAddr)
-	r.GET("/withdraw_root", controller.BatchCtrler.GetWithdrawRootByBatchIndex)
 }

bridge-history-api/internal/types/history_types.go

@@ -10,6 +10,8 @@ import (
 const (
 	// Success shows OK.
 	Success = 0
+	// InternalServerError shows a fatal error in the server
+	InternalServerError = 500
 	// ErrParameterInvalidNo is invalid params
 	ErrParameterInvalidNo = 40001
 	// ErrGetClaimablesFailure is getting all claimables txs error
@@ -24,9 +26,7 @@ const (
 
 // QueryByAddressRequest the request parameter of address api
 type QueryByAddressRequest struct {
-	Address  string `form:"address" binding:"required"`
-	Page     int    `form:"page" binding:"required"`
-	PageSize int    `form:"page_size" binding:"required"`
+	Address string `form:"address" binding:"required"`
 }
 
 // QueryByHashRequest the request parameter of hash api
@@ -78,6 +78,7 @@ type UserClaimInfo struct {
 // TxHistoryInfo the schema of tx history infos
 type TxHistoryInfo struct {
 	Hash           string         `json:"hash"`
+	MsgHash        string         `json:"msgHash"`
 	Amount         string         `json:"amount"`
 	To             string         `json:"to"` // useless
 	IsL1           bool           `json:"isL1"`
@@ -103,3 +104,28 @@ func RenderJSON(ctx *gin.Context, errCode int, err error, data interface{}) {
 	}
 	ctx.JSON(http.StatusOK, renderData)
 }
+
+// RenderSuccess renders success response with json
+func RenderSuccess(ctx *gin.Context, data interface{}) {
+	RenderJSON(ctx, Success, nil, data)
+}
+
+// RenderFailure renders failure response with json
+func RenderFailure(ctx *gin.Context, errCode int, err error) {
+	RenderJSON(ctx, errCode, err, nil)
+}
+
+// RenderFatal renders fatal response with json
+func RenderFatal(ctx *gin.Context, err error) {
+	var errMsg string
+	if err != nil {
+		errMsg = err.Error()
+	}
+	renderData := Response{
+		ErrCode: InternalServerError,
+		ErrMsg:  errMsg,
+		Data:    nil,
+	}
+	ctx.Set("errcode", InternalServerError)
+	ctx.JSON(http.StatusInternalServerError, renderData)
+}

bridge-history-api/observability/middleware.go

@@ -1,10 +1,10 @@
-package metrics
+package observability
 
 import (
 	"github.com/gin-gonic/gin"
 	"github.com/prometheus/client_golang/prometheus"
 
-	"scroll-tech/common/metrics/ginmetrics"
+	"bridge-history-api/observability/ginmetrics"
 )
 
 // Use register the gin metric

bridge-history-api/observability/probes.go

@@ -0,0 +1,35 @@
+package observability
+
+import (
+	"github.com/gin-gonic/gin"
+	"gorm.io/gorm"
+
+	"bridge-history-api/internal/types"
+	"bridge-history-api/utils"
+)
+
+// ProbesController probe check controller
+type ProbesController struct {
+	db *gorm.DB
+}
+
+// NewProbesController returns an ProbesController instance
+func NewProbesController(db *gorm.DB) *ProbesController {
+	return &ProbesController{
+		db: db,
+	}
+}
+
+// HealthCheck the api controller for health check
+func (a *ProbesController) HealthCheck(c *gin.Context) {
+	if _, err := utils.Ping(a.db); err != nil {
+		types.RenderFatal(c, err)
+		return
+	}
+	types.RenderSuccess(c, nil)
+}
+
+// Ready the api controller for ready check
+func (a *ProbesController) Ready(c *gin.Context) {
+	types.RenderSuccess(c, nil)
+}

bridge-history-api/observability/server.go

@@ -0,0 +1,53 @@
+package observability
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"time"
+
+	// enable the pprof
+	_ "net/http/pprof"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/gin-contrib/pprof"
+	"github.com/gin-gonic/gin"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/urfave/cli/v2"
+	"gorm.io/gorm"
+
+	"bridge-history-api/utils"
+)
+
+// Server starts the metrics server on the given address, will be closed when the given
+// context is canceled.
+func Server(c *cli.Context, db *gorm.DB) {
+	if !c.Bool(utils.MetricsEnabled.Name) {
+		return
+	}
+
+	r := gin.New()
+	r.Use(gin.Recovery())
+	pprof.Register(r)
+	r.GET("/metrics", func(context *gin.Context) {
+		promhttp.Handler().ServeHTTP(context.Writer, context.Request)
+	})
+
+	probeController := NewProbesController(db)
+	r.GET("/health", probeController.HealthCheck)
+	r.GET("/ready", probeController.Ready)
+
+	address := fmt.Sprintf(":%s", c.String(utils.MetricsPort.Name))
+	server := &http.Server{
+		Addr:              address,
+		Handler:           r,
+		ReadHeaderTimeout: time.Minute,
+	}
+	log.Info("Starting metrics server", "address", address)
+
+	go func() {
+		if runServerErr := server.ListenAndServe(); runServerErr != nil && !errors.Is(runServerErr, http.ErrServerClosed) {
+			log.Crit("run metrics http server failure", "error", runServerErr)
+		}
+	}()
+}

bridge-history-api/orm/batch.go

@@ -71,6 +71,16 @@ func (r *RollupBatch) GetRollupBatchByIndex(ctx context.Context, index uint64) (
 	return &result, nil
 }
 
+// GetRollupBatchesByIndexes return the rollup batches by indexes
+func (r *RollupBatch) GetRollupBatchesByIndexes(ctx context.Context, indexes []uint64) ([]*RollupBatch, error) {
+	var results []*RollupBatch
+	err := r.db.WithContext(ctx).Model(&RollupBatch{}).Where("batch_index IN (?)", indexes).Find(&results).Error
+	if err != nil {
+		return nil, fmt.Errorf("RollupBatch.GetRollupBatchesByIndexes error: %w", err)
+	}
+	return results, nil
+}
+
 // InsertRollupBatch batch insert rollup batch into db and return the transaction
 func (r *RollupBatch) InsertRollupBatch(ctx context.Context, batches []*RollupBatch, dbTx ...*gorm.DB) error {
 	if len(batches) == 0 {

bridge-history-api/orm/cross_msg.go

@@ -368,3 +368,14 @@ func (c *CrossMsg) GetCrossMsgsByAddressWithOffset(ctx context.Context, sender s
 	}
 	return messages, nil
 }
+
+// GetCrossMsgsByHashes retrieves a list of cross messages identified by their Layer 1 or Layer 2 hashes.
+func (c *CrossMsg) GetCrossMsgsByHashes(ctx context.Context, hashes []string) ([]*CrossMsg, error) {
+	var results []*CrossMsg
+	err := c.db.WithContext(ctx).Model(&CrossMsg{}).Where("layer1_hash IN (?) OR layer2_hash IN (?)", hashes, hashes).Find(&results).Error
+	if err != nil {
+		return nil, fmt.Errorf("CrossMsg.GetCrossMsgsByHashes error: %w", err)
+	}
+
+	return results, nil
+}

bridge-history-api/orm/l2_sent_msg.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/ethereum/go-ethereum/log"
@@ -54,6 +55,19 @@ func (l *L2SentMsg) GetL2SentMsgByHash(ctx context.Context, msgHash string) (*L2
 	return &result, nil
 }
 
+// GetL2SentMsgsByHashes get l2 sent msgs by hashes
+func (l *L2SentMsg) GetL2SentMsgsByHashes(ctx context.Context, msgHashes []string) ([]*L2SentMsg, error) {
+	var results []*L2SentMsg
+	err := l.db.WithContext(ctx).Model(&L2SentMsg{}).
+		Where("msg_hash IN (?)", msgHashes).
+		Find(&results).
+		Error
+	if err != nil {
+		return nil, fmt.Errorf("L2SentMsg.GetL2SentMsgsByHashes error: %w", err)
+	}
+	return results, nil
+}
+
 // GetLatestSentMsgHeightOnL2 get latest sent msg height on l2
 func (l *L2SentMsg) GetLatestSentMsgHeightOnL2(ctx context.Context) (uint64, error) {
 	var result L2SentMsg
@@ -72,26 +86,61 @@ func (l *L2SentMsg) GetLatestSentMsgHeightOnL2(ctx context.Context) (uint64, err
 	return result.Height, nil
 }
 
-// GetClaimableL2SentMsgByAddressWithOffset get claimable l2 sent msg by address with offset
-func (l *L2SentMsg) GetClaimableL2SentMsgByAddressWithOffset(ctx context.Context, address string, offset int, limit int) ([]*L2SentMsg, error) {
-	var results []*L2SentMsg
-	err := l.db.WithContext(ctx).Raw(`SELECT * FROM l2_sent_msg WHERE id NOT IN (SELECT l2_sent_msg.id FROM l2_sent_msg INNER JOIN relayed_msg ON l2_sent_msg.msg_hash = relayed_msg.msg_hash WHERE l2_sent_msg.deleted_at IS NULL AND relayed_msg.deleted_at IS NULL) AND (original_sender=$1 OR sender = $1) AND msg_proof !='' ORDER BY id DESC LIMIT $2 OFFSET $3;`, address, limit, offset).
-		Scan(&results).Error
-	if err != nil {
-		return nil, fmt.Errorf("L2SentMsg.GetClaimableL2SentMsgByAddressWithOffset error: %w", err)
+// GetClaimableL2SentMsgByAddress returns both the total number of unclaimed messages and a paginated list of those messages.
+// TODO: Add metrics about the result set sizes (total/claimed/unclaimed messages).
+func (l *L2SentMsg) GetClaimableL2SentMsgByAddress(ctx context.Context, address string) ([]*L2SentMsg, error) {
+	var totalMsgs []*L2SentMsg
+	db := l.db.WithContext(ctx)
+	db = db.Table("l2_sent_msg")
+	db = db.Where("original_sender = ? OR sender = ?", address, address)
+	db = db.Where("msg_proof != ''")
+	db = db.Where("deleted_at IS NULL")
+	db = db.Order("id DESC")
+	tx := db.Find(&totalMsgs)
+	if tx.Error != nil || tx.RowsAffected == 0 {
+		return nil, tx.Error
 	}
-	return results, nil
-}
 
-// GetClaimableL2SentMsgByAddressTotalNum get claimable l2 sent msg by address total num
-func (l *L2SentMsg) GetClaimableL2SentMsgByAddressTotalNum(ctx context.Context, address string) (uint64, error) {
-	var count uint64
-	err := l.db.WithContext(ctx).Raw(`SELECT COUNT(*) FROM l2_sent_msg WHERE id NOT IN (SELECT l2_sent_msg.id FROM l2_sent_msg INNER JOIN relayed_msg ON l2_sent_msg.msg_hash = relayed_msg.msg_hash WHERE l2_sent_msg.deleted_at IS NULL AND relayed_msg.deleted_at IS NULL) AND (original_sender=$1 OR sender = $1) AND msg_proof !='';`, address).
-		Scan(&count).Error
-	if err != nil {
-		return 0, fmt.Errorf("L2SentMsg.GetClaimableL2SentMsgByAddressTotalNum error: %w", err)
+	// Note on the use of IN vs VALUES in SQL Queries:
+	// ------------------------------------------------
+	// When using the IN predicate with a large list (>100) of values, performance may suffer.
+	// An alternative approach is to use constant subqueries with the VALUES construct.
+	// For more details and optimization tips, visit:
+	// https://postgres.cz/wiki/PostgreSQL_SQL_Tricks_I#Predicate_IN_optimalization
+	//
+	// Example using IN:
+	// SELECT * FROM tab WHERE x IN (1,2,3,...,n);  -- where n > 70
+	//
+	// Optimized example using VALUES:
+	// SELECT * FROM tab WHERE x IN (VALUES(10), (20));
+	//
+	var valuesStr string
+	for _, msg := range totalMsgs {
+		valuesStr += fmt.Sprintf("('%s'),", msg.MsgHash)
 	}
-	return count, nil
+	valuesStr = strings.TrimSuffix(valuesStr, ",")
+
+	var claimedMsgHashes []string
+	db = l.db.WithContext(ctx)
+	db = db.Table("relayed_msg")
+	db = db.Where(fmt.Sprintf("msg_hash IN (VALUES %s)", valuesStr))
+	db = db.Where("deleted_at IS NULL")
+	if err := db.Pluck("msg_hash", &claimedMsgHashes).Error; err != nil {
+		return nil, err
+	}
+
+	claimedMsgHashSet := make(map[string]struct{})
+	for _, hash := range claimedMsgHashes {
+		claimedMsgHashSet[hash] = struct{}{}
+	}
+	var unclaimedL2Msgs []*L2SentMsg
+	for _, msg := range totalMsgs {
+		if _, found := claimedMsgHashSet[msg.MsgHash]; !found {
+			unclaimedL2Msgs = append(unclaimedL2Msgs, msg)
+		}
+	}
+
+	return unclaimedL2Msgs, nil
 }
 
 // GetLatestL2SentMsgBatchIndex get latest l2 sent msg batch index

bridge-history-api/orm/l2_sent_msg_test.go

@@ -0,0 +1,77 @@
+package orm
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"bridge-history-api/orm/migrate"
+
+	"scroll-tech/common/database"
+	"scroll-tech/common/docker"
+)
+
+func TestGetClaimableL2SentMsgByAddress(t *testing.T) {
+	base := docker.NewDockerApp()
+	base.RunDBImage(t)
+
+	db, err := database.InitDB(
+		&database.Config{
+			DSN:        base.DBConfig.DSN,
+			DriverName: base.DBConfig.DriverName,
+			MaxOpenNum: base.DBConfig.MaxOpenNum,
+			MaxIdleNum: base.DBConfig.MaxIdleNum,
+		},
+	)
+	assert.NoError(t, err)
+
+	sqlDB, err := db.DB()
+	assert.NoError(t, err)
+	assert.NoError(t, migrate.ResetDB(sqlDB))
+
+	l2SentMsgOrm := NewL2SentMsg(db)
+	relayedMsgOrm := NewRelayedMsg(db)
+
+	msgs, err := l2SentMsgOrm.GetClaimableL2SentMsgByAddress(context.Background(), "sender1")
+	assert.NoError(t, err)
+	assert.Len(t, msgs, 0)
+
+	l2SentMsgs := []*L2SentMsg{
+		{
+			Sender:   "sender1",
+			MsgHash:  "hash1",
+			MsgProof: "proof1",
+			Nonce:    0,
+		},
+		{
+			OriginalSender: "sender1",
+			MsgHash:        "hash2",
+			MsgProof:       "proof2",
+			Nonce:          1,
+		},
+		{
+			OriginalSender: "sender1",
+			MsgHash:        "hash3",
+			MsgProof:       "",
+			Nonce:          2,
+		},
+	}
+	relayedMsgs := []*RelayedMsg{
+		{
+			MsgHash: "hash2",
+		},
+		{
+			MsgHash: "hash3",
+		},
+	}
+	err = l2SentMsgOrm.InsertL2SentMsg(context.Background(), l2SentMsgs)
+	assert.NoError(t, err)
+	err = relayedMsgOrm.InsertRelayedMsg(context.Background(), relayedMsgs)
+	assert.NoError(t, err)
+
+	msgs, err = l2SentMsgOrm.GetClaimableL2SentMsgByAddress(context.Background(), "sender1")
+	assert.NoError(t, err)
+	assert.Len(t, msgs, 1)
+	assert.Equal(t, "hash1", msgs[0].MsgHash)
+}

bridge-history-api/orm/relayed_msg.go

@@ -49,6 +49,19 @@ func (r *RelayedMsg) GetRelayedMsgByHash(ctx context.Context, msgHash string) (*
 	return &result, nil
 }
 
+// GetRelayedMsgsByHashes get relayed msg by hash array
+func (r *RelayedMsg) GetRelayedMsgsByHashes(ctx context.Context, msgHashes []string) ([]*RelayedMsg, error) {
+	var results []*RelayedMsg
+	err := r.db.WithContext(ctx).Model(&RelayedMsg{}).
+		Where("msg_hash IN (?)", msgHashes).
+		Find(&results).
+		Error
+	if err != nil {
+		return nil, fmt.Errorf("RelayedMsg.GetRelayedMsgsByHashes error: %w", err)
+	}
+	return results, nil
+}
+
 // GetLatestRelayedHeightOnL1 get latest relayed height on l1
 func (r *RelayedMsg) GetLatestRelayedHeightOnL1(ctx context.Context) (uint64, error) {
 	var result RelayedMsg

bridge-history-api/utils/database.go

@@ -2,6 +2,7 @@ package utils
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 	"time"
 
@@ -67,18 +68,29 @@ func InitDB(config *config.DBConfig) (*gorm.DB, error) {
 	if err != nil {
 		return nil, err
 	}
-	sqlDB, err := db.DB()
-	if err != nil {
-		return nil, err
+
+	sqlDB, pingErr := Ping(db)
+	if pingErr != nil {
+		return nil, pingErr
 	}
 
 	sqlDB.SetMaxOpenConns(config.MaxOpenNum)
 	sqlDB.SetMaxIdleConns(config.MaxIdleNum)
 
+	return db, nil
+}
+
+// Ping check db status
+func Ping(db *gorm.DB) (*sql.DB, error) {
+	sqlDB, err := db.DB()
+	if err != nil {
+		return nil, err
+	}
+
 	if err = sqlDB.Ping(); err != nil {
 		return nil, err
 	}
-	return db, nil
+	return sqlDB, nil
 }
 
 // CloseDB close the db handler. notice the db handler only can close when then program exit.

bridge-history-api/utils/parse_event.go

@@ -14,14 +14,6 @@ import (
 	"bridge-history-api/orm"
 )
 
-// CachedParsedTxCalldata store parsed batch infos
-type CachedParsedTxCalldata struct {
-	CallDataIndex uint64
-	BatchIndices  []uint64
-	StartBlocks   []uint64
-	EndBlocks     []uint64
-}
-
 // ParseBackendL1EventLogs parses L1 watched events
 func ParseBackendL1EventLogs(logs []types.Log) ([]*orm.CrossMsg, []*orm.RelayedMsg, error) {
 	// Need use contract abi to parse event Log

bridge/README.md

@@ -1,36 +0,0 @@
-# Bridge
-
-This repo contains the Scroll bridge.
-
-In addition, launching the bridge will launch a separate instance of l2geth, and sets up a communication channel
-between the two, over JSON-RPC sockets.
-
-Something we should pay attention is that all private keys inside sender instance cannot be duplicated.
-
-## Dependency
-
-+ install `abigen`
-
-``` bash
-go install -v github.com/scroll-tech/go-ethereum/cmd/abigen
-```
-
-## Build
-
-```bash
-make clean
-make bridge
-```
-
-## Start
-* use default ports and config.json
-
-```bash
-./build/bin/bridge --http
-```
-
-* use specified ports and config.json
-
-```bash
-./build/bin/bridge --config ./config.json --http --http.addr localhost --http.port 8290
-```

bridge/cmd/event_watcher/main.go

@@ -1,7 +0,0 @@
-package main
-
-import "scroll-tech/bridge/cmd/event_watcher/app"
-
-func main() {
-	app.Run()
-}

bridge/cmd/gas_oracle/main.go

@@ -1,7 +0,0 @@
-package main
-
-import "scroll-tech/bridge/cmd/gas_oracle/app"
-
-func main() {
-	app.Run()
-}

bridge/cmd/msg_relayer/app/app.go

@@ -1,94 +0,0 @@
-package app
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"os/signal"
-	"time"
-
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/scroll-tech/go-ethereum/log"
-	"github.com/urfave/cli/v2"
-
-	"scroll-tech/common/database"
-	"scroll-tech/common/metrics"
-	"scroll-tech/common/utils"
-	"scroll-tech/common/version"
-
-	"scroll-tech/bridge/internal/config"
-	"scroll-tech/bridge/internal/controller/relayer"
-)
-
-var app *cli.App
-
-func init() {
-	// Set up message-relayer app info.
-	app = cli.NewApp()
-	app.Action = action
-	app.Name = "message-relayer"
-	app.Usage = "The Scroll Message Relayer"
-	app.Description = "Message Relayer contains two main service: 1) relay l1 message to l2. 2) relay l2 message to l1."
-	app.Version = version.Version
-	app.Flags = append(app.Flags, utils.CommonFlags...)
-	app.Commands = []*cli.Command{}
-	app.Before = func(ctx *cli.Context) error {
-		return utils.LogSetup(ctx)
-	}
-	// Register `message-relayer-test` app for integration-test.
-	utils.RegisterSimulation(app, utils.MessageRelayerApp)
-}
-
-func action(ctx *cli.Context) error {
-	// Load config file.
-	cfgFile := ctx.String(utils.ConfigFileFlag.Name)
-	cfg, err := config.NewConfig(cfgFile)
-	if err != nil {
-		log.Crit("failed to load config file", "config file", cfgFile, "error", err)
-	}
-
-	subCtx, cancel := context.WithCancel(ctx.Context)
-	// Init db connection
-	db, err := database.InitDB(cfg.DBConfig)
-	if err != nil {
-		log.Crit("failed to init db connection", "err", err)
-	}
-	defer func() {
-		cancel()
-		if err = database.CloseDB(db); err != nil {
-			log.Error("can not close ormFactory", "error", err)
-		}
-	}()
-
-	registry := prometheus.DefaultRegisterer
-	metrics.Server(ctx, registry.(*prometheus.Registry))
-
-	l1relayer, err := relayer.NewLayer1Relayer(ctx.Context, db, cfg.L1Config.RelayerConfig, registry)
-	if err != nil {
-		log.Error("failed to create new l1 relayer", "config file", cfgFile, "error", err)
-		return err
-	}
-
-	// Start l1relayer process
-	go utils.Loop(subCtx, 10*time.Second, l1relayer.ProcessSavedEvents)
-
-	// Finish start all message relayer functions
-	log.Info("Start message-relayer successfully")
-
-	// Catch CTRL-C to ensure a graceful shutdown.
-	interrupt := make(chan os.Signal, 1)
-	signal.Notify(interrupt, os.Interrupt)
-
-	// Wait until the interrupt signal is received from an OS signal.
-	<-interrupt
-
-	return nil
-}
-
-// Run message_relayer cmd instance.
-func Run() {
-	if err := app.Run(os.Args); err != nil {
-		_, _ = fmt.Fprintln(os.Stderr, err)
-		os.Exit(1)
-	}
-}

bridge/cmd/msg_relayer/main.go

@@ -1,7 +0,0 @@
-package main
-
-import "scroll-tech/bridge/cmd/msg_relayer/app"
-
-func main() {
-	app.Run()
-}

bridge/cmd/rollup_relayer/main.go

@@ -1,7 +0,0 @@
-package main
-
-import "scroll-tech/bridge/cmd/rollup_relayer/app"
-
-func main() {
-	app.Run()
-}

bridge/internal/controller/relayer/l1_relayer_metrics.go

@@ -1,54 +0,0 @@
-package relayer
-
-import (
-	"sync"
-
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-)
-
-type l1RelayerMetrics struct {
-	bridgeL1RelayedMsgsTotal               prometheus.Counter
-	bridgeL1RelayedMsgsFailureTotal        prometheus.Counter
-	bridgeL1RelayerGasPriceOraclerRunTotal prometheus.Counter
-	bridgeL1RelayerLastGasPrice            prometheus.Gauge
-	bridgeL1MsgsRelayedConfirmedTotal      prometheus.Counter
-	bridgeL1GasOraclerConfirmedTotal       prometheus.Counter
-}
-
-var (
-	initL1RelayerMetricOnce sync.Once
-	l1RelayerMetric         *l1RelayerMetrics
-)
-
-func initL1RelayerMetrics(reg prometheus.Registerer) *l1RelayerMetrics {
-	initL1RelayerMetricOnce.Do(func() {
-		l1RelayerMetric = &l1RelayerMetrics{
-			bridgeL1RelayedMsgsTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-				Name: "bridge_layer1_msg_relayed_total",
-				Help: "The total number of the l1 relayed message.",
-			}),
-			bridgeL1RelayedMsgsFailureTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-				Name: "bridge_layer1_msg_relayed_failure_total",
-				Help: "The total number of the l1 relayed failure message.",
-			}),
-			bridgeL1MsgsRelayedConfirmedTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-				Name: "bridge_layer1_relayed_confirmed_total",
-				Help: "The total number of layer1 relayed confirmed",
-			}),
-			bridgeL1RelayerGasPriceOraclerRunTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-				Name: "bridge_layer1_gas_price_oracler_total",
-				Help: "The total number of layer1 gas price oracler run total",
-			}),
-			bridgeL1RelayerLastGasPrice: promauto.With(reg).NewGauge(prometheus.GaugeOpts{
-				Name: "bridge_layer1_gas_price_latest_gas_price",
-				Help: "The latest gas price of bridge relayer l1",
-			}),
-			bridgeL1GasOraclerConfirmedTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-				Name: "bridge_layer1_gas_oracler_confirmed_total",
-				Help: "The total number of layer1 relayed confirmed",
-			}),
-		}
-	})
-	return l1RelayerMetric
-}

bridge/internal/controller/relayer/l2_relayer_metrics.go

@@ -1,74 +0,0 @@
-package relayer
-
-import (
-	"sync"
-
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-)
-
-type l2RelayerMetrics struct {
-	bridgeL2RelayerProcessPendingBatchTotal                     prometheus.Counter
-	bridgeL2RelayerProcessPendingBatchSuccessTotal              prometheus.Counter
-	bridgeL2RelayerGasPriceOraclerRunTotal                      prometheus.Counter
-	bridgeL2RelayerLastGasPrice                                 prometheus.Gauge
-	bridgeL2RelayerProcessCommittedBatchesTotal                 prometheus.Counter
-	bridgeL2RelayerProcessCommittedBatchesFinalizedTotal        prometheus.Counter
-	bridgeL2RelayerProcessCommittedBatchesFinalizedSuccessTotal prometheus.Counter
-	bridgeL2BatchesCommittedConfirmedTotal                      prometheus.Counter
-	bridgeL2BatchesFinalizedConfirmedTotal                      prometheus.Counter
-	bridgeL2BatchesGasOraclerConfirmedTotal                     prometheus.Counter
-}
-
-var (
-	initL2RelayerMetricOnce sync.Once
-	l2RelayerMetric         *l2RelayerMetrics
-)
-
-func initL2RelayerMetrics(reg prometheus.Registerer) *l2RelayerMetrics {
-	initL2RelayerMetricOnce.Do(func() {
-		l2RelayerMetric = &l2RelayerMetrics{
-			bridgeL2RelayerProcessPendingBatchTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-				Name: "bridge_layer2_process_pending_batch_total",
-				Help: "The total number of layer2 process pending batch",
-			}),
-			bridgeL2RelayerProcessPendingBatchSuccessTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-				Name: "bridge_layer2_process_pending_batch_success_total",
-				Help: "The total number of layer2 process pending success batch",
-			}),
-			bridgeL2RelayerGasPriceOraclerRunTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-				Name: "bridge_layer2_gas_price_oracler_total",
-				Help: "The total number of layer2 gas price oracler run total",
-			}),
-			bridgeL2RelayerLastGasPrice: promauto.With(reg).NewGauge(prometheus.GaugeOpts{
-				Name: "bridge_layer2_gas_price_latest_gas_price",
-				Help: "The latest gas price of bridge relayer l2",
-			}),
-			bridgeL2RelayerProcessCommittedBatchesTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-				Name: "bridge_layer2_process_committed_batches_total",
-				Help: "The total number of layer2 process committed batches run total",
-			}),
-			bridgeL2RelayerProcessCommittedBatchesFinalizedTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-				Name: "bridge_layer2_process_committed_batches_finalized_total",
-				Help: "The total number of layer2 process committed batches finalized total",
-			}),
-			bridgeL2RelayerProcessCommittedBatchesFinalizedSuccessTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-				Name: "bridge_layer2_process_committed_batches_finalized_success_total",
-				Help: "The total number of layer2 process committed batches finalized success total",
-			}),
-			bridgeL2BatchesCommittedConfirmedTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-				Name: "bridge_layer2_process_committed_batches_confirmed_total",
-				Help: "The total number of layer2 process committed batches confirmed total",
-			}),
-			bridgeL2BatchesFinalizedConfirmedTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-				Name: "bridge_layer2_process_finalized_batches_confirmed_total",
-				Help: "The total number of layer2 process finalized batches confirmed total",
-			}),
-			bridgeL2BatchesGasOraclerConfirmedTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-				Name: "bridge_layer2_process_gras_oracler_confirmed_total",
-				Help: "The total number of layer2 process finalized batches confirmed total",
-			}),
-		}
-	})
-	return l2RelayerMetric
-}

bridge/internal/controller/watcher/batch_proposer.go

@@ -1,272 +0,0 @@
-package watcher
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-	"github.com/scroll-tech/go-ethereum/log"
-	"gorm.io/gorm"
-
-	"scroll-tech/common/types"
-
-	"scroll-tech/bridge/internal/config"
-	"scroll-tech/bridge/internal/orm"
-)
-
-// BatchProposer proposes batches based on available unbatched chunks.
-type BatchProposer struct {
-	ctx context.Context
-	db  *gorm.DB
-
-	batchOrm   *orm.Batch
-	chunkOrm   *orm.Chunk
-	l2BlockOrm *orm.L2Block
-
-	maxChunkNumPerBatch             uint64
-	maxL1CommitGasPerBatch          uint64
-	maxL1CommitCalldataSizePerBatch uint32
-	minChunkNumPerBatch             uint64
-	batchTimeoutSec                 uint64
-	gasCostIncreaseMultiplier       float64
-
-	batchProposerCircleTotal           prometheus.Counter
-	proposeBatchFailureTotal           prometheus.Counter
-	proposeBatchUpdateInfoTotal        prometheus.Counter
-	proposeBatchUpdateInfoFailureTotal prometheus.Counter
-	totalL1CommitGas                   prometheus.Gauge
-	totalL1CommitCalldataSize          prometheus.Gauge
-	batchChunksNum                     prometheus.Gauge
-	batchFirstChunkTimeoutReached      prometheus.Counter
-	batchChunksSuperposeNotEnoughTotal prometheus.Counter
-}
-
-// NewBatchProposer creates a new BatchProposer instance.
-func NewBatchProposer(ctx context.Context, cfg *config.BatchProposerConfig, db *gorm.DB, reg prometheus.Registerer) *BatchProposer {
-	return &BatchProposer{
-		ctx:                             ctx,
-		db:                              db,
-		batchOrm:                        orm.NewBatch(db),
-		chunkOrm:                        orm.NewChunk(db),
-		l2BlockOrm:                      orm.NewL2Block(db),
-		maxChunkNumPerBatch:             cfg.MaxChunkNumPerBatch,
-		maxL1CommitGasPerBatch:          cfg.MaxL1CommitGasPerBatch,
-		maxL1CommitCalldataSizePerBatch: cfg.MaxL1CommitCalldataSizePerBatch,
-		minChunkNumPerBatch:             cfg.MinChunkNumPerBatch,
-		batchTimeoutSec:                 cfg.BatchTimeoutSec,
-		gasCostIncreaseMultiplier:       cfg.GasCostIncreaseMultiplier,
-
-		batchProposerCircleTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-			Name: "bridge_propose_batch_circle_total",
-			Help: "Total number of propose batch total.",
-		}),
-		proposeBatchFailureTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-			Name: "bridge_propose_batch_failure_circle_total",
-			Help: "Total number of propose batch total.",
-		}),
-		proposeBatchUpdateInfoTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-			Name: "bridge_propose_batch_update_info_total",
-			Help: "Total number of propose batch update info total.",
-		}),
-		proposeBatchUpdateInfoFailureTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-			Name: "bridge_propose_batch_update_info_failure_total",
-			Help: "Total number of propose batch update info failure total.",
-		}),
-		totalL1CommitGas: promauto.With(reg).NewGauge(prometheus.GaugeOpts{
-			Name: "bridge_propose_batch_total_l1_commit_gas",
-			Help: "The total l1 commit gas",
-		}),
-		totalL1CommitCalldataSize: promauto.With(reg).NewGauge(prometheus.GaugeOpts{
-			Name: "bridge_propose_batch_total_l1_call_data_size",
-			Help: "The total l1 call data size",
-		}),
-		batchChunksNum: promauto.With(reg).NewGauge(prometheus.GaugeOpts{
-			Name: "bridge_propose_batch_chunks_number",
-			Help: "The number of chunks in the batch",
-		}),
-		batchFirstChunkTimeoutReached: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-			Name: "bridge_propose_batch_first_chunk_timeout_reached_total",
-			Help: "Total times of batch's first chunk timeout reached",
-		}),
-		batchChunksSuperposeNotEnoughTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-			Name: "bridge_propose_batch_chunks_superpose_not_enough_total",
-			Help: "Total number of batch chunk superpose not enough",
-		}),
-	}
-}
-
-// TryProposeBatch tries to propose a new batches.
-func (p *BatchProposer) TryProposeBatch() {
-	p.batchProposerCircleTotal.Inc()
-	dbChunks, err := p.proposeBatchChunks()
-	if err != nil {
-		p.proposeBatchFailureTotal.Inc()
-		log.Error("proposeBatchChunks failed", "err", err)
-		return
-	}
-	if err := p.updateBatchInfoInDB(dbChunks); err != nil {
-		p.proposeBatchUpdateInfoFailureTotal.Inc()
-		log.Error("update batch info in db failed", "err", err)
-	}
-}
-
-func (p *BatchProposer) updateBatchInfoInDB(dbChunks []*orm.Chunk) error {
-	p.proposeBatchUpdateInfoTotal.Inc()
-	numChunks := len(dbChunks)
-	if numChunks <= 0 {
-		return nil
-	}
-	chunks, err := p.dbChunksToBridgeChunks(dbChunks)
-	if err != nil {
-		return err
-	}
-
-	startChunkIndex := dbChunks[0].Index
-	startChunkHash := dbChunks[0].Hash
-	endChunkIndex := dbChunks[numChunks-1].Index
-	endChunkHash := dbChunks[numChunks-1].Hash
-	err = p.db.Transaction(func(dbTX *gorm.DB) error {
-		batch, dbErr := p.batchOrm.InsertBatch(p.ctx, startChunkIndex, endChunkIndex, startChunkHash, endChunkHash, chunks, dbTX)
-		if dbErr != nil {
-			log.Warn("BatchProposer.updateBatchInfoInDB insert batch failure", "error", "start chunk index", startChunkIndex, "end chunk index", endChunkIndex, dbErr)
-			return dbErr
-		}
-		dbErr = p.chunkOrm.UpdateBatchHashInRange(p.ctx, startChunkIndex, endChunkIndex, batch.Hash, dbTX)
-		if dbErr != nil {
-			log.Warn("BatchProposer.UpdateBatchHashInRange update the chunk's batch hash failure", "hash", batch.Hash, "error", dbErr)
-			return dbErr
-		}
-		return nil
-	})
-	return err
-}
-
-func (p *BatchProposer) proposeBatchChunks() ([]*orm.Chunk, error) {
-	dbChunks, err := p.chunkOrm.GetUnbatchedChunks(p.ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	if len(dbChunks) == 0 {
-		return nil, nil
-	}
-
-	firstChunk := dbChunks[0]
-	totalL1CommitCalldataSize := firstChunk.TotalL1CommitCalldataSize
-	totalL1CommitGas := firstChunk.TotalL1CommitGas
-	totalChunks := uint64(1)
-	totalL1MessagePopped := firstChunk.TotalL1MessagesPoppedBefore + uint64(firstChunk.TotalL1MessagesPoppedInChunk)
-
-	parentBatch, err := p.batchOrm.GetLatestBatch(p.ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	getKeccakGas := func(size uint64) uint64 {
-		return 30 + 6*((size+31)/32) // 30 + 6 * ceil(size / 32)
-	}
-
-	// Add extra gas costs
-	totalL1CommitGas += 4 * 2100                                       // 4 one-time cold sload for commitBatch
-	totalL1CommitGas += 20000                                          // 1 time sstore
-	totalL1CommitGas += 16                                             // version in calldata
-	totalL1CommitGas += 16 * (32 * (totalL1MessagePopped + 255) / 256) // _skippedL1MessageBitmap in calldata
-
-	// adjusting gas:
-	// add 1 time cold sload (2100 gas) for L1MessageQueue
-	// add 1 time cold address access (2600 gas) for L1MessageQueue
-	// minus 1 time warm sload (100 gas) & 1 time warm address access (100 gas)
-	totalL1CommitGas += (2100 + 2600 - 100 - 100)
-	totalL1CommitGas += getKeccakGas(32 * totalChunks) // batch data hash
-	if parentBatch != nil {
-		totalL1CommitGas += getKeccakGas(uint64(len(parentBatch.BatchHeader))) // parent batch header hash
-		totalL1CommitGas += 16 * uint64(len(parentBatch.BatchHeader))          // parent batch header in calldata
-	}
-	// batch header size: 89 + 32 * ceil(l1MessagePopped / 256)
-	totalL1CommitGas += getKeccakGas(89 + 32*(totalL1MessagePopped+255)/256)
-
-	p.totalL1CommitGas.Set(float64(totalL1CommitGas))
-	// Check if the first chunk breaks hard limits.
-	// If so, it indicates there are bugs in chunk-proposer, manual fix is needed.
-	if p.gasCostIncreaseMultiplier*float64(totalL1CommitGas) > float64(p.maxL1CommitGasPerBatch) {
-		return nil, fmt.Errorf(
-			"the first chunk exceeds l1 commit gas limit; start block number: %v, end block number: %v, commit gas: %v, max commit gas limit: %v",
-			firstChunk.StartBlockNumber,
-			firstChunk.EndBlockNumber,
-			totalL1CommitGas,
-			p.maxL1CommitGasPerBatch,
-		)
-	}
-
-	p.totalL1CommitCalldataSize.Set(float64(totalL1CommitCalldataSize))
-	if totalL1CommitCalldataSize > p.maxL1CommitCalldataSizePerBatch {
-		return nil, fmt.Errorf(
-			"the first chunk exceeds l1 commit calldata size limit; start block number: %v, end block number %v, calldata size: %v, max calldata size limit: %v",
-			firstChunk.StartBlockNumber,
-			firstChunk.EndBlockNumber,
-			totalL1CommitCalldataSize,
-			p.maxL1CommitCalldataSizePerBatch,
-		)
-	}
-
-	for i, chunk := range dbChunks[1:] {
-		totalL1CommitCalldataSize += chunk.TotalL1CommitCalldataSize
-		totalL1CommitGas += chunk.TotalL1CommitGas
-		// adjust batch data hash gas cost
-		totalL1CommitGas -= getKeccakGas(32 * totalChunks)
-		totalChunks++
-		totalL1CommitGas += getKeccakGas(32 * totalChunks)
-		// adjust batch header hash gas cost
-		totalL1CommitGas -= getKeccakGas(89 + 32*(totalL1MessagePopped+255)/256)
-		totalL1CommitGas -= 16 * (32 * (totalL1MessagePopped + 255) / 256)
-		totalL1MessagePopped += uint64(chunk.TotalL1MessagesPoppedInChunk)
-		totalL1CommitGas += 16 * (32 * (totalL1MessagePopped + 255) / 256)
-		totalL1CommitGas += getKeccakGas(89 + 32*(totalL1MessagePopped+255)/256)
-		if totalChunks > p.maxChunkNumPerBatch ||
-			totalL1CommitCalldataSize > p.maxL1CommitCalldataSizePerBatch ||
-			p.gasCostIncreaseMultiplier*float64(totalL1CommitGas) > float64(p.maxL1CommitGasPerBatch) {
-			return dbChunks[:i+1], nil
-		}
-	}
-
-	p.batchChunksNum.Set(float64(len(dbChunks)))
-
-	var hasChunkTimeout bool
-	currentTimeSec := uint64(time.Now().Unix())
-	if dbChunks[0].StartBlockTime+p.batchTimeoutSec < currentTimeSec {
-		log.Warn("first block timeout",
-			"start block number", dbChunks[0].StartBlockNumber,
-			"first block timestamp", dbChunks[0].StartBlockTime,
-			"chunk outdated time threshold", currentTimeSec,
-		)
-		hasChunkTimeout = true
-		p.batchFirstChunkTimeoutReached.Inc()
-	}
-
-	if !hasChunkTimeout && uint64(len(dbChunks)) < p.minChunkNumPerBatch {
-		log.Warn("The chunk number of the batch is less than the minimum limit",
-			"chunk num", len(dbChunks), "minChunkNumPerBatch", p.minChunkNumPerBatch,
-		)
-		p.batchChunksSuperposeNotEnoughTotal.Inc()
-		return nil, nil
-	}
-	return dbChunks, nil
-}
-
-func (p *BatchProposer) dbChunksToBridgeChunks(dbChunks []*orm.Chunk) ([]*types.Chunk, error) {
-	chunks := make([]*types.Chunk, len(dbChunks))
-	for i, c := range dbChunks {
-		wrappedBlocks, err := p.l2BlockOrm.GetL2BlocksInRange(p.ctx, c.StartBlockNumber, c.EndBlockNumber)
-		if err != nil {
-			log.Error("Failed to fetch wrapped blocks",
-				"start number", c.StartBlockNumber, "end number", c.EndBlockNumber, "error", err)
-			return nil, err
-		}
-		chunks[i] = &types.Chunk{
-			Blocks: wrappedBlocks,
-		}
-	}
-	return chunks, nil
-}

bridge/internal/controller/watcher/batch_proposer_test.go

@@ -1,72 +0,0 @@
-package watcher
-
-import (
-	"context"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-
-	"scroll-tech/common/database"
-	"scroll-tech/common/types"
-
-	"scroll-tech/bridge/internal/config"
-	"scroll-tech/bridge/internal/orm"
-)
-
-// TODO: Add unit tests that the limits are enforced correctly.
-func testBatchProposer(t *testing.T) {
-	db := setupDB(t)
-	defer database.CloseDB(db)
-
-	l2BlockOrm := orm.NewL2Block(db)
-	err := l2BlockOrm.InsertL2Blocks(context.Background(), []*types.WrappedBlock{wrappedBlock1, wrappedBlock2})
-	assert.NoError(t, err)
-
-	cp := NewChunkProposer(context.Background(), &config.ChunkProposerConfig{
-		MaxTxGasPerChunk:                1000000000,
-		MaxL2TxNumPerChunk:              10000,
-		MaxL1CommitGasPerChunk:          50000000000,
-		MaxL1CommitCalldataSizePerChunk: 1000000,
-		MinL1CommitCalldataSizePerChunk: 0,
-		MaxRowConsumptionPerChunk:       1048319,
-		ChunkTimeoutSec:                 300,
-	}, db, nil)
-	cp.TryProposeChunk()
-
-	bp := NewBatchProposer(context.Background(), &config.BatchProposerConfig{
-		MaxChunkNumPerBatch:             10,
-		MaxL1CommitGasPerBatch:          50000000000,
-		MaxL1CommitCalldataSizePerBatch: 1000000,
-		MinChunkNumPerBatch:             1,
-		BatchTimeoutSec:                 300,
-	}, db, nil)
-	bp.TryProposeBatch()
-
-	chunkOrm := orm.NewChunk(db)
-	chunks, err := chunkOrm.GetUnbatchedChunks(context.Background())
-	assert.NoError(t, err)
-	assert.Empty(t, chunks)
-
-	batchOrm := orm.NewBatch(db)
-	// get all batches.
-	batches, err := batchOrm.GetBatches(context.Background(), map[string]interface{}{}, []string{}, 0)
-	assert.NoError(t, err)
-	assert.Len(t, batches, 1)
-	assert.Equal(t, uint64(0), batches[0].StartChunkIndex)
-	assert.Equal(t, uint64(0), batches[0].EndChunkIndex)
-	assert.Equal(t, types.RollupPending, types.RollupStatus(batches[0].RollupStatus))
-	assert.Equal(t, types.ProvingTaskUnassigned, types.ProvingStatus(batches[0].ProvingStatus))
-
-	dbChunks, err := chunkOrm.GetChunksInRange(context.Background(), 0, 0)
-	assert.NoError(t, err)
-	assert.Len(t, batches, 1)
-	assert.Equal(t, batches[0].Hash, dbChunks[0].BatchHash)
-	assert.Equal(t, types.ProvingTaskUnassigned, types.ProvingStatus(dbChunks[0].ProvingStatus))
-
-	blockOrm := orm.NewL2Block(db)
-	blocks, err := blockOrm.GetL2Blocks(context.Background(), map[string]interface{}{}, []string{}, 0)
-	assert.NoError(t, err)
-	assert.Len(t, blocks, 2)
-	assert.Equal(t, dbChunks[0].Hash, blocks[0].ChunkHash)
-	assert.Equal(t, dbChunks[0].Hash, blocks[1].ChunkHash)
-}

bridge/internal/controller/watcher/chunk_proposer.go

@@ -1,302 +0,0 @@
-package watcher
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"time"
-
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-	gethTypes "github.com/scroll-tech/go-ethereum/core/types"
-	"github.com/scroll-tech/go-ethereum/log"
-	"gorm.io/gorm"
-
-	"scroll-tech/common/types"
-
-	"scroll-tech/bridge/internal/config"
-	"scroll-tech/bridge/internal/orm"
-)
-
-// chunkRowConsumption is map(sub-circuit name => sub-circuit row count)
-type chunkRowConsumption map[string]uint64
-
-// add accumulates row consumption per sub-circuit
-func (crc *chunkRowConsumption) add(rowConsumption *gethTypes.RowConsumption) error {
-	if rowConsumption == nil {
-		return errors.New("rowConsumption is <nil>")
-	}
-	for _, subCircuit := range *rowConsumption {
-		(*crc)[subCircuit.Name] += subCircuit.RowNumber
-	}
-	return nil
-}
-
-// max finds the maximum row consumption among all sub-circuits
-func (crc *chunkRowConsumption) max() uint64 {
-	var max uint64
-	for _, value := range *crc {
-		if value > max {
-			max = value
-		}
-	}
-	return max
-}
-
-// ChunkProposer proposes chunks based on available unchunked blocks.
-type ChunkProposer struct {
-	ctx context.Context
-	db  *gorm.DB
-
-	chunkOrm   *orm.Chunk
-	l2BlockOrm *orm.L2Block
-
-	maxTxGasPerChunk                uint64
-	maxL2TxNumPerChunk              uint64
-	maxL1CommitGasPerChunk          uint64
-	maxL1CommitCalldataSizePerChunk uint64
-	minL1CommitCalldataSizePerChunk uint64
-	maxRowConsumptionPerChunk       uint64
-	chunkTimeoutSec                 uint64
-	gasCostIncreaseMultiplier       float64
-
-	chunkProposerCircleTotal           prometheus.Counter
-	proposeChunkFailureTotal           prometheus.Counter
-	proposeChunkUpdateInfoTotal        prometheus.Counter
-	proposeChunkUpdateInfoFailureTotal prometheus.Counter
-	chunkL2TxNum                       prometheus.Gauge
-	chunkEstimateL1CommitGas           prometheus.Gauge
-	totalL1CommitCalldataSize          prometheus.Gauge
-	totalTxGasUsed                     prometheus.Gauge
-	maxTxConsumption                   prometheus.Gauge
-	chunkBlocksNum                     prometheus.Gauge
-	chunkBlockTimeoutReached           prometheus.Counter
-	chunkBlocksSuperposeNotEnoughTotal prometheus.Counter
-}
-
-// NewChunkProposer creates a new ChunkProposer instance.
-func NewChunkProposer(ctx context.Context, cfg *config.ChunkProposerConfig, db *gorm.DB, reg prometheus.Registerer) *ChunkProposer {
-	return &ChunkProposer{
-		ctx:                             ctx,
-		db:                              db,
-		chunkOrm:                        orm.NewChunk(db),
-		l2BlockOrm:                      orm.NewL2Block(db),
-		maxTxGasPerChunk:                cfg.MaxTxGasPerChunk,
-		maxL2TxNumPerChunk:              cfg.MaxL2TxNumPerChunk,
-		maxL1CommitGasPerChunk:          cfg.MaxL1CommitGasPerChunk,
-		maxL1CommitCalldataSizePerChunk: cfg.MaxL1CommitCalldataSizePerChunk,
-		minL1CommitCalldataSizePerChunk: cfg.MinL1CommitCalldataSizePerChunk,
-		maxRowConsumptionPerChunk:       cfg.MaxRowConsumptionPerChunk,
-		chunkTimeoutSec:                 cfg.ChunkTimeoutSec,
-		gasCostIncreaseMultiplier:       cfg.GasCostIncreaseMultiplier,
-
-		chunkProposerCircleTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-			Name: "bridge_propose_chunk_circle_total",
-			Help: "Total number of propose chunk total.",
-		}),
-		proposeChunkFailureTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-			Name: "bridge_propose_chunk_failure_circle_total",
-			Help: "Total number of propose chunk failure total.",
-		}),
-		proposeChunkUpdateInfoTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-			Name: "bridge_propose_chunk_update_info_total",
-			Help: "Total number of propose chunk update info total.",
-		}),
-		proposeChunkUpdateInfoFailureTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-			Name: "bridge_propose_chunk_update_info_failure_total",
-			Help: "Total number of propose chunk update info failure total.",
-		}),
-		chunkL2TxNum: promauto.With(reg).NewGauge(prometheus.GaugeOpts{
-			Name: "bridge_propose_chunk_l2_tx_num",
-			Help: "The chunk l2 tx num",
-		}),
-		chunkEstimateL1CommitGas: promauto.With(reg).NewGauge(prometheus.GaugeOpts{
-			Name: "bridge_propose_chunk_estimate_l1_commit_gas",
-			Help: "The chunk estimate l1 commit gas",
-		}),
-		totalL1CommitCalldataSize: promauto.With(reg).NewGauge(prometheus.GaugeOpts{
-			Name: "bridge_propose_chunk_total_l1_commit_call_data_size",
-			Help: "The total l1 commit call data size",
-		}),
-		totalTxGasUsed: promauto.With(reg).NewGauge(prometheus.GaugeOpts{
-			Name: "bridge_propose_chunk_total_tx_gas_used",
-			Help: "The total tx gas used",
-		}),
-		maxTxConsumption: promauto.With(reg).NewGauge(prometheus.GaugeOpts{
-			Name: "bridge_propose_chunk_max_tx_consumption",
-			Help: "The max tx consumption",
-		}),
-		chunkBlocksNum: promauto.With(reg).NewGauge(prometheus.GaugeOpts{
-			Name: "bridge_propose_chunk_chunk_block_number",
-			Help: "The number of blocks in the chunk",
-		}),
-		chunkBlockTimeoutReached: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-			Name: "bridge_propose_chunk_first_block_timeout_reached_total",
-			Help: "Total times of chunk's first block timeout reached",
-		}),
-		chunkBlocksSuperposeNotEnoughTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-			Name: "bridge_propose_chunk_blocks_superpose_not_enough_total",
-			Help: "Total number of chunk block superpose not enough",
-		}),
-	}
-}
-
-// TryProposeChunk tries to propose a new chunk.
-func (p *ChunkProposer) TryProposeChunk() {
-	p.chunkProposerCircleTotal.Inc()
-	proposedChunk, err := p.proposeChunk()
-	if err != nil {
-		p.proposeChunkFailureTotal.Inc()
-		log.Error("propose new chunk failed", "err", err)
-		return
-	}
-
-	if err := p.updateChunkInfoInDB(proposedChunk); err != nil {
-		p.proposeChunkUpdateInfoFailureTotal.Inc()
-		log.Error("update chunk info in orm failed", "err", err)
-	}
-}
-
-func (p *ChunkProposer) updateChunkInfoInDB(chunk *types.Chunk) error {
-	if chunk == nil {
-		return nil
-	}
-
-	p.proposeChunkUpdateInfoTotal.Inc()
-	err := p.db.Transaction(func(dbTX *gorm.DB) error {
-		dbChunk, err := p.chunkOrm.InsertChunk(p.ctx, chunk, dbTX)
-		if err != nil {
-			log.Warn("ChunkProposer.InsertChunk failed", "chunk hash", chunk.Hash)
-			return err
-		}
-		if err := p.l2BlockOrm.UpdateChunkHashInRange(p.ctx, dbChunk.StartBlockNumber, dbChunk.EndBlockNumber, dbChunk.Hash, dbTX); err != nil {
-			log.Error("failed to update chunk_hash for l2_blocks", "chunk hash", chunk.Hash, "start block", 0, "end block", 0, "err", err)
-			return err
-		}
-		return nil
-	})
-	return err
-}
-
-func (p *ChunkProposer) proposeChunk() (*types.Chunk, error) {
-	blocks, err := p.l2BlockOrm.GetUnchunkedBlocks(p.ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	if len(blocks) == 0 {
-		return nil, nil
-	}
-
-	chunk := &types.Chunk{Blocks: blocks[:1]}
-	firstBlock := chunk.Blocks[0]
-	totalTxGasUsed := firstBlock.Header.GasUsed
-	totalL2TxNum := firstBlock.L2TxsNum()
-	totalL1CommitCalldataSize := firstBlock.EstimateL1CommitCalldataSize()
-	crc := chunkRowConsumption{}
-	totalL1CommitGas := chunk.EstimateL1CommitGas()
-
-	if err := crc.add(firstBlock.RowConsumption); err != nil {
-		return nil, fmt.Errorf("chunk-proposer failed to update chunk row consumption: %v", err)
-	}
-
-	p.chunkL2TxNum.Set(float64(totalL2TxNum))
-	// Check if the first block breaks hard limits.
-	// If so, it indicates there are bugs in sequencer, manual fix is needed.
-	if totalL2TxNum > p.maxL2TxNumPerChunk {
-		return nil, fmt.Errorf(
-			"the first block exceeds l2 tx number limit; block number: %v, number of transactions: %v, max transaction number limit: %v",
-			firstBlock.Header.Number,
-			totalL2TxNum,
-			p.maxL2TxNumPerChunk,
-		)
-	}
-
-	p.chunkEstimateL1CommitGas.Set(float64(totalL1CommitGas))
-	if p.gasCostIncreaseMultiplier*float64(totalL1CommitGas) > float64(p.maxL1CommitGasPerChunk) {
-		return nil, fmt.Errorf(
-			"the first block exceeds l1 commit gas limit; block number: %v, commit gas: %v, max commit gas limit: %v",
-			firstBlock.Header.Number,
-			totalL1CommitGas,
-			p.maxL1CommitGasPerChunk,
-		)
-	}
-
-	p.totalL1CommitCalldataSize.Set(float64(totalL1CommitCalldataSize))
-	if totalL1CommitCalldataSize > p.maxL1CommitCalldataSizePerChunk {
-		return nil, fmt.Errorf(
-			"the first block exceeds l1 commit calldata size limit; block number: %v, calldata size: %v, max calldata size limit: %v",
-			firstBlock.Header.Number,
-			totalL1CommitCalldataSize,
-			p.maxL1CommitCalldataSizePerChunk,
-		)
-	}
-
-	p.totalTxGasUsed.Set(float64(totalTxGasUsed))
-	// Check if the first block breaks any soft limits.
-	if totalTxGasUsed > p.maxTxGasPerChunk {
-		log.Warn(
-			"The first block in chunk exceeds l2 tx gas limit",
-			"block number", firstBlock.Header.Number,
-			"gas used", totalTxGasUsed,
-			"max gas limit", p.maxTxGasPerChunk,
-		)
-	}
-
-	max := crc.max()
-	p.maxTxConsumption.Set(float64(max))
-	if max > p.maxRowConsumptionPerChunk {
-		return nil, fmt.Errorf(
-			"the first block exceeds row consumption limit; block number: %v, row consumption: %v, max: %v, limit: %v",
-			firstBlock.Header.Number,
-			crc,
-			max,
-			p.maxRowConsumptionPerChunk,
-		)
-	}
-
-	for _, block := range blocks[1:] {
-		chunk.Blocks = append(chunk.Blocks, block)
-		totalTxGasUsed += block.Header.GasUsed
-		totalL2TxNum += block.L2TxsNum()
-		totalL1CommitCalldataSize += block.EstimateL1CommitCalldataSize()
-		totalL1CommitGas = chunk.EstimateL1CommitGas()
-
-		if err := crc.add(block.RowConsumption); err != nil {
-			return nil, fmt.Errorf("chunk-proposer failed to update chunk row consumption: %v", err)
-		}
-
-		if totalTxGasUsed > p.maxTxGasPerChunk ||
-			totalL2TxNum > p.maxL2TxNumPerChunk ||
-			totalL1CommitCalldataSize > p.maxL1CommitCalldataSizePerChunk ||
-			p.gasCostIncreaseMultiplier*float64(totalL1CommitGas) > float64(p.maxL1CommitGasPerChunk) ||
-			crc.max() > p.maxRowConsumptionPerChunk {
-			chunk.Blocks = chunk.Blocks[:len(chunk.Blocks)-1] // remove the last block from chunk
-			break
-		}
-	}
-
-	p.chunkBlocksNum.Set(float64(len(chunk.Blocks)))
-
-	var hasBlockTimeout bool
-	currentTimeSec := uint64(time.Now().Unix())
-	if blocks[0].Header.Time+p.chunkTimeoutSec < currentTimeSec {
-		log.Warn("first block timeout",
-			"block number", blocks[0].Header.Number,
-			"block timestamp", blocks[0].Header.Time,
-			"block outdated time threshold", currentTimeSec,
-		)
-		p.chunkBlockTimeoutReached.Inc()
-		hasBlockTimeout = true
-	}
-
-	if !hasBlockTimeout && totalL1CommitCalldataSize < p.minL1CommitCalldataSizePerChunk {
-		log.Warn("The calldata size of the chunk is less than the minimum limit",
-			"totalL1CommitCalldataSize", totalL1CommitCalldataSize,
-			"minL1CommitCalldataSizePerChunk", p.minL1CommitCalldataSizePerChunk,
-		)
-		p.chunkBlocksSuperposeNotEnoughTotal.Inc()
-		return nil, nil
-	}
-	return chunk, nil
-}

bridge/internal/controller/watcher/chunk_proposer_test.go

@@ -1,72 +0,0 @@
-package watcher
-
-import (
-	"context"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-
-	"scroll-tech/common/database"
-	"scroll-tech/common/types"
-
-	"scroll-tech/bridge/internal/config"
-	"scroll-tech/bridge/internal/orm"
-)
-
-// TODO: Add unit tests that the limits are enforced correctly.
-func testChunkProposer(t *testing.T) {
-	db := setupDB(t)
-	defer database.CloseDB(db)
-
-	l2BlockOrm := orm.NewL2Block(db)
-	err := l2BlockOrm.InsertL2Blocks(context.Background(), []*types.WrappedBlock{wrappedBlock1, wrappedBlock2})
-	assert.NoError(t, err)
-
-	cp := NewChunkProposer(context.Background(), &config.ChunkProposerConfig{
-		MaxTxGasPerChunk:                1000000000,
-		MaxL2TxNumPerChunk:              10000,
-		MaxL1CommitGasPerChunk:          50000000000,
-		MaxL1CommitCalldataSizePerChunk: 1000000,
-		MinL1CommitCalldataSizePerChunk: 0,
-		MaxRowConsumptionPerChunk:       1048319,
-		ChunkTimeoutSec:                 300,
-	}, db, nil)
-	cp.TryProposeChunk()
-
-	expectedChunk := &types.Chunk{
-		Blocks: []*types.WrappedBlock{wrappedBlock1, wrappedBlock2},
-	}
-	expectedHash, err := expectedChunk.Hash(0)
-	assert.NoError(t, err)
-
-	chunkOrm := orm.NewChunk(db)
-	chunks, err := chunkOrm.GetUnbatchedChunks(context.Background())
-	assert.NoError(t, err)
-	assert.Len(t, chunks, 1)
-	assert.Equal(t, expectedHash.Hex(), chunks[0].Hash)
-}
-
-func testChunkProposerRowConsumption(t *testing.T) {
-	db := setupDB(t)
-	defer database.CloseDB(db)
-
-	l2BlockOrm := orm.NewL2Block(db)
-	err := l2BlockOrm.InsertL2Blocks(context.Background(), []*types.WrappedBlock{wrappedBlock1, wrappedBlock2})
-	assert.NoError(t, err)
-
-	cp := NewChunkProposer(context.Background(), &config.ChunkProposerConfig{
-		MaxTxGasPerChunk:                1000000000,
-		MaxL2TxNumPerChunk:              10000,
-		MaxL1CommitGasPerChunk:          50000000000,
-		MaxL1CommitCalldataSizePerChunk: 1000000,
-		MinL1CommitCalldataSizePerChunk: 0,
-		MaxRowConsumptionPerChunk:       0, // !
-		ChunkTimeoutSec:                 300,
-	}, db, nil)
-	cp.TryProposeChunk()
-
-	chunkOrm := orm.NewChunk(db)
-	chunks, err := chunkOrm.GetUnbatchedChunks(context.Background())
-	assert.NoError(t, err)
-	assert.Len(t, chunks, 0)
-}

bridge/tests/l1_message_relay_test.go

@@ -1,83 +0,0 @@
-package tests
-
-import (
-	"context"
-	"math/big"
-	"testing"
-
-	"github.com/scroll-tech/go-ethereum/accounts/abi/bind"
-	"github.com/scroll-tech/go-ethereum/common"
-	geth_types "github.com/scroll-tech/go-ethereum/core/types"
-	"github.com/scroll-tech/go-ethereum/rpc"
-	"github.com/stretchr/testify/assert"
-
-	"scroll-tech/common/database"
-	"scroll-tech/common/types"
-
-	"scroll-tech/bridge/internal/controller/relayer"
-	"scroll-tech/bridge/internal/controller/watcher"
-	"scroll-tech/bridge/internal/orm"
-)
-
-func testRelayL1MessageSucceed(t *testing.T) {
-	db := setupDB(t)
-	defer database.CloseDB(db)
-
-	prepareContracts(t)
-
-	l1Cfg := bridgeApp.Config.L1Config
-	l2Cfg := bridgeApp.Config.L2Config
-
-	// Create L1Relayer
-	l1Relayer, err := relayer.NewLayer1Relayer(context.Background(), db, l1Cfg.RelayerConfig, nil)
-	assert.NoError(t, err)
-	// Create L1Watcher
-	confirmations := rpc.LatestBlockNumber
-	l1Watcher := watcher.NewL1WatcherClient(context.Background(), l1Client, 0, confirmations, l1Cfg.L1MessengerAddress,
-		l1Cfg.L1MessageQueueAddress, l1Cfg.ScrollChainContractAddress, db, nil)
-
-	// Create L2Watcher
-	l2Watcher := watcher.NewL2WatcherClient(context.Background(), l2Client, confirmations, l2Cfg.L2MessengerAddress,
-		l2Cfg.L2MessageQueueAddress, l2Cfg.WithdrawTrieRootSlot, db, nil)
-
-	// send message through l1 messenger contract
-	nonce, err := l1MessengerInstance.MessageNonce(&bind.CallOpts{})
-	assert.NoError(t, err)
-	sendTx, err := l1MessengerInstance.SendMessage(l1Auth, l2Auth.From, big.NewInt(0), common.Hex2Bytes("00112233"), big.NewInt(0))
-	assert.NoError(t, err)
-	sendReceipt, err := bind.WaitMined(context.Background(), l1Client, sendTx)
-	assert.NoError(t, err)
-	if sendReceipt.Status != geth_types.ReceiptStatusSuccessful || err != nil {
-		t.Fatalf("Call failed")
-	}
-
-	// l1 watch process events
-	l1Watcher.FetchContractEvent()
-
-	l1MessageOrm := orm.NewL1Message(db)
-	// check db status
-	msg, err := l1MessageOrm.GetL1MessageByQueueIndex(nonce.Uint64())
-	assert.NoError(t, err)
-	assert.Equal(t, types.MsgStatus(msg.Status), types.MsgPending)
-	assert.Equal(t, msg.Target, l2Auth.From.String())
-
-	// process l1 messages
-	l1Relayer.ProcessSavedEvents()
-
-	l1Message, err := l1MessageOrm.GetL1MessageByQueueIndex(nonce.Uint64())
-	assert.NoError(t, err)
-	assert.NotEmpty(t, l1Message.Layer2Hash)
-	assert.Equal(t, types.MsgStatus(l1Message.Status), types.MsgSubmitted)
-
-	relayTx, _, err := l2Client.TransactionByHash(context.Background(), common.HexToHash(l1Message.Layer2Hash))
-	assert.NoError(t, err)
-	relayTxReceipt, err := bind.WaitMined(context.Background(), l2Client, relayTx)
-	assert.NoError(t, err)
-	assert.Equal(t, len(relayTxReceipt.Logs), 1)
-
-	// fetch message relayed events
-	l2Watcher.FetchContractEvent()
-	msg, err = l1MessageOrm.GetL1MessageByQueueIndex(nonce.Uint64())
-	assert.NoError(t, err)
-	assert.Equal(t, types.MsgStatus(msg.Status), types.MsgConfirmed)
-}

build/dockerfiles/coordinator.Dockerfile

@@ -20,11 +20,10 @@ RUN find ./ | grep libzktrie.so | xargs -I{} cp {} /app/target/release/
 FROM scrolltech/go-rust-builder:go-1.19-rust-nightly-2022-12-10 as base
 WORKDIR /src
 COPY go.work* ./
-COPY ./bridge/go.* ./bridge/
+COPY ./rollup/go.* ./rollup/
 COPY ./common/go.* ./common/
 COPY ./coordinator/go.* ./coordinator/
 COPY ./database/go.* ./database/
-COPY ./prover-stats-api/go.* ./prover-stats-api/
 COPY ./prover/go.* ./prover/
 COPY ./tests/integration-test/go.* ./tests/integration-test/
 COPY ./bridge-history-api/go.* ./bridge-history-api/

build/dockerfiles/db_cli.Dockerfile

@@ -3,11 +3,10 @@ FROM scrolltech/go-alpine-builder:1.19 as base
 
 WORKDIR /src
 COPY go.work* ./
-COPY ./bridge/go.* ./bridge/
+COPY ./rollup/go.* ./rollup/
 COPY ./common/go.* ./common/
 COPY ./coordinator/go.* ./coordinator/
 COPY ./database/go.* ./database/
-COPY ./prover-stats-api/go.* ./prover-stats-api/
 COPY ./prover/go.* ./prover/
 COPY ./tests/integration-test/go.* ./tests/integration-test/
 COPY ./bridge-history-api/go.* ./bridge-history-api/

build/dockerfiles/event_watcher.Dockerfile

@@ -3,11 +3,10 @@ FROM scrolltech/go-alpine-builder:1.19 as base
 
 WORKDIR /src
 COPY go.work* ./
-COPY ./bridge/go.* ./bridge/
+COPY ./rollup/go.* ./rollup/
 COPY ./common/go.* ./common/
 COPY ./coordinator/go.* ./coordinator/
 COPY ./database/go.* ./database/
-COPY ./prover-stats-api/go.* ./prover-stats-api/
 COPY ./prover/go.* ./prover/
 COPY ./tests/integration-test/go.* ./tests/integration-test/
 COPY ./bridge-history-api/go.* ./bridge-history-api/
@@ -18,7 +17,7 @@ FROM base as builder
 
 RUN --mount=target=. \
     --mount=type=cache,target=/root/.cache/go-build \
-    cd /src/bridge/cmd/event_watcher/ && go build -v -p 4 -o /bin/event_watcher
+    cd /src/rollup/cmd/event_watcher/ && go build -v -p 4 -o /bin/event_watcher
 
 # Pull event_watcher into a second stage deploy alpine container
 FROM alpine:latest

build/dockerfiles/gas_oracle.Dockerfile

@@ -3,11 +3,10 @@ FROM scrolltech/go-alpine-builder:1.19 as base
 
 WORKDIR /src
 COPY go.work* ./
-COPY ./bridge/go.* ./bridge/
+COPY ./rollup/go.* ./rollup/
 COPY ./common/go.* ./common/
 COPY ./coordinator/go.* ./coordinator/
 COPY ./database/go.* ./database/
-COPY ./prover-stats-api/go.* ./prover-stats-api/
 COPY ./prover/go.* ./prover/
 COPY ./tests/integration-test/go.* ./tests/integration-test/
 COPY ./bridge-history-api/go.* ./bridge-history-api/
@@ -18,7 +17,7 @@ FROM base as builder
 
 RUN --mount=target=. \
     --mount=type=cache,target=/root/.cache/go-build \
-    cd /src/bridge/cmd/gas_oracle/ && go build -v -p 4 -o /bin/gas_oracle
+    cd /src/rollup/cmd/gas_oracle/ && go build -v -p 4 -o /bin/gas_oracle
 
 # Pull gas_oracle into a second stage deploy alpine container
 FROM alpine:latest

build/dockerfiles/msg_relayer.Dockerfile

@@ -1,28 +0,0 @@
-# Download Go dependencies
-FROM scrolltech/go-alpine-builder:1.19 as base
-
-WORKDIR /src
-COPY go.work* ./
-COPY ./bridge/go.* ./bridge/
-COPY ./common/go.* ./common/
-COPY ./coordinator/go.* ./coordinator/
-COPY ./database/go.* ./database/
-COPY ./prover-stats-api/go.* ./prover-stats-api/
-COPY ./prover/go.* ./prover/
-COPY ./tests/integration-test/go.* ./tests/integration-test/
-COPY ./bridge-history-api/go.* ./bridge-history-api/
-RUN go mod download -x
-
-# Build msg_relayer
-FROM base as builder
-
-RUN --mount=target=. \
-    --mount=type=cache,target=/root/.cache/go-build \
-    cd /src/bridge/cmd/msg_relayer/ && go build -v -p 4 -o /bin/msg_relayer
-
-# Pull msg_relayer into a second stage deploy alpine container
-FROM alpine:latest
-
-COPY --from=builder /bin/msg_relayer /bin/
-
-ENTRYPOINT ["msg_relayer"]

build/dockerfiles/msg_relayer.Dockerfile.dockerignore

@@ -1,5 +0,0 @@
-assets/
-docs/
-l2geth/
-rpc-gateway/
-*target/*

build/dockerfiles/prover-stats-api.Dockerfile

@@ -1,31 +0,0 @@
-# Download Go dependencies
-FROM scrolltech/go-alpine-builder:1.19 as base
-
-WORKDIR /src
-COPY go.work* ./
-COPY ./bridge/go.* ./bridge/
-COPY ./common/go.* ./common/
-COPY ./coordinator/go.* ./coordinator/
-COPY ./database/go.* ./database/
-COPY ./prover-stats-api/go.* ./prover-stats-api/
-COPY ./prover/go.* ./prover/
-COPY ./tests/integration-test/go.* ./tests/integration-test/
-COPY ./bridge-history-api/go.* ./bridge-history-api/
-# Support mainland environment.
-#ENV GOPROXY="https://goproxy.cn,direct"
-RUN go mod download -x
-
-
-# Build prover-stats-api
-FROM base as builder
-
-RUN --mount=target=. \
-    --mount=type=cache,target=/root/.cache/go-build \
-    cd /src/prover-stats-api/cmd/ && go build -v -p 4 -o /bin/prover-stats-api
-
-# Pull prover-stats-api into a second stage deploy alpine container \
-FROM alpine:latest
-
-COPY --from=builder /bin/prover-stats-api /bin/
-
-ENTRYPOINT ["prover-stats-api"]

build/dockerfiles/rollup_relayer.Dockerfile

@@ -3,11 +3,10 @@ FROM scrolltech/go-alpine-builder:1.19 as base
 
 WORKDIR /src
 COPY go.work* ./
-COPY ./bridge/go.* ./bridge/
+COPY ./rollup/go.* ./rollup/
 COPY ./common/go.* ./common/
 COPY ./coordinator/go.* ./coordinator/
 COPY ./database/go.* ./database/
-COPY ./prover-stats-api/go.* ./prover-stats-api/
 COPY ./prover/go.* ./prover/
 COPY ./tests/integration-test/go.* ./tests/integration-test/
 COPY ./bridge-history-api/go.* ./bridge-history-api/
@@ -18,7 +17,7 @@ FROM base as builder
 
 RUN --mount=target=. \
     --mount=type=cache,target=/root/.cache/go-build \
-    cd /src/bridge/cmd/rollup_relayer/ && go build -v -p 4 -o /bin/rollup_relayer
+    cd /src/rollup/cmd/rollup_relayer/ && go build -v -p 4 -o /bin/rollup_relayer
 
 # Pull rollup_relayer into a second stage deploy alpine container
 FROM alpine:latest

build/run_tests.sh

@@ -3,7 +3,7 @@ set -uex
 
 profile_name=$1
 
-exclude_dirs=("scroll-tech/bridge/cmd" "scroll-tech/bridge/tests" "scroll-tech/bridge/mock_bridge" "scroll-tech/coordinator/cmd" "scroll-tech/coordinator/internal/logic/verifier")
+exclude_dirs=("scroll-tech/rollup/cmd" "scroll-tech/rollup/tests" "scroll-tech/rollup/mock_bridge" "scroll-tech/coordinator/cmd" "scroll-tech/coordinator/internal/logic/verifier")
 
 all_packages=$(go list ./... | grep -v "^scroll-tech/${profile_name}$")
 coverpkg="scroll-tech/${profile_name}"

codecov.yml

@@ -6,7 +6,7 @@ flag_management:
   default_rules:
     carryforward: true
   individual_flags:
-    - name: bridge
+    - name: rollup
       statuses:
         - type: project
           target: auto

common/cmd/cmd.go

@@ -5,8 +5,9 @@ import (
 	"os"
 	"os/exec"
 	"strings"
-	"sync"
+	"sync/atomic"
 
+	"github.com/docker/docker/pkg/reexec"
 	cmap "github.com/orcaman/concurrent-map"
 )
 
@@ -26,8 +27,9 @@ type Cmd struct {
 	name string
 	args []string
 
-	mu  sync.Mutex
-	cmd *exec.Cmd
+	isRunning uint64
+	cmd       *exec.Cmd
+	app       *exec.Cmd
 
 	checkFuncs cmap.ConcurrentMap //map[string]checkFunc
 
@@ -38,13 +40,23 @@ type Cmd struct {
 }
 
 // NewCmd create Cmd instance.
-func NewCmd(name string, args ...string) *Cmd {
-	return &Cmd{
+func NewCmd(name string, params ...string) *Cmd {
+	cmd := &Cmd{
 		checkFuncs: cmap.New(),
 		name:       name,
-		args:       args,
+		args:       params,
 		ErrChan:    make(chan error, 10),
+		cmd:        exec.Command(name, params...),
+		app: &exec.Cmd{
+			Path: reexec.Self(),
+			Args: append([]string{name}, params...),
+		},
 	}
+	cmd.cmd.Stdout = cmd
+	cmd.cmd.Stderr = cmd
+	cmd.app.Stdout = cmd
+	cmd.app.Stderr = cmd
+	return cmd
 }
 
 // RegistFunc register check func
@@ -58,15 +70,14 @@ func (c *Cmd) UnRegistFunc(key string) {
 }
 
 func (c *Cmd) runCmd() {
-	cmd := exec.Command(c.args[0], c.args[1:]...) //nolint:gosec
-	cmd.Stdout = c
-	cmd.Stderr = c
-	c.ErrChan <- cmd.Run()
+	fmt.Println("cmd:", append([]string{c.name}, c.args...))
+	if atomic.CompareAndSwapUint64(&c.isRunning, 0, 1) {
+		c.ErrChan <- c.cmd.Run()
+	}
 }
 
 // RunCmd parallel running when parallel is true.
 func (c *Cmd) RunCmd(parallel bool) {
-	fmt.Println("cmd:", c.args)
 	if parallel {
 		go c.runCmd()
 	} else {

common/cmd/cmd_app.go

@@ -3,41 +3,45 @@ package cmd
 import (
 	"fmt"
 	"os"
-	"os/exec"
 	"strings"
+	"sync/atomic"
+	"syscall"
 	"testing"
 	"time"
 
-	"github.com/docker/docker/pkg/reexec"
 	"github.com/stretchr/testify/assert"
 )
 
+// IsRunning 1 started, 0 not started.
+func (c *Cmd) IsRunning() bool {
+	return atomic.LoadUint64(&c.isRunning) == 1
+}
+
+func (c *Cmd) runApp() {
+	fmt.Println("cmd:", append([]string{c.name}, c.args...))
+	if atomic.CompareAndSwapUint64(&c.isRunning, 0, 1) {
+		c.ErrChan <- c.app.Run()
+	}
+}
+
 // RunApp exec's the current binary using name as argv[0] which will trigger the
 // reexec init function for that name (e.g. "geth-test" in cmd/geth/run_test.go)
 func (c *Cmd) RunApp(waitResult func() bool) {
-	fmt.Println("cmd: ", append([]string{c.name}, c.args...))
-	cmd := &exec.Cmd{
-		Path:   reexec.Self(),
-		Args:   append([]string{c.name}, c.args...),
-		Stderr: c,
-		Stdout: c,
-	}
 	if waitResult != nil {
 		go func() {
-			_ = cmd.Run()
+			c.runApp()
 		}()
 		waitResult()
 	} else {
-		_ = cmd.Run()
+		c.runApp()
 	}
-
-	c.mu.Lock()
-	c.cmd = cmd
-	c.mu.Unlock()
 }
 
 // WaitExit wait util process exit.
 func (c *Cmd) WaitExit() {
+	if atomic.LoadUint64(&c.isRunning) == 0 {
+		return
+	}
 	// Wait all the check functions are finished, interrupt loop when appear error.
 	var err error
 	for err == nil && !c.checkFuncs.IsEmpty() {
@@ -52,20 +56,18 @@ func (c *Cmd) WaitExit() {
 	}
 
 	// Send interrupt signal.
-	c.mu.Lock()
-	_ = c.cmd.Process.Signal(os.Interrupt)
-	// should use `_ = c.cmd.Process.Wait()` here, but we have some bugs in coordinator's graceful exit,
+	_ = c.app.Process.Signal(os.Interrupt)
+	// should use `_ = c.app.Process.Wait()` here, but we have some bugs in coordinator's graceful exit,
 	// so we use `Kill` as a temp workaround. And since `WaitExit` is only used in integration tests, so
 	// it won't really affect our functionalities.
-	_ = c.cmd.Process.Kill()
-	c.mu.Unlock()
+	if err = c.app.Process.Signal(syscall.SIGTERM); err != nil {
+		_ = c.app.Process.Kill()
+	}
 }
 
 // Interrupt send interrupt signal.
 func (c *Cmd) Interrupt() {
-	c.mu.Lock()
-	c.ErrChan <- c.cmd.Process.Signal(os.Interrupt)
-	c.mu.Unlock()
+	c.ErrChan <- c.app.Process.Signal(os.Interrupt)
 }
 
 // WaitResult return true when get the keyword during timeout.

common/cmd/cmd_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func TestCmd(t *testing.T) {
-	app := cmd.NewCmd("curTime", "date", "+%Y-%m-%d")
+	app := cmd.NewCmd("date", "+%Y-%m-%d")
 
 	tm := time.Now()
 	curTime := fmt.Sprintf("%d-%02d-%02d", tm.Year(), tm.Month(), tm.Day())

common/database/db.go

@@ -2,6 +2,7 @@ package database
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 	"time"
 
@@ -63,17 +64,15 @@ func InitDB(config *Config) (*gorm.DB, error) {
 	if err != nil {
 		return nil, err
 	}
-	sqlDB, err := db.DB()
-	if err != nil {
-		return nil, err
+
+	sqlDB, pingErr := Ping(db)
+	if pingErr != nil {
+		return nil, pingErr
 	}
 
 	sqlDB.SetMaxOpenConns(config.MaxOpenNum)
 	sqlDB.SetMaxIdleConns(config.MaxIdleNum)
 
-	if err = sqlDB.Ping(); err != nil {
-		return nil, err
-	}
 	return db, nil
 }
 
@@ -88,3 +87,16 @@ func CloseDB(db *gorm.DB) error {
 	}
 	return nil
 }
+
+// Ping check db status
+func Ping(db *gorm.DB) (*sql.DB, error) {
+	sqlDB, err := db.DB()
+	if err != nil {
+		return nil, err
+	}
+
+	if err = sqlDB.Ping(); err != nil {
+		return nil, err
+	}
+	return sqlDB, nil
+}

common/database/db_test.go

@@ -11,6 +11,10 @@ import (
 	"github.com/mattn/go-colorable"
 	"github.com/mattn/go-isatty"
 	"github.com/scroll-tech/go-ethereum/log"
+	"github.com/stretchr/testify/assert"
+
+	"scroll-tech/common/docker"
+	"scroll-tech/common/version"
 )
 
 func TestGormLogger(t *testing.T) {
@@ -33,3 +37,26 @@ func TestGormLogger(t *testing.T) {
 	gl.Info(context.Background(), "test %s warn:%v", "testInfo", errors.New("test info"))
 	gl.Trace(context.Background(), time.Now(), func() (string, int64) { return "test trace", 1 }, nil)
 }
+
+func TestDB(t *testing.T) {
+	version.Version = "v4.1.98-aaa-bbb-ccc"
+	base := docker.NewDockerApp()
+	base.RunDBImage(t)
+
+	dbCfg := &Config{
+		DSN:        base.DBConfig.DSN,
+		DriverName: base.DBConfig.DriverName,
+		MaxOpenNum: base.DBConfig.MaxOpenNum,
+		MaxIdleNum: base.DBConfig.MaxIdleNum,
+	}
+
+	var err error
+	db, err := InitDB(dbCfg)
+	assert.NoError(t, err)
+
+	sqlDB, err := Ping(db)
+	assert.NoError(t, err)
+	assert.NotNil(t, sqlDB)
+
+	assert.NoError(t, CloseDB(db))
+}

common/docker/docker_app.go

@@ -27,6 +27,7 @@ var (
 
 // AppAPI app interface.
 type AppAPI interface {
+	IsRunning() bool
 	WaitResult(t *testing.T, timeout time.Duration, keyword string) bool
 	RunApp(waitResult func() bool)
 	WaitExit()

common/docker/docker_db.go

@@ -36,7 +36,7 @@ func NewImgDB(image, password, dbName string, port int) ImgInstance {
 		dbName:   dbName,
 		port:     port,
 	}
-	img.cmd = cmd.NewCmd(img.name, img.prepare()...)
+	img.cmd = cmd.NewCmd("docker", img.prepare()...)
 	return img
 }
 
@@ -89,7 +89,7 @@ func (i *ImgDB) IsRunning() bool {
 }
 
 func (i *ImgDB) prepare() []string {
-	cmd := []string{"docker", "run", "--rm", "--name", i.name, "-p", fmt.Sprintf("%d:5432", i.port)}
+	cmd := []string{"run", "--rm", "--name", i.name, "-p", fmt.Sprintf("%d:5432", i.port)}
 	envs := []string{
 		"-e", "POSTGRES_PASSWORD=" + i.password,
 		"-e", fmt.Sprintf("POSTGRES_DB=%s", i.dbName),

common/docker/docker_geth.go

@@ -42,7 +42,7 @@ func NewImgGeth(image, volume, ipc string, hPort, wPort int) GethImgInstance {
 		httpPort: hPort,
 		wsPort:   wPort,
 	}
-	img.cmd = cmd.NewCmd(img.name, img.prepare()...)
+	img.cmd = cmd.NewCmd("docker", img.params()...)
 	return img
 }
 
@@ -149,8 +149,8 @@ func (i *ImgGeth) Stop() error {
 	return cli.ContainerRemove(ctx, i.id, types.ContainerRemoveOptions{})
 }
 
-func (i *ImgGeth) prepare() []string {
-	cmds := []string{"docker", "run", "--rm", "--name", i.name}
+func (i *ImgGeth) params() []string {
+	cmds := []string{"run", "--rm", "--name", i.name}
 	var ports []string
 	if i.httpPort != 0 {
 		ports = append(ports, []string{"-p", strconv.Itoa(i.httpPort) + ":8545"}...)

common/docker/l2geth/Dockerfile

@@ -1,4 +1,4 @@
-FROM scrolltech/l2geth:scroll-v4.3.34
+FROM scrolltech/l2geth:scroll-v4.3.55
 
 RUN mkdir -p /l2geth/keystore
 

common/go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/modern-go/reflect2 v1.0.2
 	github.com/orcaman/concurrent-map v1.0.0
 	github.com/prometheus/client_golang v1.14.0
-	github.com/scroll-tech/go-ethereum v1.10.14-0.20230812030736-25fe3ba69a28
+	github.com/scroll-tech/go-ethereum v1.10.14-0.20230829000527-f883dcdc21fc
 	github.com/stretchr/testify v1.8.3
 	github.com/urfave/cli/v2 v2.25.7
 	gorm.io/driver/postgres v1.5.0
@@ -117,7 +117,7 @@ require (
 	golang.org/x/arch v0.4.0 // indirect
 	golang.org/x/crypto v0.12.0 // indirect
 	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/net v0.12.0 // indirect
+	golang.org/x/net v0.14.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
 	golang.org/x/sys v0.11.0 // indirect
 	golang.org/x/text v0.12.0 // indirect

common/go.sum

@@ -434,8 +434,8 @@ github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/scroll-tech/go-ethereum v1.10.14-0.20230812030736-25fe3ba69a28 h1:CECBTWhZ5NGAn8lGFB4ooRAYxZns8PXoX8kTR/14c04=
-github.com/scroll-tech/go-ethereum v1.10.14-0.20230812030736-25fe3ba69a28/go.mod h1:DiN3p2inoXOxGffxSswDKqWjQ7bU+Mp0c9v0XQXKmaA=
+github.com/scroll-tech/go-ethereum v1.10.14-0.20230829000527-f883dcdc21fc h1:eK3NOpjgm/b2TQ6rYqWx92Zri0kBuxf6gKjjsVxWKr8=
+github.com/scroll-tech/go-ethereum v1.10.14-0.20230829000527-f883dcdc21fc/go.mod h1:DiN3p2inoXOxGffxSswDKqWjQ7bU+Mp0c9v0XQXKmaA=
 github.com/scroll-tech/zktrie v0.6.0 h1:xLrMAO31Yo2BiPg1jtYKzcjpEFnXy8acbB7iIsyshPs=
 github.com/scroll-tech/zktrie v0.6.0/go.mod h1:XvNo7vAk8yxNyTjBDj5WIiFzYW4bx/gJ78+NK6Zn6Uk=
 github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo=
@@ -576,8 +576,8 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
-golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
+golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
+golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=

common/libzkp/impl/Cargo.toml

@@ -8,7 +8,7 @@ edition = "2021"
 crate-type = ["cdylib"]
 
 [patch.crates-io]
-ethers-core = { git = "https://github.com/scroll-tech/ethers-rs.git", branch = "v0.17.0" }
+ethers-core = { git = "https://github.com/scroll-tech/ethers-rs.git", branch = "v2.0.7" }
 [patch."https://github.com/privacy-scaling-explorations/halo2.git"]
 halo2_proofs = { git = "https://github.com/scroll-tech/halo2.git", branch = "develop" }
 [patch."https://github.com/privacy-scaling-explorations/poseidon.git"]
@@ -19,23 +19,18 @@ maingate = { git = "https://github.com/scroll-tech/halo2wrong", branch = "halo2-
 [patch."https://github.com/privacy-scaling-explorations/halo2curves.git"]
 halo2curves = { git = "https://github.com/scroll-tech/halo2curves.git", branch = "0.3.1-derive-serde" }
 
-[patch."https://github.com/scroll-tech/snark-verifier"]
-snark-verifier = { git = "https://github.com/scroll-tech//snark-verifier", tag = "v0.1.1" }
-snark-verifier-sdk = { git = "https://github.com/scroll-tech//snark-verifier", tag = "v0.1.1" }
-
 [dependencies]
-prover = { git = "https://github.com/scroll-tech/scroll-prover", tag = "v0.6.2" }
-types = { git = "https://github.com/scroll-tech/scroll-prover", tag = "v0.6.2" }
 halo2_proofs = { git = "https://github.com/scroll-tech/halo2.git", branch = "develop" }
+prover = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.9.5", default-features = false, features = ["parallel_syn", "scroll", "shanghai"] }
 
-log = "0.4"
+base64 = "0.13.0"
 env_logger = "0.9.0"
+libc = "0.2"
+log = "0.4"
+once_cell = "1.8.0"
 serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0.66"
-libc = "0.2"
-once_cell = "1.8.0"
-
 
 [profile.test]
 opt-level = 3

common/libzkp/impl/Makefile

@@ -0,0 +1,8 @@
+.PHONY: help fmt clippy test test-ci test-all
+
+fmt:
+	@cargo fmt --all -- --check
+
+clippy:
+	@cargo check --all-features
+	@cargo clippy --release -- -D warnings

common/libzkp/impl/src/batch.rs

@@ -1,23 +1,39 @@
-use crate::utils::{c_char_to_str, c_char_to_vec, vec_to_c_char, OUTPUT_DIR};
+use crate::{
+    types::{CheckChunkProofsResponse, ProofResult},
+    utils::{
+        c_char_to_str, c_char_to_vec, file_exists, panic_catch, string_to_c_char, vec_to_c_char,
+        OUTPUT_DIR,
+    },
+};
 use libc::c_char;
 use prover::{
     aggregator::{Prover, Verifier},
+    consts::AGG_VK_FILENAME,
     utils::{chunk_trace_to_witness_block, init_env_and_log},
-    BatchProof, ChunkHash, ChunkProof,
+    BatchProof, BlockTrace, ChunkHash, ChunkProof,
 };
-use std::{cell::OnceCell, panic, ptr::null};
-use types::eth::BlockTrace;
+use std::{cell::OnceCell, env, ptr::null};
 
 static mut PROVER: OnceCell<Prover> = OnceCell::new();
 static mut VERIFIER: OnceCell<Verifier> = OnceCell::new();
 
 /// # Safety
 #[no_mangle]
-pub unsafe extern "C" fn init_batch_prover(params_dir: *const c_char) {
+pub unsafe extern "C" fn init_batch_prover(params_dir: *const c_char, assets_dir: *const c_char) {
     init_env_and_log("ffi_batch_prove");
 
     let params_dir = c_char_to_str(params_dir);
-    let prover = Prover::from_params_dir(params_dir);
+    let assets_dir = c_char_to_str(assets_dir);
+
+    // TODO: add a settings in scroll-prover.
+    env::set_var("SCROLL_PROVER_ASSETS_DIR", assets_dir);
+
+    // VK file must exist, it is optional and logged as a warning in prover.
+    if !file_exists(assets_dir, &AGG_VK_FILENAME) {
+        panic!("{} must exist in folder {}", *AGG_VK_FILENAME, assets_dir);
+    }
+
+    let prover = Prover::from_dirs(params_dir, assets_dir);
 
     PROVER.set(prover).unwrap();
 }
@@ -30,39 +46,104 @@ pub unsafe extern "C" fn init_batch_verifier(params_dir: *const c_char, assets_d
     let params_dir = c_char_to_str(params_dir);
     let assets_dir = c_char_to_str(assets_dir);
 
+    // TODO: add a settings in scroll-prover.
+    env::set_var("SCROLL_PROVER_ASSETS_DIR", assets_dir);
     let verifier = Verifier::from_dirs(params_dir, assets_dir);
 
     VERIFIER.set(verifier).unwrap();
 }
 
+/// # Safety
+#[no_mangle]
+pub unsafe extern "C" fn get_batch_vk() -> *const c_char {
+    let vk_result = panic_catch(|| PROVER.get_mut().unwrap().get_vk());
+
+    vk_result
+        .ok()
+        .flatten()
+        .map_or(null(), |vk| string_to_c_char(base64::encode(vk)))
+}
+
+/// # Safety
+#[no_mangle]
+pub unsafe extern "C" fn check_chunk_proofs(chunk_proofs: *const c_char) -> *const c_char {
+    let check_result: Result<bool, String> = panic_catch(|| {
+        let chunk_proofs = c_char_to_vec(chunk_proofs);
+        let chunk_proofs = serde_json::from_slice::<Vec<ChunkProof>>(&chunk_proofs)
+            .map_err(|e| format!("failed to deserialize chunk proofs: {e:?}"))?;
+
+        if chunk_proofs.is_empty() {
+            return Err("provided chunk proofs are empty.".to_string());
+        }
+
+        let prover_ref = PROVER.get().expect("failed to get reference to PROVER.");
+
+        let valid = prover_ref.check_chunk_proofs(&chunk_proofs);
+        Ok(valid)
+    })
+    .unwrap_or_else(|e| Err(format!("unwind error: {e:?}")));
+
+    let r = match check_result {
+        Ok(valid) => CheckChunkProofsResponse {
+            ok: valid,
+            error: None,
+        },
+        Err(err) => CheckChunkProofsResponse {
+            ok: false,
+            error: Some(err),
+        },
+    };
+
+    serde_json::to_vec(&r).map_or(null(), vec_to_c_char)
+}
+
 /// # Safety
 #[no_mangle]
 pub unsafe extern "C" fn gen_batch_proof(
     chunk_hashes: *const c_char,
     chunk_proofs: *const c_char,
 ) -> *const c_char {
-    let chunk_hashes = c_char_to_vec(chunk_hashes);
-    let chunk_proofs = c_char_to_vec(chunk_proofs);
+    let proof_result: Result<Vec<u8>, String> = panic_catch(|| {
+        let chunk_hashes = c_char_to_vec(chunk_hashes);
+        let chunk_proofs = c_char_to_vec(chunk_proofs);
 
-    let chunk_hashes = serde_json::from_slice::<Vec<ChunkHash>>(&chunk_hashes).unwrap();
-    let chunk_proofs = serde_json::from_slice::<Vec<ChunkProof>>(&chunk_proofs).unwrap();
-    assert_eq!(chunk_hashes.len(), chunk_proofs.len());
+        let chunk_hashes = serde_json::from_slice::<Vec<ChunkHash>>(&chunk_hashes)
+            .map_err(|e| format!("failed to deserialize chunk hashes: {e:?}"))?;
+        let chunk_proofs = serde_json::from_slice::<Vec<ChunkProof>>(&chunk_proofs)
+            .map_err(|e| format!("failed to deserialize chunk proofs: {e:?}"))?;
 
-    let chunk_hashes_proofs = chunk_hashes
-        .into_iter()
-        .zip(chunk_proofs.into_iter())
-        .collect();
+        if chunk_hashes.len() != chunk_proofs.len() {
+            return Err(format!("chunk hashes and chunk proofs lengths mismatch: chunk_hashes.len() = {}, chunk_proofs.len() = {}",
+                chunk_hashes.len(), chunk_proofs.len()));
+        }
+
+        let chunk_hashes_proofs = chunk_hashes
+            .into_iter()
+            .zip(chunk_proofs.into_iter())
+            .collect();
 
-    let proof_result = panic::catch_unwind(|| {
         let proof = PROVER
             .get_mut()
-            .unwrap()
+            .expect("failed to get mutable reference to PROVER.")
             .gen_agg_evm_proof(chunk_hashes_proofs, None, OUTPUT_DIR.as_deref())
-            .unwrap();
+            .map_err(|e| format!("failed to generate proof: {e:?}"))?;
 
-        serde_json::to_vec(&proof).unwrap()
-    });
-    proof_result.map_or(null(), vec_to_c_char)
+        serde_json::to_vec(&proof).map_err(|e| format!("failed to serialize the proof: {e:?}"))
+    })
+    .unwrap_or_else(|e| Err(format!("unwind error: {e:?}")));
+
+    let r = match proof_result {
+        Ok(proof_bytes) => ProofResult {
+            message: Some(proof_bytes),
+            error: None,
+        },
+        Err(err) => ProofResult {
+            message: None,
+            error: Some(err),
+        },
+    };
+
+    serde_json::to_vec(&r).map_or(null(), vec_to_c_char)
 }
 
 /// # Safety
@@ -71,7 +152,7 @@ pub unsafe extern "C" fn verify_batch_proof(proof: *const c_char) -> c_char {
     let proof = c_char_to_vec(proof);
     let proof = serde_json::from_slice::<BatchProof>(proof.as_slice()).unwrap();
 
-    let verified = panic::catch_unwind(|| VERIFIER.get().unwrap().verify_agg_evm_proof(proof));
+    let verified = panic_catch(|| VERIFIER.get().unwrap().verify_agg_evm_proof(proof));
     verified.unwrap_or(false) as c_char
 }
 

common/libzkp/impl/src/chunk.rs

@@ -1,23 +1,39 @@
-use crate::utils::{c_char_to_str, c_char_to_vec, vec_to_c_char, OUTPUT_DIR};
+use crate::{
+    types::ProofResult,
+    utils::{
+        c_char_to_str, c_char_to_vec, file_exists, panic_catch, string_to_c_char, vec_to_c_char,
+        OUTPUT_DIR,
+    },
+};
 use libc::c_char;
 use prover::{
+    consts::CHUNK_VK_FILENAME,
     utils::init_env_and_log,
     zkevm::{Prover, Verifier},
-    ChunkProof,
+    BlockTrace, ChunkProof,
 };
-use std::{cell::OnceCell, panic, ptr::null};
-use types::eth::BlockTrace;
+use std::{cell::OnceCell, env, ptr::null};
 
 static mut PROVER: OnceCell<Prover> = OnceCell::new();
 static mut VERIFIER: OnceCell<Verifier> = OnceCell::new();
 
 /// # Safety
 #[no_mangle]
-pub unsafe extern "C" fn init_chunk_prover(params_dir: *const c_char) {
+pub unsafe extern "C" fn init_chunk_prover(params_dir: *const c_char, assets_dir: *const c_char) {
     init_env_and_log("ffi_chunk_prove");
 
     let params_dir = c_char_to_str(params_dir);
-    let prover = Prover::from_params_dir(params_dir);
+    let assets_dir = c_char_to_str(assets_dir);
+
+    // TODO: add a settings in scroll-prover.
+    env::set_var("SCROLL_PROVER_ASSETS_DIR", assets_dir);
+
+    // VK file must exist, it is optional and logged as a warning in prover.
+    if !file_exists(assets_dir, &CHUNK_VK_FILENAME) {
+        panic!("{} must exist in folder {}", *CHUNK_VK_FILENAME, assets_dir);
+    }
+
+    let prover = Prover::from_dirs(params_dir, assets_dir);
 
     PROVER.set(prover).unwrap();
 }
@@ -30,6 +46,8 @@ pub unsafe extern "C" fn init_chunk_verifier(params_dir: *const c_char, assets_d
     let params_dir = c_char_to_str(params_dir);
     let assets_dir = c_char_to_str(assets_dir);
 
+    // TODO: add a settings in scroll-prover.
+    env::set_var("SCROLL_PROVER_ASSETS_DIR", assets_dir);
     let verifier = Verifier::from_dirs(params_dir, assets_dir);
 
     VERIFIER.set(verifier).unwrap();
@@ -37,21 +55,45 @@ pub unsafe extern "C" fn init_chunk_verifier(params_dir: *const c_char, assets_d
 
 /// # Safety
 #[no_mangle]
-pub unsafe extern "C" fn gen_chunk_proof(block_traces: *const c_char) -> *const c_char {
-    let block_traces = c_char_to_vec(block_traces);
-    let block_traces = serde_json::from_slice::<Vec<BlockTrace>>(&block_traces).unwrap();
+pub unsafe extern "C" fn get_chunk_vk() -> *const c_char {
+    let vk_result = panic_catch(|| PROVER.get_mut().unwrap().get_vk());
+
+    vk_result
+        .ok()
+        .flatten()
+        .map_or(null(), |vk| string_to_c_char(base64::encode(vk)))
+}
+
+/// # Safety
+#[no_mangle]
+pub unsafe extern "C" fn gen_chunk_proof(block_traces: *const c_char) -> *const c_char {
+    let proof_result: Result<Vec<u8>, String> = panic_catch(|| {
+        let block_traces = c_char_to_vec(block_traces);
+        let block_traces = serde_json::from_slice::<Vec<BlockTrace>>(&block_traces)
+            .map_err(|e| format!("failed to deserialize block traces: {e:?}"))?;
 
-    let proof_result = panic::catch_unwind(|| {
         let proof = PROVER
             .get_mut()
-            .unwrap()
-            .gen_chunk_proof(block_traces, None, OUTPUT_DIR.as_deref())
-            .unwrap();
+            .expect("failed to get mutable reference to PROVER.")
+            .gen_chunk_proof(block_traces, None, None, OUTPUT_DIR.as_deref())
+            .map_err(|e| format!("failed to generate proof: {e:?}"))?;
 
-        serde_json::to_vec(&proof).unwrap()
-    });
+        serde_json::to_vec(&proof).map_err(|e| format!("failed to serialize the proof: {e:?}"))
+    })
+    .unwrap_or_else(|e| Err(format!("unwind error: {e:?}")));
 
-    proof_result.map_or(null(), vec_to_c_char)
+    let r = match proof_result {
+        Ok(proof_bytes) => ProofResult {
+            message: Some(proof_bytes),
+            error: None,
+        },
+        Err(err) => ProofResult {
+            message: None,
+            error: Some(err),
+        },
+    };
+
+    serde_json::to_vec(&r).map_or(null(), vec_to_c_char)
 }
 
 /// # Safety
@@ -60,6 +102,6 @@ pub unsafe extern "C" fn verify_chunk_proof(proof: *const c_char) -> c_char {
     let proof = c_char_to_vec(proof);
     let proof = serde_json::from_slice::<ChunkProof>(proof.as_slice()).unwrap();
 
-    let verified = panic::catch_unwind(|| VERIFIER.get().unwrap().verify_chunk_proof(proof));
+    let verified = panic_catch(|| VERIFIER.get().unwrap().verify_chunk_proof(proof));
     verified.unwrap_or(false) as c_char
 }

common/libzkp/impl/src/lib.rs

@@ -2,4 +2,5 @@
 
 mod batch;
 mod chunk;
+mod types;
 mod utils;

common/libzkp/impl/src/types.rs

@@ -0,0 +1,22 @@
+use serde::{Deserialize, Serialize};
+
+// Represents the result of a chunk proof checking operation.
+// `ok` indicates whether the proof checking was successful.
+// `error` provides additional details in case the check failed.
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct CheckChunkProofsResponse {
+    pub ok: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub error: Option<String>,
+}
+
+// Encapsulates the result from generating a proof.
+// `message` holds the generated proof in byte slice format.
+// `error` provides additional details in case the proof generation failed.
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct ProofResult {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub message: Option<Vec<u8>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub error: Option<String>,
+}

common/libzkp/impl/src/utils.rs

@@ -3,6 +3,8 @@ use std::{
     env,
     ffi::{CStr, CString},
     os::raw::c_char,
+    panic::{catch_unwind, AssertUnwindSafe},
+    path::PathBuf,
 };
 
 // Only used for debugging.
@@ -19,6 +21,29 @@ pub(crate) fn c_char_to_vec(c: *const c_char) -> Vec<u8> {
     cstr.to_bytes().to_vec()
 }
 
+pub(crate) fn string_to_c_char(string: String) -> *const c_char {
+    CString::new(string).unwrap().into_raw()
+}
+
 pub(crate) fn vec_to_c_char(bytes: Vec<u8>) -> *const c_char {
     CString::new(bytes).unwrap().into_raw()
 }
+
+pub(crate) fn file_exists(dir: &str, filename: &str) -> bool {
+    let mut path = PathBuf::from(dir);
+    path.push(filename);
+
+    path.exists()
+}
+
+pub(crate) fn panic_catch<F: FnOnce() -> R, R>(f: F) -> Result<R, String> {
+    catch_unwind(AssertUnwindSafe(f)).map_err(|err| {
+        if let Some(s) = err.downcast_ref::<String>() {
+            s.to_string()
+        } else if let Some(s) = err.downcast_ref::<&str>() {
+            s.to_string()
+        } else {
+            format!("unable to get panic info {err:?}")
+        }
+    })
+}

common/libzkp/interface/libzkp.h

@@ -1,10 +1,13 @@
-void init_batch_prover(char* params_dir);
+void init_batch_prover(char* params_dir, char* assets_dir);
 void init_batch_verifier(char* params_dir, char* assets_dir);
+char* get_batch_vk();
+char* check_chunk_proofs(char* chunk_proofs);
 char* gen_batch_proof(char* chunk_hashes, char* chunk_proofs);
 char verify_batch_proof(char* proof);
 
-void init_chunk_prover(char* params_dir);
+void init_chunk_prover(char* params_dir, char* assets_dir);
 void init_chunk_verifier(char* params_dir, char* assets_dir);
+char* get_chunk_vk();
 char* gen_chunk_proof(char* block_traces);
 char verify_chunk_proof(char* proof);
 

common/observability/ginmetrics/bloom.go

@@ -0,0 +1,57 @@
+package ginmetrics
+
+import (
+	"github.com/bits-and-blooms/bitset"
+)
+
+const defaultSize = 2 << 24
+
+var seeds = []uint{7, 11, 13, 31, 37, 61}
+
+// BloomFilter a simple bloom filter
+type BloomFilter struct {
+	Set   *bitset.BitSet
+	Funcs [6]simpleHash
+}
+
+// NewBloomFilter new a BloomFilter
+func NewBloomFilter() *BloomFilter {
+	bf := new(BloomFilter)
+	for i := 0; i < len(bf.Funcs); i++ {
+		bf.Funcs[i] = simpleHash{defaultSize, seeds[i]}
+	}
+	bf.Set = bitset.New(defaultSize)
+	return bf
+}
+
+// Add a value to BloomFilter
+func (bf *BloomFilter) Add(value string) {
+	for _, f := range bf.Funcs {
+		bf.Set.Set(f.hash(value))
+	}
+}
+
+// Contains check the value is in bloom filter
+func (bf *BloomFilter) Contains(value string) bool {
+	if value == "" {
+		return false
+	}
+	ret := true
+	for _, f := range bf.Funcs {
+		ret = ret && bf.Set.Test(f.hash(value))
+	}
+	return ret
+}
+
+type simpleHash struct {
+	Cap  uint
+	Seed uint
+}
+
+func (s *simpleHash) hash(value string) uint {
+	var result uint = 0
+	for i := 0; i < len(value); i++ {
+		result = result*s.Seed + uint(value[i])
+	}
+	return (s.Cap - 1) & result
+}

common/observability/ginmetrics/metric.go

@@ -0,0 +1,89 @@
+package ginmetrics
+
+import (
+	"fmt"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Metric defines a metric object. Users can use it to save
+// metric data. Every metric should be globally unique by name.
+type Metric struct {
+	Type        MetricType
+	Name        string
+	Description string
+	Labels      []string
+	Buckets     []float64
+	Objectives  map[float64]float64
+
+	vec prometheus.Collector
+}
+
+// SetGaugeValue set data for Gauge type Metric.
+func (m *Metric) SetGaugeValue(labelValues []string, value float64) error {
+	if m.Type == None {
+		return fmt.Errorf("metric %s not existed", m.Name)
+	}
+
+	if m.Type != Gauge {
+		return fmt.Errorf("metric %s not Gauge type", m.Name)
+	}
+	m.vec.(*prometheus.GaugeVec).WithLabelValues(labelValues...).Set(value)
+	return nil
+}
+
+// Inc increases value for Counter/Gauge type metric, increments
+// the counter by 1
+func (m *Metric) Inc(labelValues []string) error {
+	if m.Type == None {
+		return fmt.Errorf("metric %s not existed", m.Name)
+	}
+
+	if m.Type != Gauge && m.Type != Counter {
+		return fmt.Errorf("metric %s not Gauge or Counter type", m.Name)
+	}
+	switch m.Type {
+	case Counter:
+		m.vec.(*prometheus.CounterVec).WithLabelValues(labelValues...).Inc()
+	case Gauge:
+		m.vec.(*prometheus.GaugeVec).WithLabelValues(labelValues...).Inc()
+	}
+	return nil
+}
+
+// Add adds the given value to the Metric object. Only
+// for Counter/Gauge type metric.
+func (m *Metric) Add(labelValues []string, value float64) error {
+	if m.Type == None {
+		return fmt.Errorf("metric %s not existed", m.Name)
+	}
+
+	if m.Type != Gauge && m.Type != Counter {
+		return fmt.Errorf("metric %s not Gauge or Counter type", m.Name)
+	}
+	switch m.Type {
+	case Counter:
+		m.vec.(*prometheus.CounterVec).WithLabelValues(labelValues...).Add(value)
+	case Gauge:
+		m.vec.(*prometheus.GaugeVec).WithLabelValues(labelValues...).Add(value)
+	}
+	return nil
+}
+
+// Observe is used by Histogram and Summary type metric to
+// add observations.
+func (m *Metric) Observe(labelValues []string, value float64) error {
+	if m.Type == 0 {
+		return fmt.Errorf("metric %s not existed", m.Name)
+	}
+	if m.Type != Histogram && m.Type != Summary {
+		return fmt.Errorf("metric %s not Histogram or Summary type", m.Name)
+	}
+	switch m.Type {
+	case Histogram:
+		m.vec.(*prometheus.HistogramVec).WithLabelValues(labelValues...).Observe(value)
+	case Summary:
+		m.vec.(*prometheus.SummaryVec).WithLabelValues(labelValues...).Observe(value)
+	}
+	return nil
+}

common/observability/ginmetrics/middleware.go

@@ -0,0 +1,155 @@
+package ginmetrics
+
+import (
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+var (
+	metricRequestTotal    = "request_total"
+	metricRequestUVTotal  = "request_uv_total"
+	metricURIRequestTotal = "uri_request_total"
+	metricRequestBody     = "request_body_total"
+	metricResponseBody    = "response_body_total"
+	metricRequestDuration = "request_duration"
+	metricSlowRequest     = "slow_request_total"
+
+	bloomFilter *BloomFilter
+)
+
+// Use set gin metrics middleware
+func (m *Monitor) Use(r gin.IRoutes) {
+	m.initGinMetrics()
+
+	r.Use(m.monitorInterceptor)
+	r.GET(m.metricPath, func(ctx *gin.Context) {
+		promhttp.Handler().ServeHTTP(ctx.Writer, ctx.Request)
+	})
+}
+
+// UseWithoutExposingEndpoint is used to add monitor interceptor to gin router
+// It can be called multiple times to intercept from multiple gin.IRoutes
+// http path is not set, to do that use Expose function
+func (m *Monitor) UseWithoutExposingEndpoint(r gin.IRoutes) {
+	m.initGinMetrics()
+	r.Use(m.monitorInterceptor)
+}
+
+// Expose adds metric path to a given router.
+// The router can be different with the one passed to UseWithoutExposingEndpoint.
+// This allows to expose metrics on different port.
+func (m *Monitor) Expose(r gin.IRoutes) {
+	r.GET(m.metricPath, func(ctx *gin.Context) {
+		promhttp.Handler().ServeHTTP(ctx.Writer, ctx.Request)
+	})
+}
+
+// initGinMetrics used to init gin metrics
+func (m *Monitor) initGinMetrics() {
+	bloomFilter = NewBloomFilter()
+
+	_ = monitor.AddMetric(&Metric{
+		Type:        Counter,
+		Name:        metricRequestTotal,
+		Description: "all the server received request num.",
+		Labels:      nil,
+	})
+	_ = monitor.AddMetric(&Metric{
+		Type:        Counter,
+		Name:        metricRequestUVTotal,
+		Description: "all the server received ip num.",
+		Labels:      nil,
+	})
+	_ = monitor.AddMetric(&Metric{
+		Type:        Counter,
+		Name:        metricURIRequestTotal,
+		Description: "all the server received request num with every uri.",
+		Labels:      []string{"uri", "method", "code"},
+	})
+	_ = monitor.AddMetric(&Metric{
+		Type:        Counter,
+		Name:        metricRequestBody,
+		Description: "the server received request body size, unit byte",
+		Labels:      nil,
+	})
+	_ = monitor.AddMetric(&Metric{
+		Type:        Counter,
+		Name:        metricResponseBody,
+		Description: "the server send response body size, unit byte",
+		Labels:      nil,
+	})
+	_ = monitor.AddMetric(&Metric{
+		Type:        Histogram,
+		Name:        metricRequestDuration,
+		Description: "the time server took to handle the request.",
+		Labels:      []string{"uri"},
+		Buckets:     m.reqDuration,
+	})
+	_ = monitor.AddMetric(&Metric{
+		Type:        Counter,
+		Name:        metricSlowRequest,
+		Description: fmt.Sprintf("the server handled slow requests counter, t=%d.", m.slowTime),
+		Labels:      []string{"uri", "method", "code"},
+	})
+}
+
+// monitorInterceptor as gin monitor middleware.
+func (m *Monitor) monitorInterceptor(ctx *gin.Context) {
+	if ctx.Request.URL.Path == m.metricPath {
+		ctx.Next()
+		return
+	}
+	startTime := time.Now()
+
+	// execute normal process.
+	ctx.Next()
+
+	// after request
+	m.ginMetricHandle(ctx, startTime)
+}
+
+func (m *Monitor) ginMetricHandle(ctx *gin.Context, start time.Time) {
+	r := ctx.Request
+	w := ctx.Writer
+
+	//set request total
+	_ = m.GetMetric(metricRequestTotal).Inc(nil)
+
+	// set uv
+	if clientIP := ctx.ClientIP(); !bloomFilter.Contains(clientIP) {
+		bloomFilter.Add(clientIP)
+		_ = m.GetMetric(metricRequestUVTotal).Inc(nil)
+	}
+
+	errCode := strconv.Itoa(ctx.GetInt("errcode"))
+	if len(errCode) == 0 {
+		errCode = strconv.Itoa(w.Status())
+	}
+
+	// set uri request total
+	_ = m.GetMetric(metricURIRequestTotal).Inc([]string{ctx.FullPath(), r.Method, errCode})
+
+	// set request body size
+	// since r.ContentLength can be negative (in some occasions) guard the operation
+	if r.ContentLength >= 0 {
+		_ = m.GetMetric(metricRequestBody).Add(nil, float64(r.ContentLength))
+	}
+
+	// set slow request
+	latency := time.Since(start)
+	if int32(latency.Seconds()) > m.slowTime {
+		_ = m.GetMetric(metricSlowRequest).Inc([]string{ctx.FullPath(), r.Method, strconv.Itoa(w.Status())})
+	}
+
+	// set request duration
+	_ = m.GetMetric(metricRequestDuration).Observe([]string{ctx.FullPath()}, latency.Seconds())
+
+	// set response size
+	if w.Size() > 0 {
+		_ = m.GetMetric(metricResponseBody).Add(nil, float64(w.Size()))
+	}
+}

common/observability/ginmetrics/types.go

@@ -0,0 +1,158 @@
+package ginmetrics
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+// MetricType define metric type
+type MetricType int
+
+const (
+	// None unknown metric type
+	None MetricType = iota
+	// Counter MetricType
+	Counter
+	// Gauge MetricType
+	Gauge
+	// Histogram MetricType
+	Histogram
+	// Summary MetricType
+	Summary
+
+	defaultMetricPath = "/debug/metrics"
+	defaultSlowTime   = int32(5)
+)
+
+var (
+	defaultDuration = []float64{0.1, 0.3, 1.2, 5, 10}
+	monitor         *Monitor
+
+	promTypeHandler = map[MetricType]func(metric *Metric, reg prometheus.Registerer){
+		Counter:   counterHandler,
+		Gauge:     gaugeHandler,
+		Histogram: histogramHandler,
+		Summary:   summaryHandler,
+	}
+)
+
+// Monitor is an object that uses to set gin server monitor.
+type Monitor struct {
+	slowTime    int32
+	metricPath  string
+	reqDuration []float64
+	metrics     map[string]*Metric
+	register    prometheus.Registerer
+}
+
+// GetMonitor used to get global Monitor object,
+// this function returns a singleton object.
+func GetMonitor(reg prometheus.Registerer) *Monitor {
+	if monitor == nil {
+		monitor = &Monitor{
+			metricPath:  defaultMetricPath,
+			slowTime:    defaultSlowTime,
+			reqDuration: defaultDuration,
+			metrics:     make(map[string]*Metric),
+			register:    reg,
+		}
+	}
+	return monitor
+}
+
+// GetMetric used to get metric object by metric_name.
+func (m *Monitor) GetMetric(name string) *Metric {
+	if metric, ok := m.metrics[name]; ok {
+		return metric
+	}
+	return &Metric{}
+}
+
+// SetMetricPath set metricPath property. metricPath is used for Prometheus
+// to get gin server monitoring data.
+func (m *Monitor) SetMetricPath(path string) {
+	m.metricPath = path
+}
+
+// SetSlowTime set slowTime property. slowTime is used to determine whether
+// the request is slow. For "gin_slow_request_total" metric.
+func (m *Monitor) SetSlowTime(slowTime int32) {
+	m.slowTime = slowTime
+}
+
+// SetDuration set reqDuration property. reqDuration is used to ginRequestDuration
+// metric buckets.
+func (m *Monitor) SetDuration(duration []float64) {
+	m.reqDuration = duration
+}
+
+// SetMetricPrefix set the metric prefix
+func (m *Monitor) SetMetricPrefix(prefix string) {
+	metricRequestTotal = prefix + metricRequestTotal
+	metricRequestUVTotal = prefix + metricRequestUVTotal
+	metricURIRequestTotal = prefix + metricURIRequestTotal
+	metricRequestBody = prefix + metricRequestBody
+	metricResponseBody = prefix + metricResponseBody
+	metricRequestDuration = prefix + metricRequestDuration
+	metricSlowRequest = prefix + metricSlowRequest
+}
+
+// SetMetricSuffix set the metric suffix
+func (m *Monitor) SetMetricSuffix(suffix string) {
+	metricRequestTotal += suffix
+	metricRequestUVTotal += suffix
+	metricURIRequestTotal += suffix
+	metricRequestBody += suffix
+	metricResponseBody += suffix
+	metricRequestDuration += suffix
+	metricSlowRequest += suffix
+}
+
+// AddMetric add custom monitor metric.
+func (m *Monitor) AddMetric(metric *Metric) error {
+	if _, ok := m.metrics[metric.Name]; ok {
+		return fmt.Errorf("metric %s is existed", metric.Name)
+	}
+
+	if metric.Name == "" {
+		return errors.New("metric name cannot be empty")
+	}
+
+	if f, ok := promTypeHandler[metric.Type]; ok {
+		f(metric, m.register)
+		m.metrics[metric.Name] = metric
+	}
+
+	return nil
+}
+
+func counterHandler(metric *Metric, register prometheus.Registerer) {
+	metric.vec = promauto.With(register).NewCounterVec(
+		prometheus.CounterOpts{Name: metric.Name, Help: metric.Description},
+		metric.Labels,
+	)
+}
+
+func gaugeHandler(metric *Metric, register prometheus.Registerer) {
+	metric.vec = promauto.With(register).NewGaugeVec(
+		prometheus.GaugeOpts{Name: metric.Name, Help: metric.Description},
+		metric.Labels,
+	)
+}
+
+func histogramHandler(metric *Metric, register prometheus.Registerer) {
+	metric.vec = promauto.With(register).NewHistogramVec(
+		prometheus.HistogramOpts{Name: metric.Name, Help: metric.Description, Buckets: metric.Buckets},
+		metric.Labels,
+	)
+}
+
+func summaryHandler(metric *Metric, register prometheus.Registerer) {
+	promauto.With(register).NewSummaryVec(
+		prometheus.SummaryOpts{Name: metric.Name, Help: metric.Description, Objectives: metric.Objectives},
+		metric.Labels,
+	)
+}

common/observability/middleware.go

@@ -0,0 +1,18 @@
+package observability
+
+import (
+	"github.com/gin-gonic/gin"
+	"github.com/prometheus/client_golang/prometheus"
+
+	"scroll-tech/common/observability/ginmetrics"
+)
+
+// Use register the gin metric
+func Use(router *gin.Engine, metricsPrefix string, reg prometheus.Registerer) {
+	m := ginmetrics.GetMonitor(reg)
+	m.SetMetricPath("/metrics")
+	m.SetMetricPrefix(metricsPrefix + "_")
+	m.SetSlowTime(1)
+	m.SetDuration([]float64{0.025, .05, .1, .5, 1, 5, 10})
+	m.UseWithoutExposingEndpoint(router)
+}

common/observability/probes.go

@@ -0,0 +1,35 @@
+package observability
+
+import (
+	"github.com/gin-gonic/gin"
+	"gorm.io/gorm"
+
+	"scroll-tech/common/database"
+	"scroll-tech/common/types"
+)
+
+// ProbesController probe check controller
+type ProbesController struct {
+	db *gorm.DB
+}
+
+// NewProbesController returns an ProbesController instance
+func NewProbesController(db *gorm.DB) *ProbesController {
+	return &ProbesController{
+		db: db,
+	}
+}
+
+// HealthCheck the api controller for health check
+func (a *ProbesController) HealthCheck(c *gin.Context) {
+	if _, err := database.Ping(a.db); err != nil {
+		types.RenderFatal(c, err)
+		return
+	}
+	types.RenderSuccess(c, nil)
+}
+
+// Ready the api controller for ready check
+func (a *ProbesController) Ready(c *gin.Context) {
+	types.RenderSuccess(c, nil)
+}

common/observability/server.go

@@ -1,4 +1,4 @@
-package metrics
+package observability
 
 import (
 	"errors"
@@ -11,17 +11,17 @@ import (
 
 	"github.com/gin-contrib/pprof"
 	"github.com/gin-gonic/gin"
-	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/scroll-tech/go-ethereum/log"
 	"github.com/urfave/cli/v2"
+	"gorm.io/gorm"
 
 	"scroll-tech/common/utils"
 )
 
 // Server starts the metrics server on the given address, will be closed when the given
 // context is canceled.
-func Server(c *cli.Context, reg *prometheus.Registry) {
+func Server(c *cli.Context, db *gorm.DB) {
 	if !c.Bool(utils.MetricsEnabled.Name) {
 		return
 	}
@@ -33,6 +33,10 @@ func Server(c *cli.Context, reg *prometheus.Registry) {
 		promhttp.Handler().ServeHTTP(context.Writer, context.Request)
 	})
 
+	probeController := NewProbesController(db)
+	r.GET("/health", probeController.HealthCheck)
+	r.GET("/ready", probeController.Ready)
+
 	address := fmt.Sprintf(":%s", c.String(utils.MetricsPort.Name))
 	server := &http.Server{
 		Addr:              address,

common/types/batch_header.go

@@ -10,6 +10,16 @@ import (
 	"github.com/scroll-tech/go-ethereum/crypto"
 )
 
+// BatchMeta contains metadata of a batch.
+type BatchMeta struct {
+	StartChunkIndex           uint64
+	StartChunkHash            string
+	EndChunkIndex             uint64
+	EndChunkHash              string
+	TotalL1CommitGas          uint64
+	TotalL1CommitCalldataSize uint32
+}
+
 // BatchHeader contains batch header info to be committed.
 type BatchHeader struct {
 	// Encoded in BatchHeaderV0Codec

common/types/block.go

@@ -3,20 +3,38 @@ package types
 import (
 	"encoding/binary"
 	"errors"
+	"fmt"
 	"math"
 
 	"github.com/scroll-tech/go-ethereum/common"
 	"github.com/scroll-tech/go-ethereum/common/hexutil"
 	"github.com/scroll-tech/go-ethereum/core/types"
+	"github.com/scroll-tech/go-ethereum/log"
 )
 
+// CalldataNonZeroByteGas is the gas consumption per non zero byte in calldata.
+const CalldataNonZeroByteGas = 16
+
+// GetKeccak256Gas calculates the gas cost for computing the keccak256 hash of a given size.
+func GetKeccak256Gas(size uint64) uint64 {
+	return GetMemoryExpansionCost(size) + 30 + 6*((size+31)/32)
+}
+
+// GetMemoryExpansionCost calculates the cost of memory expansion for a given memoryByteSize.
+func GetMemoryExpansionCost(memoryByteSize uint64) uint64 {
+	memorySizeWord := (memoryByteSize + 31) / 32
+	memoryCost := (memorySizeWord*memorySizeWord)/512 + (3 * memorySizeWord)
+	return memoryCost
+}
+
 // WrappedBlock contains the block's Header, Transactions and WithdrawTrieRoot hash.
 type WrappedBlock struct {
 	Header *types.Header `json:"header"`
 	// Transactions is only used for recover types.Transactions, the from of types.TransactionData field is missing.
-	Transactions   []*types.TransactionData `json:"transactions"`
-	WithdrawRoot   common.Hash              `json:"withdraw_trie_root,omitempty"`
-	RowConsumption *types.RowConsumption    `json:"row_consumption"`
+	Transactions         []*types.TransactionData `json:"transactions"`
+	WithdrawRoot         common.Hash              `json:"withdraw_trie_root,omitempty"`
+	RowConsumption       *types.RowConsumption    `json:"row_consumption"`
+	txPayloadLengthCache map[string]uint64
 }
 
 // NumL1Messages returns the number of L1 messages in this block.
@@ -87,17 +105,15 @@ func (w *WrappedBlock) EstimateL1CommitCalldataSize() uint64 {
 		if txData.Type == types.L1MessageTxType {
 			continue
 		}
-		size += uint64(len(txData.Data))
+		size += 4 // 4 bytes payload length
+		size += w.getTxPayloadLength(txData)
 	}
+	size += 60 //  60 bytes BlockContext
 	return size
 }
 
 // EstimateL1CommitGas calculates the total L1 commit gas for this block approximately.
 func (w *WrappedBlock) EstimateL1CommitGas() uint64 {
-	getKeccakGas := func(size uint64) uint64 {
-		return 30 + 6*((size+31)/32) // 30 + 6 * ceil(size / 32)
-	}
-
 	var total uint64
 	var numL1Messages uint64
 	for _, txData := range w.Transactions {
@@ -106,25 +122,15 @@ func (w *WrappedBlock) EstimateL1CommitGas() uint64 {
 			continue
 		}
 
-		data, _ := hexutil.Decode(txData.Data)
-		tx := types.NewTx(&types.LegacyTx{
-			Nonce:    txData.Nonce,
-			To:       txData.To,
-			Value:    txData.Value.ToInt(),
-			Gas:      txData.Gas,
-			GasPrice: txData.GasPrice.ToInt(),
-			Data:     data,
-			V:        txData.V.ToInt(),
-			R:        txData.R.ToInt(),
-			S:        txData.S.ToInt(),
-		})
-		rlpTxData, _ := tx.MarshalBinary()
-		txPayloadLength := uint64(len(rlpTxData))
-		total += 16 * txPayloadLength          // an over-estimate: treat each byte as non-zero
-		total += 16 * 4                        // size of a uint32 field
-		total += getKeccakGas(txPayloadLength) // l2 tx hash
+		txPayloadLength := w.getTxPayloadLength(txData)
+		total += CalldataNonZeroByteGas * txPayloadLength // an over-estimate: treat each byte as non-zero
+		total += CalldataNonZeroByteGas * 4               // 4 bytes payload length
+		total += GetKeccak256Gas(txPayloadLength)         // l2 tx hash
 	}
 
+	// 60 bytes BlockContext calldata
+	total += CalldataNonZeroByteGas * 60
+
 	// sload
 	total += 2100 * numL1Messages // numL1Messages times cold sload in L1MessageQueue
 
@@ -132,16 +138,56 @@ func (w *WrappedBlock) EstimateL1CommitGas() uint64 {
 	total += 100 * numL1Messages // numL1Messages times call to L1MessageQueue
 	total += 100 * numL1Messages // numL1Messages times warm address access to L1MessageQueue
 
+	total += GetMemoryExpansionCost(36) * numL1Messages // staticcall to proxy
+	total += 100 * numL1Messages                        // read admin in proxy
+	total += 100 * numL1Messages                        // read impl in proxy
+	total += 100 * numL1Messages                        // access impl
+	total += GetMemoryExpansionCost(36) * numL1Messages // delegatecall to impl
+
 	return total
 }
 
-// L2TxsNum calculates the number of l2 txs.
-func (w *WrappedBlock) L2TxsNum() uint64 {
-	var count uint64
-	for _, txData := range w.Transactions {
-		if txData.Type != types.L1MessageTxType {
-			count++
-		}
+func (w *WrappedBlock) getTxPayloadLength(txData *types.TransactionData) uint64 {
+	if w.txPayloadLengthCache == nil {
+		w.txPayloadLengthCache = make(map[string]uint64)
 	}
-	return count
+
+	if length, exists := w.txPayloadLengthCache[txData.TxHash]; exists {
+		return length
+	}
+
+	rlpTxData, err := convertTxDataToRLPEncoding(txData)
+	if err != nil {
+		log.Crit("convertTxDataToRLPEncoding failed, which should not happen", "hash", txData.TxHash, "err", err)
+		return 0
+	}
+	txPayloadLength := uint64(len(rlpTxData))
+	w.txPayloadLengthCache[txData.TxHash] = txPayloadLength
+	return txPayloadLength
+}
+
+func convertTxDataToRLPEncoding(txData *types.TransactionData) ([]byte, error) {
+	data, err := hexutil.Decode(txData.Data)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode txData.Data: %s, err: %w", txData.Data, err)
+	}
+
+	tx := types.NewTx(&types.LegacyTx{
+		Nonce:    txData.Nonce,
+		To:       txData.To,
+		Value:    txData.Value.ToInt(),
+		Gas:      txData.Gas,
+		GasPrice: txData.GasPrice.ToInt(),
+		Data:     data,
+		V:        txData.V.ToInt(),
+		R:        txData.R.ToInt(),
+		S:        txData.S.ToInt(),
+	})
+
+	rlpTxData, err := tx.MarshalBinary()
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal binary of the tx: %+v, err: %w", tx, err)
+	}
+
+	return rlpTxData, nil
 }

common/types/chunk.go

@@ -8,7 +8,6 @@ import (
 	"strings"
 
 	"github.com/scroll-tech/go-ethereum/common"
-	"github.com/scroll-tech/go-ethereum/common/hexutil"
 	"github.com/scroll-tech/go-ethereum/core/types"
 	"github.com/scroll-tech/go-ethereum/crypto"
 )
@@ -65,23 +64,7 @@ func (c *Chunk) Encode(totalL1MessagePoppedBefore uint64) ([]byte, error) {
 			if txData.Type == types.L1MessageTxType {
 				continue
 			}
-			data, err := hexutil.Decode(txData.Data)
-			if err != nil {
-				return nil, err
-			}
-			// right now we only support legacy tx
-			tx := types.NewTx(&types.LegacyTx{
-				Nonce:    txData.Nonce,
-				To:       txData.To,
-				Value:    txData.Value.ToInt(),
-				Gas:      txData.Gas,
-				GasPrice: txData.GasPrice.ToInt(),
-				Data:     data,
-				V:        txData.V.ToInt(),
-				R:        txData.R.ToInt(),
-				S:        txData.S.ToInt(),
-			})
-			rlpTxData, err := tx.MarshalBinary()
+			rlpTxData, err := convertTxDataToRLPEncoding(txData)
 			if err != nil {
 				return nil, err
 			}
@@ -146,14 +129,10 @@ func (c *Chunk) EstimateL1CommitGas() uint64 {
 	}
 
 	numBlocks := uint64(len(c.Blocks))
-	totalL1CommitGas += 100 * numBlocks     // numBlocks times warm sload
-	totalL1CommitGas += 16                  // numBlocks field of chunk encoding in calldata
-	totalL1CommitGas += 16 * 60 * numBlocks // BlockContext in chunk
+	totalL1CommitGas += 100 * numBlocks                         // numBlocks times warm sload
+	totalL1CommitGas += CalldataNonZeroByteGas                  // numBlocks field of chunk encoding in calldata
+	totalL1CommitGas += CalldataNonZeroByteGas * numBlocks * 60 // numBlocks of BlockContext in chunk
 
-	getKeccakGas := func(size uint64) uint64 {
-		return 30 + 6*((size+31)/32) // 30 + 6 * ceil(size / 32)
-	}
-
-	totalL1CommitGas += getKeccakGas(58*numBlocks + 32*totalTxNum) // chunk hash
+	totalL1CommitGas += GetKeccak256Gas(58*numBlocks + 32*totalTxNum) // chunk hash
 	return totalL1CommitGas
 }

common/types/chunk_test.go

@@ -38,11 +38,15 @@ func TestChunkEncode(t *testing.T) {
 	wrappedBlock := &WrappedBlock{}
 	assert.NoError(t, json.Unmarshal(templateBlockTrace, wrappedBlock))
 	assert.Equal(t, uint64(0), wrappedBlock.NumL1Messages(0))
+	assert.Equal(t, uint64(298), wrappedBlock.EstimateL1CommitCalldataSize())
+	assert.Equal(t, uint64(2), wrappedBlock.NumL2Transactions())
 	chunk = &Chunk{
 		Blocks: []*WrappedBlock{
 			wrappedBlock,
 		},
 	}
+	assert.Equal(t, uint64(0), chunk.NumL1Messages(0))
+	assert.Equal(t, uint64(6042), chunk.EstimateL1CommitGas())
 	bytes, err = chunk.Encode(0)
 	hexString := hex.EncodeToString(bytes)
 	assert.NoError(t, err)
@@ -56,11 +60,15 @@ func TestChunkEncode(t *testing.T) {
 	wrappedBlock2 := &WrappedBlock{}
 	assert.NoError(t, json.Unmarshal(templateBlockTrace2, wrappedBlock2))
 	assert.Equal(t, uint64(11), wrappedBlock2.NumL1Messages(0)) // 0..=9 skipped, 10 included
+	assert.Equal(t, uint64(96), wrappedBlock2.EstimateL1CommitCalldataSize())
+	assert.Equal(t, uint64(1), wrappedBlock2.NumL2Transactions())
 	chunk = &Chunk{
 		Blocks: []*WrappedBlock{
 			wrappedBlock2,
 		},
 	}
+	assert.Equal(t, uint64(11), chunk.NumL1Messages(0))
+	assert.Equal(t, uint64(5329), chunk.EstimateL1CommitGas())
 	bytes, err = chunk.Encode(0)
 	hexString = hex.EncodeToString(bytes)
 	assert.NoError(t, err)
@@ -75,6 +83,8 @@ func TestChunkEncode(t *testing.T) {
 			wrappedBlock2,
 		},
 	}
+	assert.Equal(t, uint64(11), chunk.NumL1Messages(0))
+	assert.Equal(t, uint64(10612), chunk.EstimateL1CommitGas())
 	bytes, err = chunk.Encode(0)
 	hexString = hex.EncodeToString(bytes)
 	assert.NoError(t, err)
@@ -136,3 +146,81 @@ func TestChunkHash(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, "0x2eb7dd63bf8fc29a0f8c10d16c2ae6f9da446907c79d50f5c164d30dc8526b60", hash.Hex())
 }
+
+func TestErrorPaths(t *testing.T) {
+	// test 1: Header.Number is not a uint64
+	templateBlockTrace, err := os.ReadFile("../testdata/blockTrace_02.json")
+	assert.NoError(t, err)
+
+	wrappedBlock := &WrappedBlock{}
+
+	assert.NoError(t, json.Unmarshal(templateBlockTrace, wrappedBlock))
+	wrappedBlock.Header.Number = wrappedBlock.Header.Number.Lsh(wrappedBlock.Header.Number, 64)
+	bytes, err := wrappedBlock.Encode(0)
+	assert.Nil(t, bytes)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "block number is not uint64")
+
+	assert.NoError(t, json.Unmarshal(templateBlockTrace, wrappedBlock))
+	for i := 0; i < 65537; i++ {
+		wrappedBlock.Transactions = append(wrappedBlock.Transactions, wrappedBlock.Transactions[0])
+	}
+
+	bytes, err = wrappedBlock.Encode(0)
+	assert.Nil(t, bytes)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "number of transactions exceeds max uint16")
+
+	chunk := &Chunk{
+		Blocks: []*WrappedBlock{
+			wrappedBlock,
+		},
+	}
+
+	bytes, err = chunk.Encode(0)
+	assert.Nil(t, bytes)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "number of transactions exceeds max uint16")
+
+	wrappedBlock.Transactions = wrappedBlock.Transactions[:1]
+	wrappedBlock.Transactions[0].Data = "not-a-hex"
+	bytes, err = chunk.Encode(0)
+	assert.Nil(t, bytes)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "hex string without 0x prefix")
+
+	assert.NoError(t, json.Unmarshal(templateBlockTrace, wrappedBlock))
+	wrappedBlock.Transactions[0].TxHash = "not-a-hex"
+	_, err = chunk.Hash(0)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "invalid byte")
+
+	templateBlockTrace2, err := os.ReadFile("../testdata/blockTrace_04.json")
+	assert.NoError(t, err)
+
+	wrappedBlock2 := &WrappedBlock{}
+	assert.NoError(t, json.Unmarshal(templateBlockTrace2, wrappedBlock2))
+	for i := 0; i < 65535; i++ {
+		tx := &wrappedBlock2.Transactions[i]
+		txCopy := *tx
+		txCopy.Nonce = uint64(i + 1)
+		wrappedBlock2.Transactions = append(wrappedBlock2.Transactions, txCopy)
+	}
+
+	bytes, err = wrappedBlock2.Encode(0)
+	assert.Nil(t, bytes)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "number of L1 messages exceeds max uint16")
+
+	chunk = &Chunk{
+		Blocks: []*WrappedBlock{
+			wrappedBlock2,
+		},
+	}
+
+	bytes, err = chunk.Encode(0)
+	assert.Nil(t, bytes)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "number of L1 messages exceeds max uint16")
+
+}

common/types/db.go

@@ -103,6 +103,12 @@ const (
 	ProverTaskFailureTypeUndefined ProverTaskFailureType = iota
 	// ProverTaskFailureTypeTimeout prover task failure of timeout
 	ProverTaskFailureTypeTimeout
+	// ProverTaskFailureTypeSubmitStatusNotOk prover task failure of submit status not ok
+	ProverTaskFailureTypeSubmitStatusNotOk
+	// ProverTaskFailureTypeVerifiedFailed prover task failure of verified failed by coordinator
+	ProverTaskFailureTypeVerifiedFailed
+	// ProverTaskFailureTypeServerError collect occur error
+	ProverTaskFailureTypeServerError
 )
 
 func (r ProverTaskFailureType) String() string {
@@ -111,8 +117,14 @@ func (r ProverTaskFailureType) String() string {
 		return "prover task failure undefined"
 	case ProverTaskFailureTypeTimeout:
 		return "prover task failure timeout"
+	case ProverTaskFailureTypeSubmitStatusNotOk:
+		return "prover task failure validated submit proof status not ok"
+	case ProverTaskFailureTypeVerifiedFailed:
+		return "prover task failure verified failed"
+	case ProverTaskFailureTypeServerError:
+		return "prover task failure server exception"
 	default:
-		return "illegal prover task failure type"
+		return fmt.Sprintf("illegal prover task failure type (%d)", int32(r))
 	}
 }
 
@@ -126,8 +138,8 @@ const (
 	ProvingTaskUnassigned
 	// ProvingTaskAssigned : proving_task is assigned to be proved
 	ProvingTaskAssigned
-	// ProvingTaskProved DEPRECATED: proof has been returned by prover
-	ProvingTaskProved
+	// ProvingTaskProvedDEPRECATED DEPRECATED: proof has been returned by prover
+	ProvingTaskProvedDEPRECATED
 	// ProvingTaskVerified : proof is valid
 	ProvingTaskVerified
 	// ProvingTaskFailed : fail to generate proof
@@ -140,7 +152,7 @@ func (ps ProvingStatus) String() string {
 		return "unassigned"
 	case ProvingTaskAssigned:
 		return "assigned"
-	case ProvingTaskProved:
+	case ProvingTaskProvedDEPRECATED:
 		return "proved"
 	case ProvingTaskVerified:
 		return "verified"

common/types/db_test.go

@@ -58,8 +58,8 @@ func TestProvingStatus(t *testing.T) {
 			"assigned",
 		},
 		{
-			"ProvingTaskProved",
-			ProvingTaskProved,
+			"ProvingTaskProvedDEPRECATED",
+			ProvingTaskProvedDEPRECATED,
 			"proved",
 		},
 		{

common/types/errno.go

@@ -3,12 +3,21 @@ package types
 const (
 	// Success shows OK.
 	Success = 0
+	// InternalServerError shows a fatal error in the server
+	InternalServerError = 500
 
 	// ErrJWTCommonErr jwt common error
 	ErrJWTCommonErr = 50000
 	// ErrJWTTokenExpired jwt token expired
 	ErrJWTTokenExpired = 50001
 
+	// ErrProverStatsAPIParameterInvalidNo is invalid params
+	ErrProverStatsAPIParameterInvalidNo = 10001
+	// ErrProverStatsAPIProverTaskFailure is getting prover task  error
+	ErrProverStatsAPIProverTaskFailure = 10002
+	// ErrProverStatsAPIProverTotalRewardFailure is getting total rewards error
+	ErrProverStatsAPIProverTotalRewardFailure = 10003
+
 	// ErrCoordinatorParameterInvalidNo is invalid params
 	ErrCoordinatorParameterInvalidNo = 20001
 	// ErrCoordinatorGetTaskFailure is getting prover task error

common/types/message/message.go

@@ -211,6 +211,7 @@ func (a *ProofMsg) PublicKey() (string, error) {
 
 // TaskMsg is a wrapper type around db ProveTask type.
 type TaskMsg struct {
+	UUID            string           `json:"uuid"`
 	ID              string           `json:"id"`
 	Type            ProofType        `json:"type,omitempty"`
 	BatchTaskDetail *BatchTaskDetail `json:"batch_task_detail,omitempty"`
@@ -262,7 +263,7 @@ type ChunkInfo struct {
 
 // ChunkProof includes the proof info that are required for chunk verification and rollup.
 type ChunkProof struct {
-	StorageTrace []byte `json:"storage_trace"`
+	StorageTrace []byte `json:"storage_trace,omitempty"`
 	Protocol     []byte `json:"protocol"`
 	Proof        []byte `json:"proof"`
 	Instances    []byte `json:"instances"`

common/types/response.go

@@ -0,0 +1,53 @@
+package types
+
+import (
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+)
+
+// Response the response schema
+type Response struct {
+	ErrCode int         `json:"errcode"`
+	ErrMsg  string      `json:"errmsg"`
+	Data    interface{} `json:"data"`
+}
+
+// RenderJSON renders response with json
+func RenderJSON(ctx *gin.Context, errCode int, err error, data interface{}) {
+	var errMsg string
+	if err != nil {
+		errMsg = err.Error()
+	}
+	renderData := Response{
+		ErrCode: errCode,
+		ErrMsg:  errMsg,
+		Data:    data,
+	}
+	ctx.JSON(http.StatusOK, renderData)
+}
+
+// RenderSuccess renders success response with json
+func RenderSuccess(ctx *gin.Context, data interface{}) {
+	RenderJSON(ctx, Success, nil, data)
+}
+
+// RenderFailure renders failure response with json
+func RenderFailure(ctx *gin.Context, errCode int, err error) {
+	RenderJSON(ctx, errCode, err, nil)
+}
+
+// RenderFatal renders fatal response with json
+func RenderFatal(ctx *gin.Context, err error) {
+	var errMsg string
+	if err != nil {
+		errMsg = err.Error()
+	}
+	renderData := Response{
+		ErrCode: InternalServerError,
+		ErrMsg:  errMsg,
+		Data:    nil,
+	}
+	ctx.Set("errcode", InternalServerError)
+	ctx.JSON(http.StatusInternalServerError, renderData)
+}