feat(rollup-relayer): use db gas estimation as min gas limit (#901)

Co-authored-by: colinlyguo <colinlyguo@users.noreply.github.com>
Co-authored-by: Xi Lin <zimpha@gmail.com>
Co-authored-by: HAOYUatHZ <37070449+HAOYUatHZ@users.noreply.github.com>

.github/pull_request_template.md

@@ -20,7 +20,7 @@ Your PR title must follow [conventional commits](https://www.conventionalcommits
 
 ### Deployment tag versioning
 
-Has `tag` in `common/version.go` been updated?
+Has `tag` in `common/version.go` been updated or have you added `bump-version` tag to this PR?
 
 - [ ] No, this PR doesn't involve a new deployment, git tag, docker image tag
 - [ ] Yes

.github/workflows/bridge.yml

@@ -10,6 +10,7 @@ on:
     paths:
       - 'bridge/**'
       - 'common/**'
+      - '!common/version/version.go'
       - 'database/**'
       - '.github/workflows/bridge.yml'
   pull_request:
@@ -21,6 +22,7 @@ on:
     paths:
       - 'bridge/**'
       - 'common/**'
+      - '!common/version/version.go'
       - 'database/**'
       - '.github/workflows/bridge.yml'
 

.github/workflows/bump_version.yml

@@ -2,15 +2,12 @@ name: Bump Version
 
 on:
   pull_request:
-    branches: [develop]
-    types:
-      - opened
-      - reopened
-      - synchronize
-      - ready_for_review
+    branches: [ develop ]
+    types: [ labeled ]
 
 jobs:
   try-to-bump:
+    if: ${{ github.event.label.name == 'bump-version' }}
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code

.github/workflows/common.yml

@@ -9,6 +9,7 @@ on:
       - alpha
     paths:
       - 'common/**'
+      - '!common/version/version.go'
       - '.github/workflows/common.yml'
   pull_request:
     types:
@@ -18,6 +19,7 @@ on:
       - ready_for_review
     paths:
       - 'common/**'
+      - '!common/version/version.go'
       - '.github/workflows/common.yml'
 
 jobs:

.github/workflows/coordinator.yml

@@ -10,6 +10,7 @@ on:
     paths:
       - 'coordinator/**'
       - 'common/**'
+      - '!common/version/version.go'
       - 'database/**'
       - '.github/workflows/coordinator.yml'
   pull_request:
@@ -21,6 +22,7 @@ on:
     paths:
       - 'coordinator/**'
       - 'common/**'
+      - '!common/version/version.go'
       - 'database/**'
       - '.github/workflows/coordinator.yml'
 

.github/workflows/database.yml

@@ -10,6 +10,7 @@ on:
     paths:
       - 'database/**'
       - 'common/**'
+      - '!common/version/version.go'
       - '.github/workflows/database.yml'
   pull_request:
     types:
@@ -20,6 +21,7 @@ on:
     paths:
       - 'database/**'
       - 'common/**'
+      - '!common/version/version.go'
       - '.github/workflows/database.yml'
 
 jobs:

bridge/conf/config.json

@@ -66,7 +66,8 @@
       "message_sender_private_key": "1212121212121212121212121212121212121212121212121212121212121212",
       "gas_oracle_sender_private_key": "1313131313131313131313131313131313131313131313131313131313131313",
       "commit_sender_private_key": "1414141414141414141414141414141414141414141414141414141414141414",
-      "finalize_sender_private_key": "1515151515151515151515151515151515151515151515151515151515151515"
+      "finalize_sender_private_key": "1515151515151515151515151515151515151515151515151515151515151515",
+      "gas_cost_increase_multiplier": 1.2
     },
     "chunk_proposer_config": {
       "max_tx_num_per_chunk": 1123,

bridge/internal/config/relayer.go

@@ -63,6 +63,8 @@ type RelayerConfig struct {
 	MessageRelayMinGasLimit uint64 `json:"message_relay_min_gas_limit,omitempty"`
 	// ChainMonitor config of monitoring service
 	ChainMonitor *ChainMonitor `json:"chain_monitor,omitempty"`
+	// GasCostIncreaseMultiplier multiplier for min gas limit estimation
+	GasCostIncreaseMultiplier float64 `json:"gas_cost_increase_multiplier,omitempty"`
 	// The private key of the relayer
 	MessageSenderPrivateKey   *ecdsa.PrivateKey `json:"-"`
 	GasOracleSenderPrivateKey *ecdsa.PrivateKey `json:"-"`

bridge/internal/controller/relayer/l2_relayer.go

@@ -332,7 +332,7 @@ func (r *Layer2Relayer) ProcessGasPriceOracle() {
 // ProcessPendingBatches processes the pending batches by sending commitBatch transactions to layer 1.
 func (r *Layer2Relayer) ProcessPendingBatches() {
 	// get pending batches from database in ascending order by their index.
-	pendingBatches, err := r.batchOrm.GetPendingBatches(r.ctx, 1)
+	pendingBatches, err := r.batchOrm.GetPendingBatches(r.ctx, 5)
 	if err != nil {
 		log.Error("Failed to fetch pending L2 batches", "err", err)
 		return
@@ -395,7 +395,8 @@ func (r *Layer2Relayer) ProcessPendingBatches() {
 
 		// send transaction
 		txID := batch.Hash + "-commit"
-		txHash, err := r.commitSender.SendTransaction(txID, &r.cfg.RollupContractAddress, big.NewInt(0), calldata, 0)
+		minGasLimit := uint64(float64(batch.TotalL1CommitGas) * r.cfg.GasCostIncreaseMultiplier)
+		txHash, err := r.commitSender.SendTransaction(txID, &r.cfg.RollupContractAddress, big.NewInt(0), calldata, minGasLimit)
 		if err != nil {
 			log.Error(
 				"Failed to send commitBatch tx to layer1",
@@ -523,7 +524,6 @@ func (r *Layer2Relayer) ProcessCommittedBatches() {
 					"RollupContractAddress", r.cfg.RollupContractAddress,
 					"err", err,
 				)
-
 				log.Debug(
 					"finalizeBatchWithProof in layer1 failed",
 					"index", batch.Index,

bridge/internal/controller/sender/estimategas.go

@@ -16,7 +16,6 @@ func (s *Sender) estimateLegacyGas(auth *bind.TransactOpts, contract *common.Add
 		log.Error("estimateLegacyGas SuggestGasPrice failure", "error", err)
 		return nil, err
 	}
-
 	gasLimit, err := s.estimateGasLimit(auth, contract, input, gasPrice, nil, nil, value, minGasLimit)
 	if err != nil {
 		log.Error("estimateLegacyGas estimateGasLimit failure", "gasPrice", gasPrice, "error", err)
@@ -46,7 +45,10 @@ func (s *Sender) estimateDynamicGas(auth *bind.TransactOpts, contract *common.Ad
 	gasLimit, err := s.estimateGasLimit(auth, contract, input, nil, gasTipCap, gasFeeCap, value, minGasLimit)
 	if err != nil {
 		log.Error("estimateDynamicGas estimateGasLimit failure", "error", err)
-		return nil, err
+		if minGasLimit == 0 {
+			return nil, err
+		}
+		gasLimit = minGasLimit
 	}
 	return &FeeData{
 		gasLimit:  gasLimit,
@@ -67,7 +69,7 @@ func (s *Sender) estimateGasLimit(opts *bind.TransactOpts, contract *common.Addr
 	}
 	gasLimit, err := s.client.EstimateGas(s.ctx, msg)
 	if err != nil {
-		log.Error("estimateGasLimit EstimateGas failure", "msg", msg, "error", err)
+		log.Error("estimateGasLimit EstimateGas failure", "error", err)
 		return 0, err
 	}
 	if minGasLimit > gasLimit {

bridge/internal/controller/sender/sender.go

@@ -56,15 +56,6 @@ type FeeData struct {
 	gasLimit uint64
 }
 
-func newEmptyFeeData() *FeeData {
-	return &FeeData{
-		gasFeeCap: big.NewInt(0),
-		gasTipCap: big.NewInt(0),
-		gasPrice:  big.NewInt(0),
-		gasLimit:  0,
-	}
-}
-
 // PendingTransaction submitted but pending transactions
 type PendingTransaction struct {
 	submitAt uint64
@@ -94,8 +85,6 @@ type Sender struct {
 	stopCh chan struct{}
 
 	metrics *senderMetrics
-
-	cachedMaxFeeData *FeeData // hacky way to avoid getFeeData error
 }
 
 // NewSender returns a new instance of transaction sender
@@ -140,20 +129,19 @@ func NewSender(ctx context.Context, config *config.SenderConfig, priv *ecdsa.Pri
 	}
 
 	sender := &Sender{
-		ctx:              ctx,
-		config:           config,
-		client:           client,
-		chainID:          chainID,
-		auth:             auth,
-		minBalance:       config.MinBalance,
-		confirmCh:        make(chan *Confirmation, 128),
-		blockNumber:      header.Number.Uint64(),
-		baseFeePerGas:    baseFeePerGas,
-		pendingTxs:       cmapV2.New[*PendingTransaction](),
-		stopCh:           make(chan struct{}),
-		name:             name,
-		service:          service,
-		cachedMaxFeeData: newEmptyFeeData(),
+		ctx:           ctx,
+		config:        config,
+		client:        client,
+		chainID:       chainID,
+		auth:          auth,
+		minBalance:    config.MinBalance,
+		confirmCh:     make(chan *Confirmation, 128),
+		blockNumber:   header.Number.Uint64(),
+		baseFeePerGas: baseFeePerGas,
+		pendingTxs:    cmapV2.New[*PendingTransaction](),
+		stopCh:        make(chan struct{}),
+		name:          name,
+		service:       service,
 	}
 	sender.metrics = initSenderMetrics(reg)
 
@@ -201,26 +189,6 @@ func (s *Sender) getFeeData(auth *bind.TransactOpts, target *common.Address, val
 	return s.estimateLegacyGas(auth, target, value, data, minGasLimit)
 }
 
-func (s *Sender) cacheMaxFeeData(feeData *FeeData) {
-	if feeData == nil {
-		log.Error("cacheMaxFeeData", "err", "feeData must not be nil")
-		return
-	}
-
-	if feeData.gasFeeCap != nil && feeData.gasFeeCap.Cmp(s.cachedMaxFeeData.gasFeeCap) > 0 {
-		s.cachedMaxFeeData.gasFeeCap = feeData.gasFeeCap
-	}
-	if feeData.gasTipCap != nil && feeData.gasTipCap.Cmp(s.cachedMaxFeeData.gasTipCap) > 0 {
-		s.cachedMaxFeeData.gasTipCap = feeData.gasTipCap
-	}
-	if feeData.gasPrice != nil && feeData.gasPrice.Cmp(s.cachedMaxFeeData.gasPrice) > 0 {
-		s.cachedMaxFeeData.gasPrice = feeData.gasPrice
-	}
-	if feeData.gasLimit > s.cachedMaxFeeData.gasLimit {
-		s.cachedMaxFeeData.gasLimit = feeData.gasLimit
-	}
-}
-
 // SendTransaction send a signed L2tL1 transaction.
 func (s *Sender) SendTransaction(ID string, target *common.Address, value *big.Int, data []byte, minGasLimit uint64) (common.Hash, error) {
 	s.metrics.sendTransactionTotal.WithLabelValues(s.service, s.name).Inc()
@@ -250,12 +218,7 @@ func (s *Sender) SendTransaction(ID string, target *common.Address, value *big.I
 	if feeData, err = s.getFeeData(s.auth, target, value, data, minGasLimit); err != nil {
 		s.metrics.sendTransactionFailureGetFee.WithLabelValues(s.service, s.name).Inc()
 		log.Error("failed to get fee data", "err", err)
-		if s.cachedMaxFeeData.gasLimit == 0 { // if no MaxFeeData cached, and getFeeData fails
-			return common.Hash{}, fmt.Errorf("failed to get fee data for the first time, err: %w", err)
-		}
-		feeData = s.cachedMaxFeeData
-	} else {
-		s.cacheMaxFeeData(feeData)
+		return common.Hash{}, fmt.Errorf("failed to get fee data, err: %w", err)
 	}
 
 	if tx, err = s.createAndSendTx(s.auth, feeData, target, value, data, nil); err != nil {
@@ -448,7 +411,6 @@ func (s *Sender) resubmitTransaction(feeData *FeeData, auth *bind.TransactOpts,
 	}
 
 	log.Debug("Transaction gas adjustment details", txInfo)
-	s.cacheMaxFeeData(feeData)
 
 	nonce := tx.Nonce()
 	s.metrics.resubmitTransactionTotal.WithLabelValues(s.service, s.name).Inc()

common/version/prover_version.go

@@ -0,0 +1,55 @@
+package version
+
+import (
+	"strconv"
+	"strings"
+)
+
+// CheckScrollProverVersion check the "scroll-prover" version, if it's different from the local one, return false
+func CheckScrollProverVersion(proverVersion string) bool {
+	// note the the version is in fact in the format of "tag-commit-scroll_prover-halo2",
+	// so split-by-'-' length should be 4
+	remote := strings.Split(proverVersion, "-")
+	if len(remote) != 4 {
+		return false
+	}
+	local := strings.Split(Version, "-")
+	if len(local) != 4 {
+		return false
+	}
+	// compare the `scroll_prover` version
+	return remote[2] == local[2]
+}
+
+// CheckScrollProverVersionTag check the "scroll-prover" version's tag, if it's too old, return false
+func CheckScrollProverVersionTag(proverVersion string) bool {
+	// note the the version is in fact in the format of "tag-commit-scroll_prover-halo2",
+	// so split-by-'-' length should be 4
+	remote := strings.Split(proverVersion, "-")
+	if len(remote) != 4 {
+		return false
+	}
+	remoteTagNums := strings.Split(strings.TrimPrefix(remote[0], "v"), ".")
+	if len(remoteTagNums) != 3 {
+		return false
+	}
+	remoteTagMajor, err := strconv.Atoi(remoteTagNums[0])
+	if err != nil {
+		return false
+	}
+	remoteTagMinor, err := strconv.Atoi(remoteTagNums[1])
+	if err != nil {
+		return false
+	}
+	remoteTagPatch, err := strconv.Atoi(remoteTagNums[2])
+	if err != nil {
+		return false
+	}
+	if remoteTagMajor < 4 {
+		return false
+	}
+	if remoteTagMinor == 1 && remoteTagPatch < 98 {
+		return false
+	}
+	return true
+}

common/version/version.go

@@ -3,11 +3,9 @@ package version
 import (
 	"fmt"
 	"runtime/debug"
-	"strconv"
-	"strings"
 )
 
-var tag = "v4.2.12"
+var tag = "v4.2.14"
 
 var commit = func() string {
 	if info, ok := debug.ReadBuildInfo(); ok {
@@ -31,52 +29,3 @@ var ZkVersion = "000000-000000"
 
 // Version denote the version of scroll protocol, including the l2geth, relayer, coordinator, prover, contracts and etc.
 var Version = fmt.Sprintf("%s-%s-%s", tag, commit, ZkVersion)
-
-// CheckScrollProverVersion check the "scroll-prover" version, if it's different from the local one, return false
-func CheckScrollProverVersion(proverVersion string) bool {
-	// note the the version is in fact in the format of "tag-commit-scroll_prover-halo2",
-	// so split-by-'-' length should be 4
-	remote := strings.Split(proverVersion, "-")
-	if len(remote) != 4 {
-		return false
-	}
-	local := strings.Split(Version, "-")
-	if len(local) != 4 {
-		return false
-	}
-	// compare the `scroll_prover` version
-	return remote[2] == local[2]
-}
-
-// CheckScrollProverVersionTag check the "scroll-prover" version's tag, if it's too old, return false
-func CheckScrollProverVersionTag(proverVersion string) bool {
-	// note the the version is in fact in the format of "tag-commit-scroll_prover-halo2",
-	// so split-by-'-' length should be 4
-	remote := strings.Split(proverVersion, "-")
-	if len(remote) != 4 {
-		return false
-	}
-	remoteTagNums := strings.Split(strings.TrimPrefix(remote[0], "v"), ".")
-	if len(remoteTagNums) != 3 {
-		return false
-	}
-	remoteTagMajor, err := strconv.Atoi(remoteTagNums[0])
-	if err != nil {
-		return false
-	}
-	remoteTagMinor, err := strconv.Atoi(remoteTagNums[1])
-	if err != nil {
-		return false
-	}
-	remoteTagPatch, err := strconv.Atoi(remoteTagNums[2])
-	if err != nil {
-		return false
-	}
-	if remoteTagMajor < 4 {
-		return false
-	}
-	if remoteTagMinor == 1 && remoteTagPatch < 98 {
-		return false
-	}
-	return true
-}

contracts/hardhat.config.ts

@@ -84,6 +84,9 @@ const config: HardhatUserConfig = {
   etherscan: {
     apiKey: process.env.ETHERSCAN_API_KEY,
   },
+  mocha: {
+    timeout: 10000000,
+  },
   dodoc: {
     runOnCompile: true,
     keepFileStructure: false,

contracts/integration-test/ScrollChain.spec.ts

@@ -1,5 +1,6 @@
 /* eslint-disable node/no-unpublished-import */
 /* eslint-disable node/no-missing-import */
+import { concat } from "ethers/lib/utils";
 import { constants } from "ethers";
 import { ethers } from "hardhat";
 import { ScrollChain, L1MessageQueue } from "../typechain";
@@ -11,22 +12,27 @@ describe("ScrollChain", async () => {
   beforeEach(async () => {
     const [deployer] = await ethers.getSigners();
 
+    const ProxyAdmin = await ethers.getContractFactory("ProxyAdmin", deployer);
+    const admin = await ProxyAdmin.deploy();
+    await admin.deployed();
+
+    const TransparentUpgradeableProxy = await ethers.getContractFactory("TransparentUpgradeableProxy", deployer);
+
     const L1MessageQueue = await ethers.getContractFactory("L1MessageQueue", deployer);
-    queue = await L1MessageQueue.deploy();
-    await queue.deployed();
+    const queueImpl = await L1MessageQueue.deploy();
+    await queueImpl.deployed();
+    const queueProxy = await TransparentUpgradeableProxy.deploy(queueImpl.address, admin.address, "0x");
+    await queueProxy.deployed();
+    queue = await ethers.getContractAt("L1MessageQueue", queueProxy.address, deployer);
 
-    const RollupVerifier = await ethers.getContractFactory("RollupVerifier", deployer);
-    const verifier = await RollupVerifier.deploy();
-    await verifier.deployed();
+    const ScrollChain = await ethers.getContractFactory("ScrollChain", deployer);
+    const chainImpl = await ScrollChain.deploy(0);
+    await chainImpl.deployed();
+    const chainProxy = await TransparentUpgradeableProxy.deploy(chainImpl.address, admin.address, "0x");
+    await chainProxy.deployed();
+    chain = await ethers.getContractAt("ScrollChain", chainProxy.address, deployer);
 
-    const ScrollChain = await ethers.getContractFactory("ScrollChain", {
-      signer: deployer,
-      libraries: { RollupVerifier: verifier.address },
-    });
-    chain = await ScrollChain.deploy(0);
-    await chain.deployed();
-
-    await chain.initialize(queue.address, constants.AddressZero, 44);
+    await chain.initialize(queue.address, constants.AddressZero, 100);
     await chain.addSequencer(deployer.address);
     await queue.initialize(
       constants.AddressZero,
@@ -38,79 +44,54 @@ describe("ScrollChain", async () => {
   });
 
   // @note skip this benchmark tests
-  /*
-  it("should succeed", async () => {
-    await chain.importGenesisBatch({
-      blocks: [
-        {
-          blockHash: "0x92826bd3aad2ef70d8061dc4e25150b305d1233d9cd7579433a77d6eb01dae1c",
-          parentHash: constants.HashZero,
-          blockNumber: 0,
-          timestamp: 1639724192,
-          baseFee: 1000000000,
-          gasLimit: 940000000,
-          numTransactions: 0,
-          numL1Messages: 0,
-        },
-      ],
-      prevStateRoot: constants.HashZero,
-      newStateRoot: "0x1b186a7a90ec3b41a2417062fe44dce8ce82ae76bfbb09eae786a4f1be1895f5",
-      withdrawTrieRoot: constants.HashZero,
-      batchIndex: 0,
-      parentBatchHash: constants.HashZero,
-      l2Transactions: [],
-    });
-    const parentBatchHash = await chain.lastFinalizedBatchHash();
+  it.skip("should succeed", async () => {
+    const batchHeader0 = new Uint8Array(89);
+    batchHeader0[25] = 1;
+    await chain.importGenesisBatch(batchHeader0, "0x0000000000000000000000000000000000000000000000000000000000000001");
+    const parentBatchHash = await chain.committedBatches(0);
     console.log("genesis batch hash:", parentBatchHash);
+    console.log(`ChunkPerBatch`, `BlockPerChunk`, `TxPerBlock`, `BytesPerTx`, `TotalBytes`, `EstimateGas`);
+    for (let numChunks = 3; numChunks <= 6; ++numChunks) {
+      for (let numBlocks = 1; numBlocks <= 5; ++numBlocks) {
+        for (let numTx = 20; numTx <= Math.min(30, 100 / numBlocks); ++numTx) {
+          for (let txLength = 800; txLength <= 1000; txLength += 100) {
+            const txs: Array<Uint8Array> = [];
+            for (let i = 0; i < numTx; i++) {
+              const tx = new Uint8Array(4 + txLength);
+              let offset = 3;
+              for (let x = txLength; x > 0; x = Math.floor(x / 256)) {
+                tx[offset] = x % 256;
+                offset -= 1;
+              }
+              tx.fill(1, 4);
+              txs.push(tx);
+            }
+            const chunk = new Uint8Array(1 + 60 * numBlocks);
+            chunk[0] = numBlocks;
+            for (let i = 0; i < numBlocks; i++) {
+              chunk[1 + i * 60 + 57] = numTx;
+            }
+            const chunks: Array<Uint8Array> = [];
+            for (let i = 0; i < numChunks; i++) {
+              const txsInChunk: Array<Uint8Array> = [];
+              for (let j = 0; j < numBlocks; j++) {
+                txsInChunk.push(concat(txs));
+              }
+              chunks.push(concat([chunk, concat(txsInChunk)]));
+            }
 
-    for (let numTx = 1; numTx <= 25; ++numTx) {
-      for (let txLength = 100; txLength <= 1000; txLength += 100) {
-        const txs: Array<Uint8Array> = [];
-        for (let i = 0; i < numTx; i++) {
-          const tx = new Uint8Array(4 + txLength);
-          let offset = 3;
-          for (let x = txLength; x > 0; x = Math.floor(x / 256)) {
-            tx[offset] = x % 256;
-            offset -= 1;
+            const estimateGas = await chain.estimateGas.commitBatch(0, batchHeader0, chunks, "0x");
+            console.log(
+              `${numChunks}`,
+              `${numBlocks}`,
+              `${numTx}`,
+              `${txLength}`,
+              `${numChunks * numBlocks * numTx * (txLength + 1)}`,
+              `${estimateGas.toString()}`
+            );
           }
-          tx.fill(1, 4);
-          txs.push(tx);
         }
-        const batch = {
-          blocks: [
-            {
-              blockHash: "0xb5baa665b2664c3bfed7eb46e00ebc110ecf2ebd257854a9bf2b9dbc9b2c08f6",
-              parentHash: "0x92826bd3aad2ef70d8061dc4e25150b305d1233d9cd7579433a77d6eb01dae1c",
-              blockNumber: 1,
-              timestamp: numTx * 100000 + txLength,
-              baseFee: 0,
-              gasLimit: 0,
-              numTransactions: 0,
-              numL1Messages: 0,
-            },
-          ],
-          prevStateRoot: "0x1b186a7a90ec3b41a2417062fe44dce8ce82ae76bfbb09eae786a4f1be1895f5",
-          newStateRoot: "0xb5baa665b2664c3bfed7eb46e00ebc110ecf2ebd257854a9bf2b9dbc9b2c08f6",
-          withdrawTrieRoot: "0xb5baa665b2664c3bfed7eb46e00ebc110ecf2ebd257854a9bf2b9dbc9b2c08f6",
-          batchIndex: 1,
-          parentBatchHash: parentBatchHash,
-          l2Transactions: concat(txs),
-        };
-        const estimateGas = await chain.estimateGas.commitBatch(batch);
-        const tx = await chain.commitBatch(batch, { gasLimit: estimateGas.mul(12).div(10) });
-        const receipt = await tx.wait();
-        console.log(
-          "Commit batch with l2TransactionsBytes:",
-          numTx * (txLength + 4),
-          "gasLimit:",
-          tx.gasLimit.toString(),
-          "estimateGas:",
-          estimateGas.toString(),
-          "gasUsed:",
-          receipt.gasUsed.toString()
-        );
       }
     }
   });
-  */
 });