update test stuffs

rollup/tests/integration_tool/block_fetching.go

@@ -0,0 +1,83 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"math/big"
+
+	"github.com/scroll-tech/da-codec/encoding"
+	"github.com/scroll-tech/go-ethereum/common"
+	"github.com/scroll-tech/go-ethereum/core/types"
+	"github.com/scroll-tech/go-ethereum/ethclient"
+	"github.com/scroll-tech/go-ethereum/log"
+	"github.com/scroll-tech/go-ethereum/rpc"
+)
+
+func fetchAndStoreBlocks(ctx context.Context, from, to uint64) ([]*encoding.Block, error) {
+	validiumMode := cfg.ValidiumMode
+	cfg := cfg.FetchConfig
+	client, err := rpc.Dial(cfg.Endpoint)
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect l2 geth, endpoint %s, err %v", cfg.Endpoint, err)
+	}
+
+	ethCli := ethclient.NewClient(client)
+	var blocks []*encoding.Block
+	for number := from; number <= to; number++ {
+		log.Debug("retrieving block", "height", number)
+		block, err := ethCli.BlockByNumber(ctx, new(big.Int).SetUint64(number))
+		if err != nil {
+			return nil, fmt.Errorf("failed to BlockByNumber: %v. number: %v", err, number)
+		}
+
+		blockTxs := block.Transactions()
+
+		var count int
+		for _, tx := range blockTxs {
+			if tx.IsL1MessageTx() {
+				count++
+			}
+		}
+		log.Info("retrieved block", "height", block.Header().Number, "hash", block.Header().Hash().String(), "L1 message count", count)
+
+		// use original (encrypted) L1 message txs in validium mode
+		if validiumMode {
+			var txs []*types.Transaction
+
+			if count > 0 {
+				log.Info("Fetching encrypted messages in validium mode")
+				err = client.CallContext(ctx, &txs, "scroll_getL1MessagesInBlock", block.Hash(), "synced")
+				if err != nil {
+					return nil, fmt.Errorf("failed to get L1 messages: %v, block hash: %v", err, block.Hash().Hex())
+				}
+			}
+
+			// sanity check
+			if len(txs) != count {
+				return nil, fmt.Errorf("L1 message count mismatch: expected %d, got %d", count, len(txs))
+			}
+
+			for ii := 0; ii < count; ii++ {
+				// sanity check
+				if blockTxs[ii].AsL1MessageTx().QueueIndex != txs[ii].AsL1MessageTx().QueueIndex {
+					return nil, fmt.Errorf("L1 message queue index mismatch at index %d: expected %d, got %d", ii, blockTxs[ii].AsL1MessageTx().QueueIndex, txs[ii].AsL1MessageTx().QueueIndex)
+				}
+
+				log.Info("Replacing L1 message tx in validium mode", "index", ii, "queueIndex", txs[ii].AsL1MessageTx().QueueIndex, "decryptedTxHash", blockTxs[ii].Hash().Hex(), "originalTxHash", txs[ii].Hash().Hex())
+				blockTxs[ii] = txs[ii]
+			}
+		}
+
+		withdrawRoot, err3 := ethCli.StorageAt(ctx, cfg.L2MessageQueueAddress, cfg.WithdrawTrieRootSlot, big.NewInt(int64(number)))
+		if err3 != nil {
+			return nil, fmt.Errorf("failed to get withdrawRoot: %v. number: %v", err3, number)
+		}
+		blocks = append(blocks, &encoding.Block{
+			Header:       block.Header(),
+			Transactions: encoding.TxsToTxsData(blockTxs),
+			WithdrawRoot: common.BytesToHash(withdrawRoot),
+		})
+	}
+
+	return blocks, nil
+}

rollup/tests/integration_tool/imports.go

@@ -42,13 +42,21 @@ func randomPickKfromN(n, k int, rng *rand.Rand) []int {
 	return ret
 }
 
-func importData(ctx context.Context, beginBlk, endBlk uint64, chkNum, batchNum, bundleNum int, seed int64) (*importRecord, error) {
+func importData(ctx context.Context, beginBlk, endBlk uint64, blocks []*encoding.Block, chkNum, batchNum, bundleNum int, seed int64) (*importRecord, error) {
 
 	db, err := database.InitDB(cfg.DBConfig)
 	if err != nil {
 		return nil, err
 	}
 
+	if len(blocks) > 0 {
+		log.Info("import block")
+		blockOrm := orm.NewL2Block(db)
+		if err := blockOrm.InsertL2Blocks(ctx, blocks); err != nil {
+			return nil, err
+		}
+	}
+
 	ret := &importRecord{}
 	// Create a new random source with the provided seed
 	source := rand.NewSource(seed)

rollup/tests/integration_tool/main.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"github.com/scroll-tech/da-codec/encoding"
+	"github.com/scroll-tech/go-ethereum/common"
 	"github.com/scroll-tech/go-ethereum/log"
 	"github.com/urfave/cli/v2"
 
@@ -40,12 +41,6 @@ var seedFlag = cli.Int64Flag{
 	Value: 0,
 }
 
-var codecFlag = cli.IntFlag{
-	Name:  "codec",
-	Usage: "codec version, valid from 6, default(auto) is 0",
-	Value: 0,
-}
-
 func parseThreeIntegers(value string) (int, int, int, error) {
 	// Split the input string by comma
 	parts := strings.Split(value, ",")
@@ -84,10 +79,21 @@ func parseThreeIntegers(value string) (int, int, int, error) {
 	return values[0], values[1], values[2], nil
 }
 
+type fetchConfig struct {
+	// node url.
+	Endpoint string `json:"endpoint"`
+	// The L2MessageQueue contract address deployed on layer 2 chain.
+	L2MessageQueueAddress common.Address `json:"l2_message_queue_address"`
+	// The WithdrawTrieRootSlot in L2MessageQueue contract.
+	WithdrawTrieRootSlot common.Hash `json:"withdraw_trie_root_slot,omitempty"`
+}
+
 // load a comptabile type of config for rollup
 type config struct {
 	DBConfig     *database.Config `json:"db_config"`
+	FetchConfig  *fetchConfig     `json:"fetch_config,omitempty"`
 	ValidiumMode bool             `json:"validium_mode"`
+	CodecVersion int              `json:"codec_version"`
 }
 
 func init() {
@@ -97,7 +103,7 @@ func init() {
 	app.Name = "integration-test-tool"
 	app.Usage = "The Scroll L2 Integration Test Tool"
 	app.Version = version.Version
-	app.Flags = append(app.Flags, &codecFlag, &seedFlag, &outputNumFlag, &outputPathFlag)
+	app.Flags = append(app.Flags, &seedFlag, &outputNumFlag, &outputPathFlag)
 	app.Flags = append(app.Flags, utils.CommonFlags...)
 	app.Before = func(ctx *cli.Context) error {
 		if err := utils.LogSetup(ctx); err != nil {
@@ -135,9 +141,8 @@ func action(ctx *cli.Context) error {
 		return fmt.Errorf("specify begin and end block number")
 	}
 
-	codecFl := ctx.Int(codecFlag.Name)
-	if codecFl != 0 {
-		switch codecFl {
+	if cfg.CodecVersion != 0 {
+		switch cfg.CodecVersion {
 		case 6:
 			codecCfg = encoding.CodecV6
 		case 7:
@@ -147,7 +152,7 @@ func action(ctx *cli.Context) error {
 		case 9:
 			codecCfg = encoding.CodecV9
 		default:
-			return fmt.Errorf("invalid codec version %d", codecFl)
+			return fmt.Errorf("invalid codec version %d", cfg.CodecVersion)
 		}
 		log.Info("set codec", "version", codecCfg)
 	}
@@ -161,6 +166,14 @@ func action(ctx *cli.Context) error {
 		return fmt.Errorf("invalid begin block number: %w", err)
 	}
 
+	var import_blocks []*encoding.Block
+	if cfg.FetchConfig != nil {
+		import_blocks, err = fetchAndStoreBlocks(ctx.Context, beginBlk, endBlk)
+		if err != nil {
+			return err
+		}
+	}
+
 	chkNum, batchNum, bundleNum, err := parseThreeIntegers(ctx.String(outputNumFlag.Name))
 	if err != nil {
 		return err
@@ -174,7 +187,7 @@ func action(ctx *cli.Context) error {
 
 	outputPath := ctx.String(outputPathFlag.Name)
 	log.Info("output", "Seed", seed, "file", outputPath)
-	ret, err := importData(ctx.Context, beginBlk, endBlk, chkNum, batchNum, bundleNum, seed)
+	ret, err := importData(ctx.Context, beginBlk, endBlk, import_blocks, chkNum, batchNum, bundleNum, seed)
 	if err != nil {
 		return err
 	}

tests/prover-e2e/Makefile

@@ -9,6 +9,9 @@ ifndef END_BLOCK
 $(error END_BLOCK is not set. Define it in .make.env or pass END_BLOCK=<end_block>)
 endif
 
+BLOCK_PRE_MIGRATIONS := $(wildcard conf/*.sql)
+.OPTIONAL: $(BLOCK_PRE_MIGRATIONS)
+
 all: setup_db test_tool import_data
 
 clean:
@@ -25,6 +28,11 @@ check_vars: | conf
         exit 1; \
     fi
 
+migration_blocks: $(BLOCK_PRE_MIGRATIONS)
+ifneq ($(strip $(BLOCK_PRE_MIGRATIONS)),)
+	GOOSE_MIGRATION_DIR=conf ${GOOSE_CMD} up-to 100
+endif
+
 setup_db: clean
 	docker compose up --detach
 	@echo "Waiting for PostgreSQL to be ready..."
@@ -42,30 +50,18 @@ setup_db: clean
         fi; \
     done	
 	${GOOSE_CMD} up
-	GOOSE_MIGRATION_DIR=conf ${GOOSE_CMD} up-to 100
 
 reset_db:
-	GOOSE_MIGRATION_DIR=conf ${GOOSE_CMD} down
 	${GOOSE_CMD} down-to 0
 	${GOOSE_CMD} up
-	GOOSE_MIGRATION_DIR=conf ${GOOSE_CMD} up-to 100
 
 test_tool:
 	go build -o $(PWD)/build/bin/e2e_tool ../../rollup/tests/integration_tool
 
 build/bin/e2e_tool: test_tool
 
-import_data_euclid: build/bin/e2e_tool check_vars
-	build/bin/e2e_tool --config conf/config.json --codec 7 ${BEGIN_BLOCK} ${END_BLOCK}
-
-import_data_feynman: build/bin/e2e_tool check_vars
-	build/bin/e2e_tool --config conf/config.json --codec 8 ${BEGIN_BLOCK} ${END_BLOCK}
-
-import_data_galileo: build/bin/e2e_tool check_vars
-	build/bin/e2e_tool --config conf/config.json --codec 9 ${BEGIN_BLOCK} ${END_BLOCK}
-
-import_data: build/bin/e2e_tool check_vars
-	build/bin/e2e_tool --config conf/config.json --codec ${CODEC_VERSION} ${BEGIN_BLOCK} ${END_BLOCK}
+import_data: build/bin/e2e_tool check_vars migration_blocks
+	build/bin/e2e_tool --config conf/config.json ${BEGIN_BLOCK} ${END_BLOCK}
 
 reimport_data: reset_db import_data
 

tests/prover-e2e/cloak-xen/.make.env

@@ -1,4 +1,3 @@
 BEGIN_BLOCK?=35
 END_BLOCK?=49
-CODEC_VERSION?=8
 SCROLL_FORK_NAME=feynman

tests/prover-e2e/cloak-xen/config.json

@@ -5,5 +5,6 @@
     "maxOpenNum": 5,
     "maxIdleNum": 1
   },
-  "validium_mode": true
+  "validium_mode": true,
+  "codec_version": 8
 }

tests/prover-e2e/sepolia-feynman/.make.env

@@ -1,4 +1,3 @@
 BEGIN_BLOCK?=10973711
 END_BLOCK?=10973721
-CODEC_VERSION?=8
 SCROLL_FORK_NAME=feynman

tests/prover-e2e/sepolia-feynman/config.json

@@ -5,5 +5,6 @@
     "maxOpenNum": 5,
     "maxIdleNum": 1
   },
-  "validium_mode": false
+  "validium_mode": false,
+  "codec_version": 8
 }

tests/prover-e2e/sepolia-galileo/.make.env

@@ -1,4 +1,3 @@
 BEGIN_BLOCK?=20278022
 END_BLOCK?=20278025
-CODEC_VERSION?=9
 SCROLL_FORK_NAME=galileo

tests/prover-e2e/sepolia-galileo/config.json

@@ -5,5 +5,6 @@
     "maxOpenNum": 5,
     "maxIdleNum": 1
   },
-  "validium_mode": false
+  "validium_mode": false,
+  "codec_version": 9
 }

tests/prover-e2e/sepolia-galileoV2/.make.env

@@ -0,0 +1,4 @@
+BEGIN_BLOCK?=20239245
+END_BLOCK?=20239250
+CODEC_VERSION?=9
+SCROLL_FORK_NAME=galileo

tests/prover-e2e/sepolia-galileoV2/config.json

@@ -0,0 +1,15 @@
+{
+    "db_config": {
+    "driver_name": "postgres",
+    "dsn": "postgres://dev:dev@localhost:5432/scroll?sslmode=disable",
+    "maxOpenNum": 5,
+    "maxIdleNum": 1
+  },
+  "fetch_config": {
+    "endpoint": "http://l2-sequencer-galileo-6.devnet.scroll.tech:8545",
+    "l2_message_queue_address": "0x5300000000000000000000000000000000000000"
+  },
+  "validium_mode": false,
+  "codec_version": 9
+
+}

tests/prover-e2e/sepolia-galileoV2/config.template.json

@@ -0,0 +1,40 @@
+{
+  "prover_manager": {
+    "provers_per_session": 1,
+    "session_attempts": 5,
+    "external_prover_threshold": 32,
+    "bundle_collection_time_sec": 180,
+    "batch_collection_time_sec": 180,
+    "chunk_collection_time_sec": 180,
+    "verifier": {
+      "min_prover_version": "v4.4.33",
+      "verifiers": [
+        {
+          "assets_path": "assets",
+          "fork_name": "galileo"
+        }
+      ]
+    }
+  },
+  "db": {
+    "driver_name": "postgres",
+    "dsn": "postgres://dev:dev@localhost/scroll?sslmode=disable",
+    "maxOpenNum": 200,
+    "maxIdleNum": 20
+  },
+  "l2": {
+    "validium_mode": false,
+    "chain_id": 534351,
+    "l2geth": {
+      "endpoint": "<serach a public rpc endpoint like alchemy>"
+    }
+  },
+  "auth": {
+    "secret": "prover secret key",
+    "challenge_expire_duration_sec": 3600,
+    "login_expire_duration_sec": 3600
+  },
+  "sequencer": {
+    "decryption_key": "not need"
+  }
+}

zkvm-prover/Makefile

@@ -62,4 +62,6 @@ test_run:
 test_e2e_run: ${E2E_HANDLE_SET}
 	GO_TAG=${GO_TAG} GIT_REV=${GIT_REV} ZK_VERSION=${ZK_VERSION} cargo run --release -p prover -- --config ./config.json handle ${E2E_HANDLE_SET}
 
+test_e2e_run_gpu: ${E2E_HANDLE_SET}
+	GO_TAG=${GO_TAG} GIT_REV=${GIT_REV} ZK_VERSION=${ZK_VERSION} cargo run --release --features cuda -p prover -- --config ./config.json handle ${E2E_HANDLE_SET}