fix postgres bulk insert issue

bridge-history-api/internal/logic/history_logic.go

@@ -372,7 +372,7 @@ func (h *HistoryLogic) getCachedTxsInfo(ctx context.Context, cacheKey string, pa
 	}
 
 	if start >= total {
-		return nil, 0, false, nil
+		return nil, 0, true, nil
 	}
 
 	values, err := h.redis.ZRevRange(ctx, cacheKey, start, end).Result()

common/database/db.go

@@ -51,7 +51,8 @@ func InitDB(config *Config) (*gorm.DB, error) {
 	}
 
 	db, err := gorm.Open(postgres.Open(config.DSN), &gorm.Config{
-		Logger: &tmpGormLogger,
+		CreateBatchSize: 1000,
+		Logger:          &tmpGormLogger,
 		NowFunc: func() time.Time {
 			// why set time to UTC.
 			// if now set this, the inserted data time will use local timezone. like 2023-07-18 18:24:00 CST+8

scripts/derive_galileo_gas_parameter/.env.example

@@ -1,3 +0,0 @@
-SCROLL_URL=https://rpc.scroll.io
-MAINNET_URL=https://eth-mainnet.g.alchemy.com/v2/YOUR_API_KEY
-BEACON_URL=https://eth-mainnet-beacon.g.alchemy.com/v2/YOUR_API_KEY

scripts/derive_galileo_gas_parameter/.gitignore

@@ -1,41 +0,0 @@
-# Environment variables (contains API keys)
-.env
-
-# Cached data files
-*.pkl
-
-# Python
-__pycache__/
-*.py[cod]
-*$py.class
-*.so
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-
-# Virtual environments
-.venv
-venv/
-ENV/
-
-# IDE
-.idea/
-.vscode/
-*.swp
-*.swo
-
-# Logs
-*.log

scripts/derive_galileo_gas_parameter/Pipfile

@@ -1,18 +0,0 @@
-[[source]]
-url = "https://pypi.org/simple"
-verify_ssl = true
-name = "pypi"
-
-[packages]
-web3 = "<7,>=6"
-pandas = "*"
-numpy = "*"
-rlp = "*"
-zstandard = "*"
-requests = "*"
-async_timeout = "*"
-
-[dev-packages]
-
-[requires]
-python_version = "3.10"

scripts/derive_galileo_gas_parameter/README.md

@@ -1,54 +0,0 @@
-# Galileo Gas Parameter Derivation
-
-Derives L1 fee parameters (`commit_scalar`, `blob_scalar`, `penalty_multiplier`) for Scroll's Galileo upgrade by analyzing historical on-chain data.
-
-## Prerequisites
-
-- Python 3.10+
-- pipenv
-- gcc and python3-devel
-
-```bash
-# Ubuntu/Debian
-sudo apt install python3-dev gcc
-# Fedora/RHEL
-sudo dnf install python3-devel gcc
-```
-
-## Installation
-
-```bash
-cd scripts/derive_galileo_gas_parameter
-pipenv install
-```
-
-## Running
-
-### 1. Create `.env` file
-
-```bash
-SCROLL_URL=https://rpc.scroll.io
-MAINNET_URL=https://eth-mainnet.g.alchemy.com/v2/YOUR_API_KEY
-BEACON_URL=https://eth-mainnet-beacon.g.alchemy.com/v2/YOUR_API_KEY
-```
-
-### 2. Usage
-
-```bash
-# Collect data from 30 recent batches
-pipenv run python -u derive_galileo_gas_parameter.py --mode collect --n-batches 30
-
-# Load previously cached data
-pipenv run python -u derive_galileo_gas_parameter.py --mode load --start-batch 494041 --end-batch 494070
-```
-
-**Options:**
-- `--mode {collect,load}` - Collect new data or load from cache (required)
-- `--n-batches N` - Number of batches to collect (default: 30)
-- `--start-batch N` / `--end-batch N` - Batch range for load mode
-- `--target-penalty FLOAT` - Target penalty at P95 (default: 0.1 = 10%)
-- `--penalty-multiplier FLOAT` - Use fixed penalty multiplier instead of calculating
-
-### 3. Cached Data
-
-Collected data is saved to `galileo_data_batch_{start}_{end}.pkl` for re-analysis without re-fetching from RPC. Use `--mode load` to reload.

scripts/derive_galileo_gas_parameter/claude.md

@@ -1,70 +0,0 @@
-# Project Description
-
-This is a Scroll L2 blockchain gas fee parameter calculation project for deriving Scroll gas fee parameters.
-
-# Environment Setup
-
-Claude is executed within a pipenv shell environment, so all packages should already be installed.
-
-# Guidelines
-
-- All Python script comments must be in English
-- When executing `derive_galileo_gas_parameter.py`, output should be redirected to a fixed log file
-  - Use the `-u` option to disable output buffering for real-time logging
-  - Always use `/tmp/derive_galileo_gas_parameter.log` as the log file
-  - Example: `python -u derive_galileo_gas_parameter.py --mode collect --n-batches 5 > /tmp/derive_galileo_gas_parameter.log 2>&1 &`
-
-# Python Script Execution
-
-## Command Format
-
-```bash
-python -u derive_galileo_gas_parameter.py [OPTIONS] > /tmp/derive_galileo_gas_parameter.log 2>&1 &
-```
-
-## Options
-
-- `--mode {collect,load}` (required): Operation mode
-  - `collect`: Collect new data from blockchain
-  - `load`: Load data from cached pickle file
-
-- `--n-batches N`: Number of batches to collect (default: 30)
-  - Only used in `collect` mode
-
-- `--start-batch N`: Start batch index (required for `load` mode)
-
-- `--end-batch N`: End batch index (required for `load` mode)
-
-- `--target-penalty FLOAT`: Target penalty at P95 (default: 0.1 = 10%)
-
-- `--penalty-multiplier FLOAT`: Fixed penalty multiplier (optional, will be calculated from P95 if not specified)
-
-## Examples
-
-Collect data for 3 batches:
-```bash
-python -u derive_galileo_gas_parameter.py --mode collect --n-batches 3 > /tmp/derive_galileo_gas_parameter.log 2>&1 &
-```
-
-Load cached data:
-```bash
-python -u derive_galileo_gas_parameter.py --mode load --start-batch 12345 --end-batch 12347 > /tmp/derive_galileo_gas_parameter.log 2>&1 &
-```
-
-Collect with custom target penalty:
-```bash
-python -u derive_galileo_gas_parameter.py --mode collect --n-batches 30 --target-penalty 0.15 > /tmp/derive_galileo_gas_parameter.log 2>&1 &
-```
-
-## Monitoring Execution
-
-Check log output in real-time:
-```bash
-tail -f /tmp/derive_galileo_gas_parameter.log
-```
-
-View complete log:
-```bash
-cat /tmp/derive_galileo_gas_parameter.log
-```
-

scripts/derive_galileo_gas_parameter/show_batches.py

@@ -1,81 +0,0 @@
-#!/usr/bin/env python3
-"""
-Show batch IDs in a saved data file
-"""
-
-import pickle
-import sys
-from pathlib import Path
-
-def show_batches(filename):
-    """Show batch IDs in the data file"""
-
-    if not Path(filename).exists():
-        print(f"Error: File not found: {filename}")
-        return
-
-    try:
-        with open(filename, 'rb') as f:
-            data = pickle.load(f)
-
-        batch_df = data['batch_df']
-        tx_df = data['tx_df']
-
-        print(f"\n{'='*60}")
-        print(f"File: {filename}")
-        print(f"{'='*60}")
-        print(f"\nBatch range: {data['start_batch']} - {data['end_batch']}")
-        print(f"Total batches: {len(batch_df)}")
-        print(f"Total transactions: {len(tx_df)}")
-
-        print(f"\nBatch IDs:")
-        batch_ids = sorted(batch_df.index.tolist())
-
-        # Print batch IDs in a compact format
-        for i in range(0, len(batch_ids), 10):
-            batch_subset = batch_ids[i:i+10]
-            print(f"  {', '.join(map(str, batch_subset))}")
-
-        # Show detailed information for each batch
-        print(f"\n{'='*60}")
-        print("BATCH DETAILS")
-        print(f"{'='*60}")
-
-        for batch_id in sorted(batch_df.index.tolist()):
-            batch_row = batch_df.loc[batch_id]
-            print(f"\n{'─'*60}")
-            print(f"Batch {int(batch_id)}")
-            print(f"{'─'*60}")
-            print(f"  versioned_hash: {batch_row['versioned_hash']}")
-            print(f"  commit_cost: {batch_row['commit_cost'] / 1e9:.9f} gwei")
-            print(f"  blob_cost: {batch_row['blob_cost'] / 1e9:.9f} gwei")
-            print(f"  finalize_cost: {batch_row['finalize_cost'] / 1e9:.9f} gwei")
-
-            # Get transactions for this batch
-            batch_txs = tx_df[tx_df['batch_index'] == batch_id]
-            print(f"\n  Transactions ({len(batch_txs)} total):")
-
-            if len(batch_txs) > 0:
-                for idx, tx in batch_txs.iterrows():
-                    print(f"\n    Transaction {idx}:")
-                    print(f"      hash: {tx['hash']}")
-                    print(f"      block_number: {tx['block_number']}")
-                    print(f"      size: {tx['size']}")
-                    print(f"      compressed_tx_size: {tx['compressed_tx_size']}")
-                    print(f"      l1_base_fee: {tx['l1_base_fee'] / 1e9:.9f} gwei")
-                    print(f"      l1_blob_base_fee: {tx['l1_blob_base_fee'] / 1e9:.9f} gwei")
-            else:
-                print("      (No transactions)")
-
-        print(f"\n{'='*60}\n")
-
-    except Exception as e:
-        print(f"Error reading file: {e}")
-
-if __name__ == "__main__":
-    if len(sys.argv) < 2:
-        print("Usage: python show_batches.py <data_file.pkl>")
-        print("\nExample: python show_batches.py galileo_data_batch_491125_491136.pkl")
-        sys.exit(1)
-
-    show_batches(sys.argv[1])