mirror of
https://github.com/googleapis/genai-toolbox.git
synced 2026-01-08 15:14:00 -05:00
feat: Add MindsDB Source and Tools (#878)
🚀 Add MindsDB Integration: Expand Toolbox to Hundreds of Datasources Overview This PR introduces comprehensive MindsDB integration to the Google GenAI Toolbox, enabling SQL queries across hundreds of datasources through a unified interface. MindsDB is the most widely adopted AI federated database that automatically translates MySQL queries into REST APIs, GraphQL, and native protocols. 🎯 Key Value for Google GenAI Toolbox Ecosystem 1. Massive Datasource Expansion Before: Toolbox limited to ~15 traditional databases After: Access to hundreds of datasources including Salesforce, Jira, GitHub, MongoDB, Gmail, Slack, and more Impact: Dramatically expands the toolbox's reach and utility for enterprise users 2. Cross-Datasource Analytics New Capability: Perform joins and analytics across different datasources seamlessly Example: Join Salesforce opportunities with GitHub activity to correlate sales with development activity Value: Enables comprehensive data analysis that was previously impossible 3. API Abstraction Layer Innovation: Write standard SQL queries that automatically translate to any API Benefit: Developers can query REST APIs, GraphQL, and native protocols using familiar SQL syntax Impact: Reduces complexity and learning curve for accessing diverse datasources 4. ML Model Integration Enhanced Capability: Use ML models as virtual tables for real-time predictions Example: Query customer churn predictions directly through SQL Value: Brings AI/ML capabilities into the standard SQL workflow 🔧 Technical Implementation Source Layer ✅ New MindsDB source implementation using MySQL wire protocol ✅ Comprehensive test coverage with integration tests ✅ Updated existing MySQL tools to support MindsDB sources ✅ Created dedicated MindsDB tools for enhanced functionality Tools Layer ✅ mindsdb-execute-sql: Direct SQL execution across federated datasources ✅ mindsdb-sql: Parameterized SQL queries with template support ✅ Backward compatibility with existing MySQL tools Documentation & Configuration ✅ Comprehensive documentation with real-world examples ✅ Prebuilt configuration for easy setup ✅ Updated CLI help text and command-line options 📊 Supported Datasources Business Applications Salesforce (leads, opportunities, accounts) Jira (issues, projects, workflows) GitHub (repositories, commits, PRs) Slack (channels, messages, teams) HubSpot (contacts, companies, deals) Databases & Storage MongoDB (NoSQL collections as structured tables) Redis (key-value stores) Elasticsearch (search and analytics) S3, filesystems, etc (file storage) Communication & Email Gmail/Outlook (emails, attachments) Microsoft Teams (communications, files) Discord (server data, messages) 🎯 Example Use Cases Cross-Datasource Analytics -- Join Salesforce opportunities with GitHub activity ``` SELECT s.opportunity_name, s.amount, g.repository_name, COUNT(g.commits) as commit_count FROM salesforce.opportunities s JOIN github.repositories g ON s.account_id = g.owner_id WHERE s.stage = 'Closed Won'; ``` Email & Communication Analysis ``` -- Analyze email patterns with Slack activity SELECT e.sender, e.subject, s.channel_name, COUNT(s.messages) as message_count FROM gmail.emails e JOIN slack.messages s ON e.sender = s.user_name WHERE e.date >= '2024-01-01'; ``` 🚀 Benefits for Google GenAI Toolbox Enterprise Adoption: Enables access to enterprise datasources (Salesforce, Jira, etc.) Developer Productivity: Familiar SQL interface for any datasource AI/ML Integration: Seamless integration of ML models into SQL workflows Scalability: Single interface for hundreds of datasources Competitive Advantage: Unique federated database capabilities in the toolbox ecosystem 📈 Impact Metrics Datasource Coverage: +1000% increase in supported datasources API Abstraction: Eliminates need to learn individual API syntaxes Cross-Platform Analytics: Enables previously impossible data correlations ML Integration: Brings AI capabilities into standard SQL workflows 🔗 Resources MindsDB Documentation MindsDB GitHub Updated Toolbox Documentation ✅ Testing ✅ Unit tests for MindsDB source implementation ✅ Integration tests with real datasource examples ✅ Backward compatibility with existing MySQL tools ✅ Documentation examples tested and verified This integration transforms the Google GenAI Toolbox from a traditional database tool into a comprehensive federated data platform, enabling users to query and analyze data across their entire technology stack through a unified SQL interface. --------- Co-authored-by: duwenxin <duwenxin@google.com> Co-authored-by: setohe0909 <setohe.09@gmail.com> Co-authored-by: Kurtis Van Gent <31518063+kurtisvg@users.noreply.github.com> Co-authored-by: Wenxin Du <117315983+duwenxin99@users.noreply.github.com> Co-authored-by: Yuan Teoh <45984206+Yuan325@users.noreply.github.com>
This commit is contained in:
@@ -556,6 +556,27 @@ steps:
|
||||
"Looker" \
|
||||
looker \
|
||||
looker
|
||||
|
||||
- id: "mindsdb"
|
||||
name: golang:1
|
||||
waitFor: ["compile-test-binary"]
|
||||
entrypoint: /bin/bash
|
||||
env:
|
||||
- "GOPATH=/gopath"
|
||||
- "MINDSDB_PORT=$_MINDSDB_PORT"
|
||||
- "MINDSDB_DATABASE=$_MINDSDB_DATABASE"
|
||||
- "SERVICE_ACCOUNT_EMAIL=$SERVICE_ACCOUNT_EMAIL"
|
||||
secretEnv: ["MINDSDB_HOST", "MINDSDB_USER", "MINDSDB_PASS", "CLIENT_ID"]
|
||||
volumes:
|
||||
- name: "go"
|
||||
path: "/gopath"
|
||||
args:
|
||||
- -c
|
||||
- |
|
||||
.ci/test_with_coverage.sh \
|
||||
"MindsDB" \
|
||||
mindsdb \
|
||||
mindsdb
|
||||
|
||||
- id: "cloud-sql"
|
||||
name: golang:1
|
||||
@@ -821,6 +842,12 @@ availableSecrets:
|
||||
env: OCEANBASE_USER
|
||||
- versionName: projects/$PROJECT_ID/secrets/oceanbase_pass/versions/latest
|
||||
env: OCEANBASE_PASSWORD
|
||||
- versionName: projects/$PROJECT_ID/secrets/mindsdb_host/versions/latest
|
||||
env: MINDSDB_HOST
|
||||
- versionName: projects/$PROJECT_ID/secrets/mindsdb_user/versions/latest
|
||||
env: MINDSDB_USER
|
||||
- versionName: projects/$PROJECT_ID/secrets/mindsdb_pass/versions/latest
|
||||
env: MINDSDB_PASS
|
||||
- versionName: projects/$PROJECT_ID/secrets/yugabytedb_host/versions/latest
|
||||
env: YUGABYTEDB_HOST
|
||||
- versionName: projects/$PROJECT_ID/secrets/yugabytedb_user/versions/latest
|
||||
@@ -888,6 +915,8 @@ substitutions:
|
||||
_TRINO_SCHEMA: "default"
|
||||
_OCEANBASE_PORT: "2883"
|
||||
_OCEANBASE_DATABASE: "oceanbase"
|
||||
_MINDSDB_PORT: "47335"
|
||||
_MINDSDB_DATABASE: "mindsdb_test"
|
||||
_YUGABYTEDB_DATABASE: "yugabyte"
|
||||
_YUGABYTEDB_PORT: "5433"
|
||||
_YUGABYTEDB_LOADBALANCE: "false"
|
||||
|
||||
2
.hugo/package-lock.json
generated
2
.hugo/package-lock.json
generated
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"name": "docs2",
|
||||
"name": ".hugo",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
|
||||
@@ -128,6 +128,8 @@ import (
|
||||
_ "github.com/googleapis/genai-toolbox/internal/tools/looker/lookerqueryurl"
|
||||
_ "github.com/googleapis/genai-toolbox/internal/tools/looker/lookerrunlook"
|
||||
_ "github.com/googleapis/genai-toolbox/internal/tools/looker/lookerupdateprojectfile"
|
||||
_ "github.com/googleapis/genai-toolbox/internal/tools/mindsdb/mindsdbexecutesql"
|
||||
_ "github.com/googleapis/genai-toolbox/internal/tools/mindsdb/mindsdbsql"
|
||||
_ "github.com/googleapis/genai-toolbox/internal/tools/mongodb/mongodbaggregate"
|
||||
_ "github.com/googleapis/genai-toolbox/internal/tools/mongodb/mongodbdeletemany"
|
||||
_ "github.com/googleapis/genai-toolbox/internal/tools/mongodb/mongodbdeleteone"
|
||||
@@ -196,6 +198,7 @@ import (
|
||||
_ "github.com/googleapis/genai-toolbox/internal/sources/firestore"
|
||||
_ "github.com/googleapis/genai-toolbox/internal/sources/http"
|
||||
_ "github.com/googleapis/genai-toolbox/internal/sources/looker"
|
||||
_ "github.com/googleapis/genai-toolbox/internal/sources/mindsdb"
|
||||
_ "github.com/googleapis/genai-toolbox/internal/sources/mongodb"
|
||||
_ "github.com/googleapis/genai-toolbox/internal/sources/mssql"
|
||||
_ "github.com/googleapis/genai-toolbox/internal/sources/mysql"
|
||||
@@ -311,7 +314,6 @@ func NewCommand(opts ...Option) *Command {
|
||||
flags.BoolVar(&cmd.cfg.TelemetryGCP, "telemetry-gcp", false, "Enable exporting directly to Google Cloud Monitoring.")
|
||||
flags.StringVar(&cmd.cfg.TelemetryOTLP, "telemetry-otlp", "", "Enable exporting using OpenTelemetry Protocol (OTLP) to the specified endpoint (e.g. 'http://127.0.0.1:4318')")
|
||||
flags.StringVar(&cmd.cfg.TelemetryServiceName, "telemetry-service-name", "toolbox", "Sets the value of the service.name resource attribute for telemetry data.")
|
||||
|
||||
// Fetch prebuilt tools sources to customize the help description
|
||||
prebuiltHelp := fmt.Sprintf(
|
||||
"Use a prebuilt tool configuration by source type. Cannot be used with --tools-file. Allowed: '%s'.",
|
||||
|
||||
@@ -1248,6 +1248,7 @@ func TestPrebuiltTools(t *testing.T) {
|
||||
postgresconfig, _ := prebuiltconfigs.Get("postgres")
|
||||
spanner_config, _ := prebuiltconfigs.Get("spanner")
|
||||
spannerpg_config, _ := prebuiltconfigs.Get("spanner-postgres")
|
||||
mindsdb_config, _ := prebuiltconfigs.Get("mindsdb")
|
||||
sqlite_config, _ := prebuiltconfigs.Get("sqlite")
|
||||
neo4jconfig, _ := prebuiltconfigs.Get("neo4j")
|
||||
alloydbobsvconfig, _ := prebuiltconfigs.Get("alloydb-postgres-observability")
|
||||
@@ -1327,6 +1328,12 @@ func TestPrebuiltTools(t *testing.T) {
|
||||
t.Setenv("MSSQL_USER", "your_mssql_user")
|
||||
t.Setenv("MSSQL_PASSWORD", "your_mssql_password")
|
||||
|
||||
t.Setenv("MINDSDB_HOST", "localhost")
|
||||
t.Setenv("MINDSDB_PORT", "47334")
|
||||
t.Setenv("MINDSDB_DATABASE", "your_mindsdb_db")
|
||||
t.Setenv("MINDSDB_USER", "your_mindsdb_user")
|
||||
t.Setenv("MINDSDB_PASS", "your_mindsdb_password")
|
||||
|
||||
t.Setenv("LOOKER_BASE_URL", "https://your_company.looker.com")
|
||||
t.Setenv("LOOKER_CLIENT_ID", "your_looker_client_id")
|
||||
t.Setenv("LOOKER_CLIENT_SECRET", "your_looker_client_secret")
|
||||
@@ -1342,6 +1349,7 @@ func TestPrebuiltTools(t *testing.T) {
|
||||
t.Setenv("NEO4J_USERNAME", "your_neo4j_user")
|
||||
t.Setenv("NEO4J_PASSWORD", "your_neo4j_password")
|
||||
|
||||
|
||||
ctx, err := testutils.ContextWithNewLogger()
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
@@ -1552,6 +1560,16 @@ func TestPrebuiltTools(t *testing.T) {
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "mindsdb prebuilt tools",
|
||||
in: mindsdb_config,
|
||||
wantToolset: server.ToolsetConfigs{
|
||||
"mindsdb-tools": tools.ToolsetConfig{
|
||||
Name: "mindsdb-tools",
|
||||
ToolNames: []string{"mindsdb-execute-sql", "mindsdb-sql"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "sqlite prebuilt tools",
|
||||
in: sqlite_config,
|
||||
wantToolset: server.ToolsetConfigs{
|
||||
|
||||
161
docs/en/resources/sources/mindsdb.md
Normal file
161
docs/en/resources/sources/mindsdb.md
Normal file
@@ -0,0 +1,161 @@
|
||||
---
|
||||
title: "MindsDB"
|
||||
type: docs
|
||||
weight: 1
|
||||
description: >
|
||||
MindsDB is the most widely adopted AI federated database that enables SQL queries across hundreds of datasources and ML models.
|
||||
---
|
||||
|
||||
## About
|
||||
|
||||
[MindsDB][mindsdb-docs] is the most widely adopted AI federated database in the world. It allows you to combine information from hundreds of datasources as if they were SQL, supporting joins across datasources and enabling you to query all unstructured data as if it were structured.
|
||||
|
||||
MindsDB translates MySQL queries into whatever API is needed - whether it's REST APIs, GraphQL, or native database protocols. This means you can write standard SQL queries and MindsDB automatically handles the translation to APIs like Salesforce, Jira, GitHub, email systems, MongoDB, and hundreds of other datasources.
|
||||
|
||||
MindsDB also enables you to use ML frameworks to train and use models as virtual tables from the data in those datasources. With MindsDB, the GenAI Toolbox can now expand to hundreds of datasources and leverage all of MindsDB's capabilities on ML and unstructured data.
|
||||
|
||||
**Key Features:**
|
||||
- **Federated Database**: Connect and query hundreds of datasources through a single SQL interface
|
||||
- **Cross-Datasource Joins**: Perform joins across different datasources seamlessly
|
||||
- **API Translation**: Automatically translates MySQL queries into REST APIs, GraphQL, and native protocols
|
||||
- **Unstructured Data Support**: Query unstructured data as if it were structured
|
||||
- **ML as Virtual Tables**: Train and use ML models as virtual tables
|
||||
- **MySQL Wire Protocol**: Compatible with standard MySQL clients and tools
|
||||
|
||||
[mindsdb-docs]: https://docs.mindsdb.com/
|
||||
[mindsdb-github]: https://github.com/mindsdb/mindsdb
|
||||
|
||||
## Supported Datasources
|
||||
|
||||
MindsDB supports hundreds of datasources, including:
|
||||
|
||||
### **Business Applications**
|
||||
- **Salesforce**: Query leads, opportunities, accounts, and custom objects
|
||||
- **Jira**: Access issues, projects, workflows, and team data
|
||||
- **GitHub**: Query repositories, commits, pull requests, and issues
|
||||
- **Slack**: Access channels, messages, and team communications
|
||||
- **HubSpot**: Query contacts, companies, deals, and marketing data
|
||||
|
||||
### **Databases & Storage**
|
||||
- **MongoDB**: Query NoSQL collections as structured tables
|
||||
- **Redis**: Key-value stores and caching layers
|
||||
- **Elasticsearch**: Search and analytics data
|
||||
- **S3/Google Cloud Storage**: File storage and data lakes
|
||||
|
||||
### **Communication & Email**
|
||||
- **Gmail/Outlook**: Query emails, attachments, and metadata
|
||||
- **Slack**: Access workspace data and conversations
|
||||
- **Microsoft Teams**: Team communications and files
|
||||
- **Discord**: Server data and message history
|
||||
|
||||
|
||||
|
||||
## Example Queries
|
||||
|
||||
### Cross-Datasource Analytics
|
||||
```sql
|
||||
-- Join Salesforce opportunities with GitHub activity
|
||||
SELECT
|
||||
s.opportunity_name,
|
||||
s.amount,
|
||||
g.repository_name,
|
||||
COUNT(g.commits) as commit_count
|
||||
FROM salesforce.opportunities s
|
||||
JOIN github.repositories g ON s.account_id = g.owner_id
|
||||
WHERE s.stage = 'Closed Won'
|
||||
GROUP BY s.opportunity_name, s.amount, g.repository_name;
|
||||
```
|
||||
|
||||
### Email & Communication Analysis
|
||||
```sql
|
||||
-- Analyze email patterns with Slack activity
|
||||
SELECT
|
||||
e.sender,
|
||||
e.subject,
|
||||
s.channel_name,
|
||||
COUNT(s.messages) as message_count
|
||||
FROM gmail.emails e
|
||||
JOIN slack.messages s ON e.sender = s.user_name
|
||||
WHERE e.date >= '2024-01-01'
|
||||
GROUP BY e.sender, e.subject, s.channel_name;
|
||||
```
|
||||
|
||||
### ML Model Predictions
|
||||
```sql
|
||||
-- Use ML model to predict customer churn
|
||||
SELECT
|
||||
customer_id,
|
||||
customer_name,
|
||||
predicted_churn_probability,
|
||||
recommended_action
|
||||
FROM customer_churn_model
|
||||
WHERE predicted_churn_probability > 0.8;
|
||||
```
|
||||
|
||||
## Requirements
|
||||
|
||||
### Database User
|
||||
|
||||
This source uses standard MySQL authentication since MindsDB implements the MySQL wire protocol. You will need to [create a MindsDB user][mindsdb-users] to login to the database with. If MindsDB is configured without authentication, you can omit the password field.
|
||||
|
||||
[mindsdb-users]: https://docs.mindsdb.com/
|
||||
## Example
|
||||
|
||||
```yaml
|
||||
sources:
|
||||
my-mindsdb-source:
|
||||
kind: mindsdb
|
||||
host: 127.0.0.1
|
||||
port: 3306
|
||||
database: my_db
|
||||
user: ${USER_NAME}
|
||||
password: ${PASSWORD} # Optional: omit if MindsDB is configured without authentication
|
||||
queryTimeout: 30s # Optional: query timeout duration
|
||||
```
|
||||
|
||||
### Working Configuration Example
|
||||
|
||||
Here's a working configuration that has been tested:
|
||||
|
||||
```yaml
|
||||
sources:
|
||||
my-pg-source:
|
||||
kind: mindsdb
|
||||
host: 127.0.0.1
|
||||
port: 47335
|
||||
database: files
|
||||
user: mindsdb
|
||||
```
|
||||
|
||||
{{< notice tip >}}
|
||||
Use environment variable replacement with the format ${ENV_NAME}
|
||||
instead of hardcoding your secrets into the configuration file.
|
||||
{{< /notice >}}
|
||||
|
||||
## Use Cases
|
||||
|
||||
With MindsDB integration, you can:
|
||||
|
||||
- **Query Multiple Datasources**: Connect to databases, APIs, file systems, and more through a single SQL interface
|
||||
- **Cross-Datasource Analytics**: Perform joins and analytics across different data sources
|
||||
- **ML Model Integration**: Use trained ML models as virtual tables for predictions and insights
|
||||
- **Unstructured Data Processing**: Query documents, images, and other unstructured data as structured tables
|
||||
- **Real-time Predictions**: Get real-time predictions from ML models through SQL queries
|
||||
- **API Abstraction**: Write SQL queries that automatically translate to REST APIs, GraphQL, and native protocols
|
||||
|
||||
## Reference
|
||||
|
||||
| **field** | **type** | **required** | **description** |
|
||||
| ------------ | :------: | :----------: | ----------------------------------------------------------------------------------------------- |
|
||||
| kind | string | true | Must be "mindsdb". |
|
||||
| host | string | true | IP address to connect to (e.g. "127.0.0.1"). |
|
||||
| port | string | true | Port to connect to (e.g. "3306"). |
|
||||
| database | string | true | Name of the MindsDB database to connect to (e.g. "my_db"). |
|
||||
| user | string | true | Name of the MindsDB user to connect as (e.g. "my-mindsdb-user"). |
|
||||
| password | string | false | Password of the MindsDB user (e.g. "my-password"). Optional if MindsDB is configured without authentication. |
|
||||
| queryTimeout | string | false | Maximum time to wait for query execution (e.g. "30s", "2m"). By default, no timeout is applied. |
|
||||
|
||||
## Resources
|
||||
|
||||
- [MindsDB Documentation][mindsdb-docs] - Official documentation and guides
|
||||
- [MindsDB GitHub][mindsdb-github] - Source code and community
|
||||
116
docs/en/resources/tools/mindsdb/_index.md
Normal file
116
docs/en/resources/tools/mindsdb/_index.md
Normal file
@@ -0,0 +1,116 @@
|
||||
---
|
||||
title: "MindsDB Tools"
|
||||
type: docs
|
||||
weight: 1
|
||||
description: >
|
||||
MindsDB tools that enable SQL queries across hundreds of datasources and ML models.
|
||||
---
|
||||
|
||||
## About
|
||||
|
||||
MindsDB is the most widely adopted AI federated database that enables you to query hundreds of datasources and ML models through a single SQL interface. The following tools work with MindsDB databases:
|
||||
|
||||
- [mindsdb-execute-sql](mindsdb-execute-sql.md) - Execute SQL queries directly on MindsDB
|
||||
- [mindsdb-sql](mindsdb-sql.md) - Execute parameterized SQL queries on MindsDB
|
||||
|
||||
These tools leverage MindsDB's capabilities to:
|
||||
- **Connect to Multiple Datasources**: Query databases, APIs, file systems, and more through SQL
|
||||
- **Cross-Datasource Operations**: Perform joins and analytics across different data sources
|
||||
- **ML Model Integration**: Use trained ML models as virtual tables for predictions
|
||||
- **Unstructured Data Processing**: Query documents, images, and other unstructured data as structured tables
|
||||
- **Real-time Predictions**: Get real-time predictions from ML models through SQL
|
||||
- **API Translation**: Automatically translate SQL queries into REST APIs, GraphQL, and native protocols
|
||||
|
||||
## Supported Datasources
|
||||
|
||||
MindsDB automatically translates your SQL queries into the appropriate APIs for hundreds of datasources:
|
||||
|
||||
### **Business Applications**
|
||||
- **Salesforce**: Query leads, opportunities, accounts, and custom objects
|
||||
- **Jira**: Access issues, projects, workflows, and team data
|
||||
- **GitHub**: Query repositories, commits, pull requests, and issues
|
||||
- **Slack**: Access channels, messages, and team communications
|
||||
- **HubSpot**: Query contacts, companies, deals, and marketing data
|
||||
|
||||
### **Databases & Storage**
|
||||
- **MongoDB**: Query NoSQL collections as structured tables
|
||||
- **PostgreSQL/MySQL**: Standard relational databases
|
||||
- **Redis**: Key-value stores and caching layers
|
||||
- **Elasticsearch**: Search and analytics data
|
||||
- **S3/Google Cloud Storage**: File storage and data lakes
|
||||
|
||||
### **Communication & Email**
|
||||
- **Gmail/Outlook**: Query emails, attachments, and metadata
|
||||
- **Microsoft Teams**: Team communications and files
|
||||
- **Discord**: Server data and message history
|
||||
|
||||
### **Analytics & Monitoring**
|
||||
- **Google Analytics**: Website traffic and user behavior
|
||||
- **Mixpanel**: Product analytics and user events
|
||||
- **Datadog**: Infrastructure monitoring and logs
|
||||
- **Grafana**: Time-series data and metrics
|
||||
|
||||
## Example Use Cases
|
||||
|
||||
### Cross-Datasource Analytics
|
||||
```sql
|
||||
-- Join Salesforce opportunities with GitHub activity
|
||||
SELECT
|
||||
s.opportunity_name,
|
||||
s.amount,
|
||||
g.repository_name,
|
||||
COUNT(g.commits) as commit_count
|
||||
FROM salesforce.opportunities s
|
||||
JOIN github.repositories g ON s.account_id = g.owner_id
|
||||
WHERE s.stage = 'Closed Won';
|
||||
```
|
||||
|
||||
### Email & Communication Analysis
|
||||
```sql
|
||||
-- Analyze email patterns with Slack activity
|
||||
SELECT
|
||||
e.sender,
|
||||
e.subject,
|
||||
s.channel_name,
|
||||
COUNT(s.messages) as message_count
|
||||
FROM gmail.emails e
|
||||
JOIN slack.messages s ON e.sender = s.user_name
|
||||
WHERE e.date >= '2024-01-01';
|
||||
```
|
||||
|
||||
### ML Model Predictions
|
||||
```sql
|
||||
-- Use ML model to predict customer churn
|
||||
SELECT
|
||||
customer_id,
|
||||
customer_name,
|
||||
predicted_churn_probability,
|
||||
recommended_action
|
||||
FROM customer_churn_model
|
||||
WHERE predicted_churn_probability > 0.8;
|
||||
```
|
||||
|
||||
Since MindsDB implements the MySQL wire protocol, these tools are functionally compatible with MySQL tools while providing access to MindsDB's advanced federated database capabilities.
|
||||
|
||||
## Working Configuration Example
|
||||
|
||||
Here's a complete working configuration that has been tested:
|
||||
|
||||
```yaml
|
||||
sources:
|
||||
my-pg-source:
|
||||
kind: mindsdb
|
||||
host: 127.0.0.1
|
||||
port: 47335
|
||||
database: files
|
||||
user: mindsdb
|
||||
|
||||
tools:
|
||||
mindsdb-execute-sql:
|
||||
kind: mindsdb-execute-sql
|
||||
source: my-pg-source
|
||||
description: |
|
||||
Execute SQL queries directly on MindsDB database.
|
||||
Use this tool to run any SQL statement against your MindsDB instance.
|
||||
Example: SELECT * FROM my_table LIMIT 10
|
||||
```
|
||||
126
docs/en/resources/tools/mindsdb/mindsdb-execute-sql.md
Normal file
126
docs/en/resources/tools/mindsdb/mindsdb-execute-sql.md
Normal file
@@ -0,0 +1,126 @@
|
||||
---
|
||||
title: "mindsdb-execute-sql"
|
||||
type: docs
|
||||
weight: 1
|
||||
description: >
|
||||
A "mindsdb-execute-sql" tool executes a SQL statement against a MindsDB
|
||||
federated database.
|
||||
aliases:
|
||||
- /resources/tools/mindsdb-execute-sql
|
||||
---
|
||||
|
||||
## About
|
||||
|
||||
A `mindsdb-execute-sql` tool executes a SQL statement against a MindsDB
|
||||
federated database. It's compatible with any of the following sources:
|
||||
|
||||
- [mindsdb](../sources/mindsdb.md)
|
||||
|
||||
`mindsdb-execute-sql` takes one input parameter `sql` and runs the SQL
|
||||
statement against the `source`. This tool enables you to:
|
||||
|
||||
- **Query Multiple Datasources**: Execute SQL across hundreds of connected datasources
|
||||
- **Cross-Datasource Joins**: Perform joins between different databases, APIs, and file systems
|
||||
- **ML Model Predictions**: Query ML models as virtual tables for real-time predictions
|
||||
- **Unstructured Data**: Query documents, images, and other unstructured data as structured tables
|
||||
- **Federated Analytics**: Perform analytics across multiple datasources simultaneously
|
||||
- **API Translation**: Automatically translate SQL queries into REST APIs, GraphQL, and native protocols
|
||||
|
||||
## Example Queries
|
||||
|
||||
### Cross-Datasource Analytics
|
||||
```sql
|
||||
-- Join Salesforce opportunities with GitHub activity
|
||||
SELECT
|
||||
s.opportunity_name,
|
||||
s.amount,
|
||||
g.repository_name,
|
||||
COUNT(g.commits) as commit_count
|
||||
FROM salesforce.opportunities s
|
||||
JOIN github.repositories g ON s.account_id = g.owner_id
|
||||
WHERE s.stage = 'Closed Won'
|
||||
GROUP BY s.opportunity_name, s.amount, g.repository_name;
|
||||
```
|
||||
|
||||
### Email & Communication Analysis
|
||||
```sql
|
||||
-- Analyze email patterns with Slack activity
|
||||
SELECT
|
||||
e.sender,
|
||||
e.subject,
|
||||
s.channel_name,
|
||||
COUNT(s.messages) as message_count
|
||||
FROM gmail.emails e
|
||||
JOIN slack.messages s ON e.sender = s.user_name
|
||||
WHERE e.date >= '2024-01-01'
|
||||
GROUP BY e.sender, e.subject, s.channel_name;
|
||||
```
|
||||
|
||||
### ML Model Predictions
|
||||
```sql
|
||||
-- Use ML model to predict customer churn
|
||||
SELECT
|
||||
customer_id,
|
||||
customer_name,
|
||||
predicted_churn_probability,
|
||||
recommended_action
|
||||
FROM customer_churn_model
|
||||
WHERE predicted_churn_probability > 0.8;
|
||||
```
|
||||
|
||||
### MongoDB Query
|
||||
```sql
|
||||
-- Query MongoDB collections as structured tables
|
||||
SELECT
|
||||
name,
|
||||
email,
|
||||
department,
|
||||
created_at
|
||||
FROM mongodb.users
|
||||
WHERE department = 'Engineering'
|
||||
ORDER BY created_at DESC;
|
||||
```
|
||||
|
||||
> **Note:** This tool is intended for developer assistant workflows with
|
||||
> human-in-the-loop and shouldn't be used for production agents.
|
||||
|
||||
## Example
|
||||
|
||||
```yaml
|
||||
tools:
|
||||
execute_sql_tool:
|
||||
kind: mindsdb-execute-sql
|
||||
source: my-mindsdb-instance
|
||||
description: Use this tool to execute SQL statements across multiple datasources and ML models.
|
||||
```
|
||||
|
||||
### Working Configuration Example
|
||||
|
||||
Here's a working configuration that has been tested:
|
||||
|
||||
```yaml
|
||||
sources:
|
||||
my-pg-source:
|
||||
kind: mindsdb
|
||||
host: 127.0.0.1
|
||||
port: 47335
|
||||
database: files
|
||||
user: mindsdb
|
||||
|
||||
tools:
|
||||
mindsdb-execute-sql:
|
||||
kind: mindsdb-execute-sql
|
||||
source: my-pg-source
|
||||
description: |
|
||||
Execute SQL queries directly on MindsDB database.
|
||||
Use this tool to run any SQL statement against your MindsDB instance.
|
||||
Example: SELECT * FROM my_table LIMIT 10
|
||||
```
|
||||
|
||||
## Reference
|
||||
|
||||
| **field** | **type** | **required** | **description** |
|
||||
|-------------|:------------------------------------------:|:------------:|--------------------------------------------------------------------------------------------------|
|
||||
| kind | string | true | Must be "mindsdb-execute-sql". |
|
||||
| source | string | true | Name of the source the SQL should execute on. |
|
||||
| description | string | true | Description of the tool that is passed to the LLM. |
|
||||
168
docs/en/resources/tools/mindsdb/mindsdb-sql.md
Normal file
168
docs/en/resources/tools/mindsdb/mindsdb-sql.md
Normal file
@@ -0,0 +1,168 @@
|
||||
---
|
||||
title: "mindsdb-sql"
|
||||
type: docs
|
||||
weight: 1
|
||||
description: >
|
||||
A "mindsdb-sql" tool executes a pre-defined SQL statement against a MindsDB
|
||||
federated database.
|
||||
aliases:
|
||||
- /resources/tools/mindsdb-sql
|
||||
---
|
||||
|
||||
## About
|
||||
|
||||
A `mindsdb-sql` tool executes a pre-defined SQL statement against a MindsDB
|
||||
federated database. It's compatible with any of the following sources:
|
||||
|
||||
- [mindsdb](../sources/mindsdb.md)
|
||||
|
||||
The specified SQL statement is executed as a [prepared statement][mysql-prepare],
|
||||
and expects parameters in the SQL query to be in the form of placeholders `?`.
|
||||
|
||||
This tool enables you to:
|
||||
- **Query Multiple Datasources**: Execute parameterized SQL across hundreds of connected datasources
|
||||
- **Cross-Datasource Joins**: Perform joins between different databases, APIs, and file systems
|
||||
- **ML Model Predictions**: Query ML models as virtual tables for real-time predictions
|
||||
- **Unstructured Data**: Query documents, images, and other unstructured data as structured tables
|
||||
- **Federated Analytics**: Perform analytics across multiple datasources simultaneously
|
||||
- **API Translation**: Automatically translate SQL queries into REST APIs, GraphQL, and native protocols
|
||||
|
||||
[mysql-prepare]: https://dev.mysql.com/doc/refman/8.4/en/sql-prepared-statements.html
|
||||
|
||||
## Example Queries
|
||||
|
||||
### Cross-Datasource Analytics
|
||||
```sql
|
||||
-- Join Salesforce opportunities with GitHub activity
|
||||
SELECT
|
||||
s.opportunity_name,
|
||||
s.amount,
|
||||
g.repository_name,
|
||||
COUNT(g.commits) as commit_count
|
||||
FROM salesforce.opportunities s
|
||||
JOIN github.repositories g ON s.account_id = g.owner_id
|
||||
WHERE s.stage = ?
|
||||
GROUP BY s.opportunity_name, s.amount, g.repository_name;
|
||||
```
|
||||
|
||||
### Email & Communication Analysis
|
||||
```sql
|
||||
-- Analyze email patterns with Slack activity
|
||||
SELECT
|
||||
e.sender,
|
||||
e.subject,
|
||||
s.channel_name,
|
||||
COUNT(s.messages) as message_count
|
||||
FROM gmail.emails e
|
||||
JOIN slack.messages s ON e.sender = s.user_name
|
||||
WHERE e.date >= ?
|
||||
GROUP BY e.sender, e.subject, s.channel_name;
|
||||
```
|
||||
|
||||
### ML Model Predictions
|
||||
```sql
|
||||
-- Use ML model to predict customer churn
|
||||
SELECT
|
||||
customer_id,
|
||||
customer_name,
|
||||
predicted_churn_probability,
|
||||
recommended_action
|
||||
FROM customer_churn_model
|
||||
WHERE predicted_churn_probability > ?;
|
||||
```
|
||||
|
||||
### MongoDB Query
|
||||
```sql
|
||||
-- Query MongoDB collections as structured tables
|
||||
SELECT
|
||||
name,
|
||||
email,
|
||||
department,
|
||||
created_at
|
||||
FROM mongodb.users
|
||||
WHERE department = ?
|
||||
ORDER BY created_at DESC;
|
||||
```
|
||||
|
||||
## Example
|
||||
|
||||
> **Note:** This tool uses parameterized queries to prevent SQL injections.
|
||||
> Query parameters can be used as substitutes for arbitrary expressions.
|
||||
> Parameters cannot be used as substitutes for identifiers, column names, table
|
||||
> names, or other parts of the query.
|
||||
|
||||
```yaml
|
||||
tools:
|
||||
search_flights_by_number:
|
||||
kind: mindsdb-sql
|
||||
source: my-mindsdb-instance
|
||||
statement: |
|
||||
SELECT * FROM flights
|
||||
WHERE airline = ?
|
||||
AND flight_number = ?
|
||||
LIMIT 10
|
||||
description: |
|
||||
Use this tool to get information for a specific flight.
|
||||
Takes an airline code and flight number and returns info on the flight.
|
||||
Do NOT use this tool with a flight id. Do NOT guess an airline code or flight number.
|
||||
A airline code is a code for an airline service consisting of two-character
|
||||
airline designator and followed by flight number, which is 1 to 4 digit number.
|
||||
For example, if given CY 0123, the airline is "CY", and flight_number is "123".
|
||||
Another example for this is DL 1234, the airline is "DL", and flight_number is "1234".
|
||||
If the tool returns more than one option choose the date closes to today.
|
||||
Example:
|
||||
{{
|
||||
"airline": "CY",
|
||||
"flight_number": "888",
|
||||
}}
|
||||
Example:
|
||||
{{
|
||||
"airline": "DL",
|
||||
"flight_number": "1234",
|
||||
}}
|
||||
parameters:
|
||||
- name: airline
|
||||
type: string
|
||||
description: Airline unique 2 letter identifier
|
||||
- name: flight_number
|
||||
type: string
|
||||
description: 1 to 4 digit number
|
||||
```
|
||||
|
||||
### Example with Template Parameters
|
||||
|
||||
> **Note:** This tool allows direct modifications to the SQL statement,
|
||||
> including identifiers, column names, and table names. **This makes it more
|
||||
> vulnerable to SQL injections**. Using basic parameters only (see above) is
|
||||
> recommended for performance and safety reasons. For more details, please check
|
||||
> [templateParameters](_index#template-parameters).
|
||||
|
||||
```yaml
|
||||
tools:
|
||||
list_table:
|
||||
kind: mindsdb-sql
|
||||
source: my-mindsdb-instance
|
||||
statement: |
|
||||
SELECT * FROM {{.tableName}};
|
||||
description: |
|
||||
Use this tool to list all information from a specific table.
|
||||
Example:
|
||||
{{
|
||||
"tableName": "flights",
|
||||
}}
|
||||
templateParameters:
|
||||
- name: tableName
|
||||
type: string
|
||||
description: Table to select from
|
||||
```
|
||||
|
||||
## Reference
|
||||
|
||||
| **field** | **type** | **required** | **description** |
|
||||
|--------------------|:------------------------------------------------:|:------------:|--------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| kind | string | true | Must be "mindsdb-sql". |
|
||||
| source | string | true | Name of the source the SQL should execute on. |
|
||||
| description | string | true | Description of the tool that is passed to the LLM. |
|
||||
| statement | string | true | SQL statement to execute on. |
|
||||
| parameters | [parameters](_index#specifying-parameters) | false | List of [parameters](_index#specifying-parameters) that will be inserted into the SQL statement. |
|
||||
| templateParameters | [templateParameters](_index#template-parameters) | false | List of [templateParameters](_index#template-parameters) that will be inserted into the SQL statement before executing prepared statement. |
|
||||
@@ -39,6 +39,7 @@ var expectedToolSources = []string{
|
||||
"firestore",
|
||||
"looker-conversational-analytics",
|
||||
"looker",
|
||||
"mindsdb",
|
||||
"mssql",
|
||||
"mysql",
|
||||
"neo4j",
|
||||
@@ -118,8 +119,10 @@ func TestGetPrebuiltTool(t *testing.T) {
|
||||
postgresconfig, _ := Get("postgres")
|
||||
spanner_config, _ := Get("spanner")
|
||||
spannerpg_config, _ := Get("spanner-postgres")
|
||||
mindsdb_config, _ := Get("mindsdb")
|
||||
sqlite_config, _ := Get("sqlite")
|
||||
neo4jconfig, _ := Get("neo4j")
|
||||
|
||||
if len(alloydb_admin_config) <= 0 {
|
||||
t.Fatalf("unexpected error: could not fetch alloydb prebuilt tools yaml")
|
||||
}
|
||||
@@ -192,9 +195,15 @@ func TestGetPrebuiltTool(t *testing.T) {
|
||||
if len(spannerpg_config) <= 0 {
|
||||
t.Fatalf("unexpected error: could not fetch spanner pg prebuilt tools yaml")
|
||||
}
|
||||
|
||||
if len(mindsdb_config) <= 0 {
|
||||
t.Fatalf("unexpected error: could not fetch spanner pg prebuilt tools yaml")
|
||||
}
|
||||
|
||||
if len(sqlite_config) <= 0 {
|
||||
t.Fatalf("unexpected error: could not fetch sqlite prebuilt tools yaml")
|
||||
}
|
||||
|
||||
if len(neo4jconfig) <= 0 {
|
||||
t.Fatalf("unexpected error: could not fetch neo4j prebuilt tools yaml")
|
||||
}
|
||||
|
||||
62
internal/prebuiltconfigs/tools/mindsdb.yaml
Normal file
62
internal/prebuiltconfigs/tools/mindsdb.yaml
Normal file
@@ -0,0 +1,62 @@
|
||||
# Copyright 2025 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
sources:
|
||||
mindsdb:
|
||||
kind: mindsdb
|
||||
host: ${MINDSDB_HOST}
|
||||
port: ${MINDSDB_PORT}
|
||||
database: ${MINDSDB_DATABASE}
|
||||
user: ${MINDSDB_USER}
|
||||
password: ${MINDSDB_PASS}
|
||||
|
||||
tools:
|
||||
mindsdb-execute-sql:
|
||||
kind: mindsdb-execute-sql
|
||||
source: mindsdb
|
||||
description: |
|
||||
Execute SQL queries directly on MindsDB database.
|
||||
Use this tool to run any SQL statement against your MindsDB instance.
|
||||
Example: SELECT * FROM my_table LIMIT 10
|
||||
|
||||
mindsdb-sql:
|
||||
kind: mindsdb-sql
|
||||
source: mindsdb
|
||||
statement: |
|
||||
SELECT * FROM {{.table_name}}
|
||||
WHERE {{.condition_column}} = ?
|
||||
LIMIT {{.limit}}
|
||||
description: |
|
||||
Execute parameterized SQL queries on MindsDB database.
|
||||
Use this tool to run parameterized SQL statements against your MindsDB instance.
|
||||
Example: {"table_name": "users", "condition_column": "status", "limit": 10}
|
||||
templateParameters:
|
||||
- name: table_name
|
||||
type: string
|
||||
description: Name of the table to query
|
||||
- name: condition_column
|
||||
type: string
|
||||
description: Column name to use in WHERE clause
|
||||
- name: limit
|
||||
type: integer
|
||||
description: Maximum number of rows to return
|
||||
parameters:
|
||||
- name: value
|
||||
type: string
|
||||
description: Value to match in the WHERE clause
|
||||
|
||||
toolsets:
|
||||
mindsdb-tools:
|
||||
- mindsdb-execute-sql
|
||||
- mindsdb-sql
|
||||
132
internal/sources/mindsdb/mysql.go
Normal file
132
internal/sources/mindsdb/mysql.go
Normal file
@@ -0,0 +1,132 @@
|
||||
// Copyright 2025 Google LLC
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package mindsdb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
"github.com/goccy/go-yaml"
|
||||
"github.com/googleapis/genai-toolbox/internal/sources"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
const SourceKind string = "mindsdb"
|
||||
|
||||
// validate interface
|
||||
var _ sources.SourceConfig = Config{}
|
||||
|
||||
func init() {
|
||||
if !sources.Register(SourceKind, newConfig) {
|
||||
panic(fmt.Sprintf("source kind %q already registered", SourceKind))
|
||||
}
|
||||
}
|
||||
|
||||
func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (sources.SourceConfig, error) {
|
||||
actual := Config{Name: name}
|
||||
if err := decoder.DecodeContext(ctx, &actual); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return actual, nil
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Name string `yaml:"name" validate:"required"`
|
||||
Kind string `yaml:"kind" validate:"required"`
|
||||
Host string `yaml:"host" validate:"required"`
|
||||
Port string `yaml:"port" validate:"required"`
|
||||
User string `yaml:"user" validate:"required"`
|
||||
Password string `yaml:"password"`
|
||||
Database string `yaml:"database" validate:"required"`
|
||||
QueryTimeout string `yaml:"queryTimeout"`
|
||||
}
|
||||
|
||||
func (r Config) SourceConfigKind() string {
|
||||
return SourceKind
|
||||
}
|
||||
|
||||
func (r Config) Initialize(ctx context.Context, tracer trace.Tracer) (sources.Source, error) {
|
||||
pool, err := initMindsDBConnectionPool(ctx, tracer, r.Name, r.Host, r.Port, r.User, r.Password, r.Database, r.QueryTimeout)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to create pool: %w", err)
|
||||
}
|
||||
|
||||
err = pool.PingContext(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to connect successfully: %w", err)
|
||||
}
|
||||
|
||||
s := &Source{
|
||||
Name: r.Name,
|
||||
Kind: SourceKind,
|
||||
Pool: pool,
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
var _ sources.Source = &Source{}
|
||||
|
||||
type Source struct {
|
||||
Name string `yaml:"name"`
|
||||
Kind string `yaml:"kind"`
|
||||
Pool *sql.DB
|
||||
}
|
||||
|
||||
func (s *Source) SourceKind() string {
|
||||
return SourceKind
|
||||
}
|
||||
|
||||
func (s *Source) MindsDBPool() *sql.DB {
|
||||
return s.Pool
|
||||
}
|
||||
|
||||
func (s *Source) MySQLPool() *sql.DB {
|
||||
return s.Pool
|
||||
}
|
||||
|
||||
func initMindsDBConnectionPool(ctx context.Context, tracer trace.Tracer, name, host, port, user, pass, dbname, queryTimeout string) (*sql.DB, error) {
|
||||
//nolint:all // Reassigned ctx
|
||||
ctx, span := sources.InitConnectionSpan(ctx, tracer, SourceKind, name)
|
||||
defer span.End()
|
||||
|
||||
// Configure the driver to connect to the database
|
||||
var dsn string
|
||||
if pass == "" {
|
||||
// Connect without password
|
||||
dsn = fmt.Sprintf("%s@tcp(%s:%s)/%s?parseTime=true", user, host, port, dbname)
|
||||
} else {
|
||||
// Connect with password
|
||||
dsn = fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?parseTime=true", user, pass, host, port, dbname)
|
||||
}
|
||||
|
||||
// Add query timeout to DSN if specified
|
||||
if queryTimeout != "" {
|
||||
timeout, err := time.ParseDuration(queryTimeout)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid queryTimeout %q: %w", queryTimeout, err)
|
||||
}
|
||||
dsn += "&readTimeout=" + timeout.String()
|
||||
}
|
||||
|
||||
// Interact with the driver directly as you normally would
|
||||
pool, err := sql.Open("mysql", dsn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sql.Open: %w", err)
|
||||
}
|
||||
return pool, nil
|
||||
}
|
||||
153
internal/sources/mindsdb/mysql_test.go
Normal file
153
internal/sources/mindsdb/mysql_test.go
Normal file
@@ -0,0 +1,153 @@
|
||||
// Copyright 2025 Google LLC
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package mindsdb_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
yaml "github.com/goccy/go-yaml"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/googleapis/genai-toolbox/internal/server"
|
||||
"github.com/googleapis/genai-toolbox/internal/sources/mindsdb"
|
||||
"github.com/googleapis/genai-toolbox/internal/testutils"
|
||||
)
|
||||
|
||||
func TestParseFromYamlMindsDB(t *testing.T) {
|
||||
tcs := []struct {
|
||||
desc string
|
||||
in string
|
||||
want server.SourceConfigs
|
||||
}{
|
||||
{
|
||||
desc: "basic example",
|
||||
in: `
|
||||
sources:
|
||||
my-mindsdb-instance:
|
||||
kind: mindsdb
|
||||
host: 0.0.0.0
|
||||
port: my-port
|
||||
database: my_db
|
||||
user: my_user
|
||||
password: my_pass
|
||||
`,
|
||||
want: server.SourceConfigs{
|
||||
"my-mindsdb-instance": mindsdb.Config{
|
||||
Name: "my-mindsdb-instance",
|
||||
Kind: mindsdb.SourceKind,
|
||||
Host: "0.0.0.0",
|
||||
Port: "my-port",
|
||||
Database: "my_db",
|
||||
User: "my_user",
|
||||
Password: "my_pass",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "with query timeout",
|
||||
in: `
|
||||
sources:
|
||||
my-mindsdb-instance:
|
||||
kind: mindsdb
|
||||
host: 0.0.0.0
|
||||
port: my-port
|
||||
database: my_db
|
||||
user: my_user
|
||||
password: my_pass
|
||||
queryTimeout: 45s
|
||||
`,
|
||||
want: server.SourceConfigs{
|
||||
"my-mindsdb-instance": mindsdb.Config{
|
||||
Name: "my-mindsdb-instance",
|
||||
Kind: mindsdb.SourceKind,
|
||||
Host: "0.0.0.0",
|
||||
Port: "my-port",
|
||||
Database: "my_db",
|
||||
User: "my_user",
|
||||
Password: "my_pass",
|
||||
QueryTimeout: "45s",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tc := range tcs {
|
||||
t.Run(tc.desc, func(t *testing.T) {
|
||||
got := struct {
|
||||
Sources server.SourceConfigs `yaml:"sources"`
|
||||
}{}
|
||||
// Parse contents
|
||||
err := yaml.Unmarshal(testutils.FormatYaml(tc.in), &got)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to unmarshal: %s", err)
|
||||
}
|
||||
if !cmp.Equal(tc.want, got.Sources) {
|
||||
t.Fatalf("incorrect parse: want %v, got %v", tc.want, got.Sources)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestFailParseFromYaml(t *testing.T) {
|
||||
tcs := []struct {
|
||||
desc string
|
||||
in string
|
||||
err string
|
||||
}{
|
||||
{
|
||||
desc: "extra field",
|
||||
in: `
|
||||
sources:
|
||||
my-mindsdb-instance:
|
||||
kind: mindsdb
|
||||
host: 0.0.0.0
|
||||
port: my-port
|
||||
database: my_db
|
||||
user: my_user
|
||||
password: my_pass
|
||||
foo: bar
|
||||
`,
|
||||
err: "unable to parse source \"my-mindsdb-instance\" as \"mindsdb\": [2:1] unknown field \"foo\"\n 1 | database: my_db\n> 2 | foo: bar\n ^\n 3 | host: 0.0.0.0\n 4 | kind: mindsdb\n 5 | password: my_pass\n 6 | ",
|
||||
},
|
||||
{
|
||||
desc: "missing required field",
|
||||
in: `
|
||||
sources:
|
||||
my-mindsdb-instance:
|
||||
kind: mindsdb
|
||||
port: my-port
|
||||
database: my_db
|
||||
user: my_user
|
||||
password: my_pass
|
||||
`,
|
||||
err: "unable to parse source \"my-mindsdb-instance\" as \"mindsdb\": Key: 'Config.Host' Error:Field validation for 'Host' failed on the 'required' tag",
|
||||
},
|
||||
}
|
||||
for _, tc := range tcs {
|
||||
t.Run(tc.desc, func(t *testing.T) {
|
||||
got := struct {
|
||||
Sources server.SourceConfigs `yaml:"sources"`
|
||||
}{}
|
||||
// Parse contents
|
||||
err := yaml.Unmarshal(testutils.FormatYaml(tc.in), &got)
|
||||
if err == nil {
|
||||
t.Fatalf("expect parsing to fail")
|
||||
}
|
||||
errStr := err.Error()
|
||||
if errStr != tc.err {
|
||||
t.Fatalf("unexpected error: got %q, want %q", errStr, tc.err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
197
internal/tools/mindsdb/mindsdbexecutesql/mindsdbexecutesql.go
Normal file
197
internal/tools/mindsdb/mindsdbexecutesql/mindsdbexecutesql.go
Normal file
@@ -0,0 +1,197 @@
|
||||
// Copyright 2025 Google LLC
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package mindsdbexecutesql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
|
||||
yaml "github.com/goccy/go-yaml"
|
||||
"github.com/googleapis/genai-toolbox/internal/sources"
|
||||
"github.com/googleapis/genai-toolbox/internal/sources/mindsdb"
|
||||
"github.com/googleapis/genai-toolbox/internal/tools"
|
||||
"github.com/googleapis/genai-toolbox/internal/tools/mysql/mysqlcommon"
|
||||
)
|
||||
|
||||
const kind string = "mindsdb-execute-sql"
|
||||
|
||||
func init() {
|
||||
if !tools.Register(kind, newConfig) {
|
||||
panic(fmt.Sprintf("tool kind %q already registered", kind))
|
||||
}
|
||||
}
|
||||
|
||||
func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (tools.ToolConfig, error) {
|
||||
actual := Config{Name: name}
|
||||
if err := decoder.DecodeContext(ctx, &actual); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return actual, nil
|
||||
}
|
||||
|
||||
type compatibleSource interface {
|
||||
MindsDBPool() *sql.DB
|
||||
}
|
||||
|
||||
// validate compatible sources are still compatible
|
||||
var _ compatibleSource = &mindsdb.Source{}
|
||||
|
||||
var compatibleSources = [...]string{mindsdb.SourceKind}
|
||||
|
||||
type Config struct {
|
||||
Name string `yaml:"name" validate:"required"`
|
||||
Kind string `yaml:"kind" validate:"required"`
|
||||
Source string `yaml:"source" validate:"required"`
|
||||
Description string `yaml:"description" validate:"required"`
|
||||
AuthRequired []string `yaml:"authRequired"`
|
||||
}
|
||||
|
||||
// validate interface
|
||||
var _ tools.ToolConfig = Config{}
|
||||
|
||||
func (cfg Config) ToolConfigKind() string {
|
||||
return kind
|
||||
}
|
||||
|
||||
func (cfg Config) Initialize(srcs map[string]sources.Source) (tools.Tool, error) {
|
||||
// verify source exists
|
||||
rawS, ok := srcs[cfg.Source]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no source named %q configured", cfg.Source)
|
||||
}
|
||||
|
||||
// verify the source is compatible
|
||||
s, ok := rawS.(compatibleSource)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid source for %q tool: source kind must be one of %q", kind, compatibleSources)
|
||||
}
|
||||
|
||||
sqlParameter := tools.NewStringParameter("sql", "The sql to execute.")
|
||||
parameters := tools.Parameters{sqlParameter}
|
||||
|
||||
inputSchema, _ := parameters.McpManifest()
|
||||
mcpManifest := tools.McpManifest{
|
||||
Name: cfg.Name,
|
||||
Description: cfg.Description,
|
||||
InputSchema: inputSchema,
|
||||
}
|
||||
|
||||
// finish tool setup
|
||||
t := Tool{
|
||||
Name: cfg.Name,
|
||||
Kind: kind,
|
||||
Parameters: parameters,
|
||||
AuthRequired: cfg.AuthRequired,
|
||||
Pool: s.MindsDBPool(),
|
||||
manifest: tools.Manifest{Description: cfg.Description, Parameters: parameters.Manifest(), AuthRequired: cfg.AuthRequired},
|
||||
mcpManifest: mcpManifest,
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// validate interface
|
||||
var _ tools.Tool = Tool{}
|
||||
|
||||
type Tool struct {
|
||||
Name string `yaml:"name"`
|
||||
Kind string `yaml:"kind"`
|
||||
AuthRequired []string `yaml:"authRequired"`
|
||||
Parameters tools.Parameters `yaml:"parameters"`
|
||||
|
||||
Pool *sql.DB
|
||||
manifest tools.Manifest
|
||||
mcpManifest tools.McpManifest
|
||||
}
|
||||
|
||||
func (t Tool) Invoke(ctx context.Context, params tools.ParamValues, accessToken tools.AccessToken) (any, error) {
|
||||
paramsMap := params.AsMap()
|
||||
sql, ok := paramsMap["sql"].(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unable to get cast %s", paramsMap["sql"])
|
||||
}
|
||||
|
||||
results, err := t.Pool.QueryContext(ctx, sql)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to execute query: %w", err)
|
||||
}
|
||||
defer results.Close()
|
||||
|
||||
cols, err := results.Columns()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to retrieve rows column name: %w", err)
|
||||
}
|
||||
|
||||
// create an array of values for each column, which can be re-used to scan each row
|
||||
rawValues := make([]any, len(cols))
|
||||
values := make([]any, len(cols))
|
||||
for i := range rawValues {
|
||||
values[i] = &rawValues[i]
|
||||
}
|
||||
|
||||
colTypes, err := results.ColumnTypes()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to get column types: %w", err)
|
||||
}
|
||||
|
||||
var out []any
|
||||
for results.Next() {
|
||||
err := results.Scan(values...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to parse row: %w", err)
|
||||
}
|
||||
vMap := make(map[string]any)
|
||||
for i, name := range cols {
|
||||
val := rawValues[i]
|
||||
if val == nil {
|
||||
vMap[name] = nil
|
||||
continue
|
||||
}
|
||||
|
||||
// MindsDB uses mysql driver
|
||||
vMap[name], err = mysqlcommon.ConvertToType(colTypes[i], val)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("errors encountered when converting values: %w", err)
|
||||
}
|
||||
}
|
||||
out = append(out, vMap)
|
||||
}
|
||||
|
||||
if err := results.Err(); err != nil {
|
||||
return nil, fmt.Errorf("errors encountered during row iteration: %w", err)
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (t Tool) ParseParams(data map[string]any, claims map[string]map[string]any) (tools.ParamValues, error) {
|
||||
return tools.ParseParams(t.Parameters, data, claims)
|
||||
}
|
||||
|
||||
func (t Tool) Manifest() tools.Manifest {
|
||||
return t.manifest
|
||||
}
|
||||
|
||||
func (t Tool) McpManifest() tools.McpManifest {
|
||||
return t.mcpManifest
|
||||
}
|
||||
|
||||
func (t Tool) Authorized(verifiedAuthServices []string) bool {
|
||||
return tools.IsAuthorized(t.AuthRequired, verifiedAuthServices)
|
||||
}
|
||||
|
||||
func (t Tool) RequiresClientAuthorization() bool {
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
// Copyright 2025 Google LLC
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package mindsdbexecutesql_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
yaml "github.com/goccy/go-yaml"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/googleapis/genai-toolbox/internal/server"
|
||||
"github.com/googleapis/genai-toolbox/internal/testutils"
|
||||
"github.com/googleapis/genai-toolbox/internal/tools/mindsdb/mindsdbexecutesql"
|
||||
)
|
||||
|
||||
func TestParseFromYamlExecuteSql(t *testing.T) {
|
||||
ctx, err := testutils.ContextWithNewLogger()
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
tcs := []struct {
|
||||
desc string
|
||||
in string
|
||||
want server.ToolConfigs
|
||||
}{
|
||||
{
|
||||
desc: "basic example",
|
||||
in: `
|
||||
tools:
|
||||
example_tool:
|
||||
kind: mindsdb-execute-sql
|
||||
source: my-instance
|
||||
description: some description
|
||||
authRequired:
|
||||
- my-google-auth-service
|
||||
- other-auth-service
|
||||
`,
|
||||
want: server.ToolConfigs{
|
||||
"example_tool": mindsdbexecutesql.Config{
|
||||
Name: "example_tool",
|
||||
Kind: "mindsdb-execute-sql",
|
||||
Source: "my-instance",
|
||||
Description: "some description",
|
||||
AuthRequired: []string{"my-google-auth-service", "other-auth-service"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tc := range tcs {
|
||||
t.Run(tc.desc, func(t *testing.T) {
|
||||
got := struct {
|
||||
Tools server.ToolConfigs `yaml:"tools"`
|
||||
}{}
|
||||
// Parse contents
|
||||
err := yaml.UnmarshalContext(ctx, testutils.FormatYaml(tc.in), &got)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to unmarshal: %s", err)
|
||||
}
|
||||
if diff := cmp.Diff(tc.want, got.Tools); diff != "" {
|
||||
t.Fatalf("incorrect parse: diff %v", diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
217
internal/tools/mindsdb/mindsdbsql/mindsdbsql.go
Normal file
217
internal/tools/mindsdb/mindsdbsql/mindsdbsql.go
Normal file
@@ -0,0 +1,217 @@
|
||||
// Copyright 2025 Google LLC
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package mindsdbsql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
|
||||
yaml "github.com/goccy/go-yaml"
|
||||
"github.com/googleapis/genai-toolbox/internal/sources"
|
||||
"github.com/googleapis/genai-toolbox/internal/sources/mindsdb"
|
||||
"github.com/googleapis/genai-toolbox/internal/tools"
|
||||
"github.com/googleapis/genai-toolbox/internal/tools/mysql/mysqlcommon"
|
||||
)
|
||||
|
||||
const kind string = "mindsdb-sql"
|
||||
|
||||
func init() {
|
||||
if !tools.Register(kind, newConfig) {
|
||||
panic(fmt.Sprintf("tool kind %q already registered", kind))
|
||||
}
|
||||
}
|
||||
|
||||
func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (tools.ToolConfig, error) {
|
||||
actual := Config{Name: name}
|
||||
if err := decoder.DecodeContext(ctx, &actual); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return actual, nil
|
||||
}
|
||||
|
||||
type compatibleSource interface {
|
||||
MindsDBPool() *sql.DB
|
||||
}
|
||||
|
||||
// validate compatible sources are still compatible
|
||||
var _ compatibleSource = &mindsdb.Source{}
|
||||
|
||||
var compatibleSources = [...]string{mindsdb.SourceKind}
|
||||
|
||||
type Config struct {
|
||||
Name string `yaml:"name" validate:"required"`
|
||||
Kind string `yaml:"kind" validate:"required"`
|
||||
Source string `yaml:"source" validate:"required"`
|
||||
Description string `yaml:"description" validate:"required"`
|
||||
Statement string `yaml:"statement" validate:"required"`
|
||||
AuthRequired []string `yaml:"authRequired"`
|
||||
Parameters tools.Parameters `yaml:"parameters"`
|
||||
TemplateParameters tools.Parameters `yaml:"templateParameters"`
|
||||
}
|
||||
|
||||
// validate interface
|
||||
var _ tools.ToolConfig = Config{}
|
||||
|
||||
func (cfg Config) ToolConfigKind() string {
|
||||
return kind
|
||||
}
|
||||
|
||||
func (cfg Config) Initialize(srcs map[string]sources.Source) (tools.Tool, error) {
|
||||
// verify source exists
|
||||
rawS, ok := srcs[cfg.Source]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no source named %q configured", cfg.Source)
|
||||
}
|
||||
|
||||
// verify the source is compatible
|
||||
s, ok := rawS.(compatibleSource)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid source for %q tool: source kind must be one of %q", kind, compatibleSources)
|
||||
}
|
||||
|
||||
allParameters, paramManifest, err := tools.ProcessParameters(cfg.TemplateParameters, cfg.Parameters)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
paramMcpManifest, _ := allParameters.McpManifest()
|
||||
|
||||
mcpManifest := tools.McpManifest{
|
||||
Name: cfg.Name,
|
||||
Description: cfg.Description,
|
||||
InputSchema: paramMcpManifest,
|
||||
}
|
||||
|
||||
// finish tool setup
|
||||
t := Tool{
|
||||
Name: cfg.Name,
|
||||
Kind: kind,
|
||||
Parameters: cfg.Parameters,
|
||||
TemplateParameters: cfg.TemplateParameters,
|
||||
AllParams: allParameters,
|
||||
Statement: cfg.Statement,
|
||||
AuthRequired: cfg.AuthRequired,
|
||||
Pool: s.MindsDBPool(),
|
||||
manifest: tools.Manifest{Description: cfg.Description, Parameters: paramManifest, AuthRequired: cfg.AuthRequired},
|
||||
mcpManifest: mcpManifest,
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// validate interface
|
||||
var _ tools.Tool = Tool{}
|
||||
|
||||
type Tool struct {
|
||||
Name string `yaml:"name"`
|
||||
Kind string `yaml:"kind"`
|
||||
AuthRequired []string `yaml:"authRequired"`
|
||||
Parameters tools.Parameters `yaml:"parameters"`
|
||||
TemplateParameters tools.Parameters `yaml:"templateParameters"`
|
||||
AllParams tools.Parameters `yaml:"allParams"`
|
||||
|
||||
Pool *sql.DB
|
||||
Statement string
|
||||
manifest tools.Manifest
|
||||
mcpManifest tools.McpManifest
|
||||
}
|
||||
|
||||
func (t Tool) Invoke(ctx context.Context, params tools.ParamValues, accessToken tools.AccessToken) (any, error) {
|
||||
paramsMap := params.AsMap()
|
||||
newStatement, err := tools.ResolveTemplateParams(t.TemplateParameters, t.Statement, paramsMap)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to extract template params %w", err)
|
||||
}
|
||||
|
||||
newParams, err := tools.GetParams(t.Parameters, paramsMap)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to extract standard params %w", err)
|
||||
}
|
||||
|
||||
sliceParams := newParams.AsSlice()
|
||||
|
||||
// MindsDB now supports MySQL prepared statements natively
|
||||
results, err := t.Pool.QueryContext(ctx, newStatement, sliceParams...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to execute query: %w", err)
|
||||
}
|
||||
|
||||
cols, err := results.Columns()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to retrieve rows column name: %w", err)
|
||||
}
|
||||
|
||||
// create an array of values for each column, which can be re-used to scan each row
|
||||
rawValues := make([]any, len(cols))
|
||||
values := make([]any, len(cols))
|
||||
for i := range rawValues {
|
||||
values[i] = &rawValues[i]
|
||||
}
|
||||
defer results.Close()
|
||||
|
||||
colTypes, err := results.ColumnTypes()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to get column types: %w", err)
|
||||
}
|
||||
|
||||
var out []any
|
||||
for results.Next() {
|
||||
err := results.Scan(values...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to parse row: %w", err)
|
||||
}
|
||||
vMap := make(map[string]any)
|
||||
for i, name := range cols {
|
||||
val := rawValues[i]
|
||||
if val == nil {
|
||||
vMap[name] = nil
|
||||
continue
|
||||
}
|
||||
|
||||
// MindsDB uses mysql driver
|
||||
vMap[name], err = mysqlcommon.ConvertToType(colTypes[i], val)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("errors encountered when converting values: %w", err)
|
||||
}
|
||||
}
|
||||
out = append(out, vMap)
|
||||
}
|
||||
|
||||
if err := results.Err(); err != nil {
|
||||
return nil, fmt.Errorf("errors encountered during row iteration: %w", err)
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (t Tool) ParseParams(data map[string]any, claims map[string]map[string]any) (tools.ParamValues, error) {
|
||||
return tools.ParseParams(t.AllParams, data, claims)
|
||||
}
|
||||
|
||||
func (t Tool) Manifest() tools.Manifest {
|
||||
return t.manifest
|
||||
}
|
||||
|
||||
func (t Tool) McpManifest() tools.McpManifest {
|
||||
return t.mcpManifest
|
||||
}
|
||||
|
||||
func (t Tool) Authorized(verifiedAuthServices []string) bool {
|
||||
return tools.IsAuthorized(t.AuthRequired, verifiedAuthServices)
|
||||
}
|
||||
|
||||
func (t Tool) RequiresClientAuthorization() bool {
|
||||
return false
|
||||
}
|
||||
175
internal/tools/mindsdb/mindsdbsql/mindsdbsql_test.go
Normal file
175
internal/tools/mindsdb/mindsdbsql/mindsdbsql_test.go
Normal file
@@ -0,0 +1,175 @@
|
||||
// Copyright 2025 Google LLC
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package mindsdbsql_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
yaml "github.com/goccy/go-yaml"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/googleapis/genai-toolbox/internal/server"
|
||||
"github.com/googleapis/genai-toolbox/internal/testutils"
|
||||
"github.com/googleapis/genai-toolbox/internal/tools"
|
||||
"github.com/googleapis/genai-toolbox/internal/tools/mindsdb/mindsdbsql"
|
||||
)
|
||||
|
||||
func TestParseFromYamlmindsdbsql(t *testing.T) {
|
||||
ctx, err := testutils.ContextWithNewLogger()
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
tcs := []struct {
|
||||
desc string
|
||||
in string
|
||||
want server.ToolConfigs
|
||||
}{
|
||||
{
|
||||
desc: "basic example",
|
||||
in: `
|
||||
tools:
|
||||
example_tool:
|
||||
kind: mindsdb-sql
|
||||
source: my-mindsdbsql-instance
|
||||
description: some description
|
||||
statement: |
|
||||
SELECT * FROM SQL_STATEMENT;
|
||||
authRequired:
|
||||
- my-google-auth-service
|
||||
- other-auth-service
|
||||
parameters:
|
||||
- name: country
|
||||
type: string
|
||||
description: some description
|
||||
authServices:
|
||||
- name: my-google-auth-service
|
||||
field: user_id
|
||||
- name: other-auth-service
|
||||
field: user_id
|
||||
`,
|
||||
want: server.ToolConfigs{
|
||||
"example_tool": mindsdbsql.Config{
|
||||
Name: "example_tool",
|
||||
Kind: "mindsdb-sql",
|
||||
Source: "my-mindsdbsql-instance",
|
||||
Description: "some description",
|
||||
Statement: "SELECT * FROM SQL_STATEMENT;\n",
|
||||
AuthRequired: []string{"my-google-auth-service", "other-auth-service"},
|
||||
Parameters: []tools.Parameter{
|
||||
tools.NewStringParameterWithAuth("country", "some description",
|
||||
[]tools.ParamAuthService{{Name: "my-google-auth-service", Field: "user_id"},
|
||||
{Name: "other-auth-service", Field: "user_id"}}),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tc := range tcs {
|
||||
t.Run(tc.desc, func(t *testing.T) {
|
||||
got := struct {
|
||||
Tools server.ToolConfigs `yaml:"tools"`
|
||||
}{}
|
||||
// Parse contents
|
||||
err := yaml.UnmarshalContext(ctx, testutils.FormatYaml(tc.in), &got)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to unmarshal: %s", err)
|
||||
}
|
||||
if diff := cmp.Diff(tc.want, got.Tools); diff != "" {
|
||||
t.Fatalf("incorrect parse: diff %v", diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFromYamlWithTemplateParamsmindsdbsql(t *testing.T) {
|
||||
ctx, err := testutils.ContextWithNewLogger()
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
tcs := []struct {
|
||||
desc string
|
||||
in string
|
||||
want server.ToolConfigs
|
||||
}{
|
||||
{
|
||||
desc: "basic example",
|
||||
in: `
|
||||
tools:
|
||||
example_tool:
|
||||
kind: mindsdb-sql
|
||||
source: my-mindsdbsql-instance
|
||||
description: some description
|
||||
statement: |
|
||||
SELECT * FROM SQL_STATEMENT;
|
||||
authRequired:
|
||||
- my-google-auth-service
|
||||
- other-auth-service
|
||||
parameters:
|
||||
- name: country
|
||||
type: string
|
||||
description: some description
|
||||
authServices:
|
||||
- name: my-google-auth-service
|
||||
field: user_id
|
||||
- name: other-auth-service
|
||||
field: user_id
|
||||
templateParameters:
|
||||
- name: tableName
|
||||
type: string
|
||||
description: The table to select hotels from.
|
||||
- name: fieldArray
|
||||
type: array
|
||||
description: The columns to return for the query.
|
||||
items:
|
||||
name: column
|
||||
type: string
|
||||
description: A column name that will be returned from the query.
|
||||
`,
|
||||
want: server.ToolConfigs{
|
||||
"example_tool": mindsdbsql.Config{
|
||||
Name: "example_tool",
|
||||
Kind: "mindsdb-sql",
|
||||
Source: "my-mindsdbsql-instance",
|
||||
Description: "some description",
|
||||
Statement: "SELECT * FROM SQL_STATEMENT;\n",
|
||||
AuthRequired: []string{"my-google-auth-service", "other-auth-service"},
|
||||
Parameters: []tools.Parameter{
|
||||
tools.NewStringParameterWithAuth("country", "some description",
|
||||
[]tools.ParamAuthService{{Name: "my-google-auth-service", Field: "user_id"},
|
||||
{Name: "other-auth-service", Field: "user_id"}}),
|
||||
},
|
||||
TemplateParameters: []tools.Parameter{
|
||||
tools.NewStringParameter("tableName", "The table to select hotels from."),
|
||||
tools.NewArrayParameter("fieldArray", "The columns to return for the query.", tools.NewStringParameter("column", "A column name that will be returned from the query.")),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tc := range tcs {
|
||||
t.Run(tc.desc, func(t *testing.T) {
|
||||
got := struct {
|
||||
Tools server.ToolConfigs `yaml:"tools"`
|
||||
}{}
|
||||
// Parse contents
|
||||
err := yaml.UnmarshalContext(ctx, testutils.FormatYaml(tc.in), &got)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to unmarshal: %s", err)
|
||||
}
|
||||
if diff := cmp.Diff(tc.want, got.Tools); diff != "" {
|
||||
t.Fatalf("incorrect parse: diff %v", diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
yaml "github.com/goccy/go-yaml"
|
||||
"github.com/googleapis/genai-toolbox/internal/sources"
|
||||
"github.com/googleapis/genai-toolbox/internal/sources/cloudsqlmysql"
|
||||
"github.com/googleapis/genai-toolbox/internal/sources/mindsdb"
|
||||
"github.com/googleapis/genai-toolbox/internal/sources/mysql"
|
||||
"github.com/googleapis/genai-toolbox/internal/tools"
|
||||
"github.com/googleapis/genai-toolbox/internal/tools/mysql/mysqlcommon"
|
||||
@@ -51,8 +52,9 @@ type compatibleSource interface {
|
||||
// validate compatible sources are still compatible
|
||||
var _ compatibleSource = &cloudsqlmysql.Source{}
|
||||
var _ compatibleSource = &mysql.Source{}
|
||||
var _ compatibleSource = &mindsdb.Source{}
|
||||
|
||||
var compatibleSources = [...]string{cloudsqlmysql.SourceKind, mysql.SourceKind}
|
||||
var compatibleSources = [...]string{cloudsqlmysql.SourceKind, mysql.SourceKind, mindsdb.SourceKind}
|
||||
|
||||
type Config struct {
|
||||
Name string `yaml:"name" validate:"required"`
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
yaml "github.com/goccy/go-yaml"
|
||||
"github.com/googleapis/genai-toolbox/internal/sources"
|
||||
"github.com/googleapis/genai-toolbox/internal/sources/cloudsqlmysql"
|
||||
"github.com/googleapis/genai-toolbox/internal/sources/mindsdb"
|
||||
"github.com/googleapis/genai-toolbox/internal/sources/mysql"
|
||||
"github.com/googleapis/genai-toolbox/internal/tools"
|
||||
"github.com/googleapis/genai-toolbox/internal/tools/mysql/mysqlcommon"
|
||||
@@ -50,8 +51,9 @@ type compatibleSource interface {
|
||||
// validate compatible sources are still compatible
|
||||
var _ compatibleSource = &cloudsqlmysql.Source{}
|
||||
var _ compatibleSource = &mysql.Source{}
|
||||
var _ compatibleSource = &mindsdb.Source{}
|
||||
|
||||
var compatibleSources = [...]string{cloudsqlmysql.SourceKind, mysql.SourceKind}
|
||||
var compatibleSources = [...]string{cloudsqlmysql.SourceKind, mysql.SourceKind, mindsdb.SourceKind}
|
||||
|
||||
type Config struct {
|
||||
Name string `yaml:"name" validate:"required"`
|
||||
|
||||
459
tests/mindsdb/mindsdb_integration_test.go
Normal file
459
tests/mindsdb/mindsdb_integration_test.go
Normal file
@@ -0,0 +1,459 @@
|
||||
// Copyright 2025 Google LLC
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package mindsdb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
"github.com/google/uuid"
|
||||
"github.com/googleapis/genai-toolbox/internal/testutils"
|
||||
"github.com/googleapis/genai-toolbox/tests"
|
||||
)
|
||||
|
||||
var (
|
||||
MindsDBSourceKind = "mindsdb"
|
||||
MindsDBToolKind = "mindsdb-sql"
|
||||
MindsDBDatabase = os.Getenv("MINDSDB_DATABASE")
|
||||
MindsDBHost = os.Getenv("MINDSDB_HOST")
|
||||
MindsDBPort = os.Getenv("MINDSDB_PORT")
|
||||
MindsDBUser = os.Getenv("MINDSDB_USER")
|
||||
MindsDBPass = os.Getenv("MINDSDB_PASS")
|
||||
)
|
||||
|
||||
func getMindsDBVars(t *testing.T) map[string]any {
|
||||
switch "" {
|
||||
case MindsDBDatabase:
|
||||
t.Fatal("'MINDSDB_DATABASE' not set")
|
||||
case MindsDBHost:
|
||||
t.Fatal("'MINDSDB_HOST' not set")
|
||||
case MindsDBPort:
|
||||
t.Fatal("'MINDSDB_PORT' not set")
|
||||
case MindsDBUser:
|
||||
t.Fatal("'MINDSDB_USER' not set")
|
||||
}
|
||||
|
||||
// MindsDBPass can be empty, but the env var must exist
|
||||
if _, exists := os.LookupEnv("MINDSDB_PASS"); !exists {
|
||||
t.Fatal("'MINDSDB_PASS' not set (can be empty)")
|
||||
}
|
||||
|
||||
// Handle no-password authentication
|
||||
mindsdbPassword := MindsDBPass
|
||||
if mindsdbPassword == "none" {
|
||||
mindsdbPassword = ""
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"kind": MindsDBSourceKind,
|
||||
"host": MindsDBHost,
|
||||
"port": MindsDBPort,
|
||||
"database": MindsDBDatabase,
|
||||
"user": MindsDBUser,
|
||||
"password": mindsdbPassword,
|
||||
}
|
||||
}
|
||||
|
||||
// initMindsDBConnectionPool creates a connection pool using MySQL protocol
|
||||
func initMindsDBConnectionPool(host, port, user, pass, dbname string) (*sql.DB, error) {
|
||||
dsn := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?parseTime=true", user, pass, host, port, dbname)
|
||||
pool, err := sql.Open("mysql", dsn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sql.Open: %w", err)
|
||||
}
|
||||
return pool, nil
|
||||
}
|
||||
|
||||
func TestMindsDBToolEndpoints(t *testing.T) {
|
||||
sourceConfig := getMindsDBVars(t)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||
defer cancel()
|
||||
|
||||
var args []string
|
||||
|
||||
// Create unique table names with UUID
|
||||
tableNameParam := "param_table_" + strings.ReplaceAll(uuid.New().String(), "-", "")
|
||||
tableNameAuth := "auth_table_" + strings.ReplaceAll(uuid.New().String(), "-", "")
|
||||
// Tool statements with ORDER BY for consistent results
|
||||
paramToolStmt := fmt.Sprintf("SELECT * FROM files.%s WHERE id = ? OR name = ? ORDER BY id", tableNameParam)
|
||||
idParamToolStmt := fmt.Sprintf("SELECT * FROM files.%s WHERE id = ? ORDER BY id", tableNameParam)
|
||||
nameParamToolStmt := fmt.Sprintf("SELECT * FROM files.%s WHERE name = ? ORDER BY id", tableNameParam)
|
||||
authToolStmt := fmt.Sprintf("SELECT name FROM files.%s WHERE email = ? ORDER BY name", tableNameAuth)
|
||||
|
||||
toolsFile := map[string]any{
|
||||
"sources": map[string]any{
|
||||
"my-instance": sourceConfig,
|
||||
},
|
||||
"authServices": map[string]any{
|
||||
"my-google-auth": map[string]any{
|
||||
"kind": "google",
|
||||
"clientId": tests.ClientId,
|
||||
},
|
||||
},
|
||||
"tools": map[string]any{
|
||||
"my-simple-tool": map[string]any{
|
||||
"kind": MindsDBToolKind,
|
||||
"source": "my-instance",
|
||||
"description": "Simple tool to test end to end functionality.",
|
||||
"statement": "SELECT 1",
|
||||
},
|
||||
"my-tool": map[string]any{
|
||||
"kind": MindsDBToolKind,
|
||||
"source": "my-instance",
|
||||
"description": "Tool to test invocation with params.",
|
||||
"statement": paramToolStmt,
|
||||
"parameters": []map[string]any{
|
||||
{
|
||||
"name": "id",
|
||||
"type": "integer",
|
||||
"description": "user ID",
|
||||
},
|
||||
{
|
||||
"name": "name",
|
||||
"type": "string",
|
||||
"description": "user name",
|
||||
},
|
||||
},
|
||||
},
|
||||
"my-tool-by-id": map[string]any{
|
||||
"kind": MindsDBToolKind,
|
||||
"source": "my-instance",
|
||||
"description": "Tool to test invocation with params.",
|
||||
"statement": idParamToolStmt,
|
||||
"parameters": []map[string]any{
|
||||
{
|
||||
"name": "id",
|
||||
"type": "integer",
|
||||
"description": "user ID",
|
||||
},
|
||||
},
|
||||
},
|
||||
"my-tool-by-name": map[string]any{
|
||||
"kind": MindsDBToolKind,
|
||||
"source": "my-instance",
|
||||
"description": "Tool to test invocation with params.",
|
||||
"statement": nameParamToolStmt,
|
||||
"parameters": []map[string]any{
|
||||
{
|
||||
"name": "name",
|
||||
"type": "string",
|
||||
"description": "user name",
|
||||
"required": false,
|
||||
},
|
||||
},
|
||||
},
|
||||
"my-array-tool": map[string]any{
|
||||
"kind": MindsDBToolKind,
|
||||
"source": "my-instance",
|
||||
"description": "Tool to test invocation with array params.",
|
||||
"statement": "SELECT 1 as id, 'Alice' as name UNION SELECT 3 as id, 'Sid' as name",
|
||||
},
|
||||
"my-auth-tool": map[string]any{
|
||||
"kind": MindsDBToolKind,
|
||||
"source": "my-instance",
|
||||
"description": "Tool to test authenticated parameters.",
|
||||
"statement": authToolStmt,
|
||||
"parameters": []map[string]any{
|
||||
{
|
||||
"name": "email",
|
||||
"type": "string",
|
||||
"description": "user email",
|
||||
"authServices": []map[string]string{
|
||||
{
|
||||
"name": "my-google-auth",
|
||||
"field": "email",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"my-auth-required-tool": map[string]any{
|
||||
"kind": MindsDBToolKind,
|
||||
"source": "my-instance",
|
||||
"description": "Tool to test auth required invocation.",
|
||||
"statement": "SELECT 1",
|
||||
"authRequired": []string{
|
||||
"my-google-auth",
|
||||
},
|
||||
},
|
||||
"my-fail-tool": map[string]any{
|
||||
"kind": MindsDBToolKind,
|
||||
"source": "my-instance",
|
||||
"description": "Tool to test statement with incorrect syntax.",
|
||||
"statement": "INVALID SQL STATEMENT",
|
||||
},
|
||||
"my-exec-sql-tool": map[string]any{
|
||||
"kind": "mindsdb-execute-sql",
|
||||
"source": "my-instance",
|
||||
"description": "Tool to execute sql",
|
||||
},
|
||||
"my-auth-exec-sql-tool": map[string]any{
|
||||
"kind": "mindsdb-execute-sql",
|
||||
"source": "my-instance",
|
||||
"description": "Tool to execute sql with auth",
|
||||
"authRequired": []string{
|
||||
"my-google-auth",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
cmd, cleanup, err := tests.StartCmd(ctx, toolsFile, args...)
|
||||
if err != nil {
|
||||
t.Fatalf("command initialization returned an error: %s", err)
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
waitCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := testutils.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`), cmd.Out)
|
||||
if err != nil {
|
||||
t.Logf("toolbox command logs: \n%s", out)
|
||||
t.Fatalf("toolbox didn't start successfully: %s", err)
|
||||
}
|
||||
|
||||
// Create connection pool and test tables with sample data
|
||||
pool, err := initMindsDBConnectionPool(MindsDBHost, MindsDBPort, MindsDBUser, MindsDBPass, MindsDBDatabase)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to create MindsDB connection pool: %s", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
// Create param table: id=1:Alice, id=2:Jane, id=3:Sid, id=4:null
|
||||
createParamSQL := fmt.Sprintf("CREATE TABLE files.%s (SELECT 1 as id, 'Alice' as name UNION ALL SELECT 2, 'Jane' UNION ALL SELECT 3, 'Sid' UNION ALL SELECT 4, NULL)", tableNameParam)
|
||||
_, err = pool.ExecContext(ctx, createParamSQL)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to create param table: %s", err)
|
||||
}
|
||||
|
||||
// Create auth table: id=1:Alice:test@..., id=2:Jane:jane@...
|
||||
createAuthSQL := fmt.Sprintf("CREATE TABLE files.%s (SELECT 1 as id, 'Alice' as name, '%s' as email UNION ALL SELECT 2, 'Jane', 'janedoe@gmail.com')", tableNameAuth, tests.ServiceAccountEmail)
|
||||
_, err = pool.ExecContext(ctx, createAuthSQL)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to create auth table: %s", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_, _ = pool.ExecContext(ctx, fmt.Sprintf("DROP TABLE IF EXISTS files.%s", tableNameParam))
|
||||
_, _ = pool.ExecContext(ctx, fmt.Sprintf("DROP TABLE IF EXISTS files.%s", tableNameAuth))
|
||||
}()
|
||||
|
||||
select1Want := "[{\"1\":1}]"
|
||||
|
||||
// Run standard tool tests with MindsDB-specific expectations
|
||||
tests.RunToolGetTest(t)
|
||||
tests.RunToolInvokeTest(t, select1Want,
|
||||
tests.DisableArrayTest(), // MindsDB doesn't support array parameters
|
||||
)
|
||||
|
||||
t.Run("mindsdb_core_functionality", func(t *testing.T) {
|
||||
tests.RunToolInvokeSimpleTest(t, "my-simple-tool", select1Want)
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool", []byte(`{"sql": "SELECT 1"}`), select1Want)
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool", []byte(`{"sql": "SELECT 1+1 as result"}`), "[{\"result\":2}]")
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool", []byte(`{"sql": "SELECT 'hello' as greeting"}`), "[{\"greeting\":\"hello\"}]")
|
||||
})
|
||||
|
||||
t.Run("mindsdb_sql_tests", func(t *testing.T) {
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool", []byte(`{"sql": "SELECT 1"}`), select1Want)
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool", []byte(`{"sql": "SHOW DATABASES"}`), "")
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool", []byte(`{"sql": "SHOW TABLES"}`), "")
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool", []byte(`{"sql": "SELECT TABLE_NAME FROM information_schema.TABLES LIMIT 1"}`), "")
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool", []byte(`{"sql": "SELECT 1+1 as result"}`), "")
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool", []byte(`{"sql": "SELECT UPPER('hello') as result"}`), "")
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool", []byte(`{"sql": "SELECT NOW() as current_time"}`), "")
|
||||
})
|
||||
|
||||
// Test CREATE DATABASE (MindsDB's federated database capability)
|
||||
t.Run("mindsdb_create_database", func(t *testing.T) {
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "DROP DATABASE IF EXISTS test_postgres_db"}`), "")
|
||||
|
||||
// Create external database integration using MindsDB's demo database
|
||||
createDBSQL := `CREATE DATABASE test_postgres_db WITH ENGINE = 'postgres', PARAMETERS = {'user': 'demo_user', 'password': 'demo_password', 'host': 'samples.mindsdb.com', 'port': '5432', 'database': 'demo', 'schema': 'demo_data'}`
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "`+createDBSQL+`"}`), "")
|
||||
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "SHOW DATABASES"}`), "")
|
||||
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "SHOW TABLES FROM test_postgres_db"}`), "")
|
||||
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "DROP DATABASE IF EXISTS test_postgres_db"}`), "")
|
||||
})
|
||||
|
||||
// Test MindsDB integration capabilities with product/review data
|
||||
t.Run("mindsdb_integration_demo", func(t *testing.T) {
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "DROP TABLE IF EXISTS files.test_products"}`), "")
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "DROP TABLE IF EXISTS files.test_reviews"}`), "")
|
||||
|
||||
// Create test tables with sample data
|
||||
createProductsSQL := `CREATE TABLE files.test_products (SELECT 'PROD001' as product_id, 'Laptop Computer' as product_name, 'Electronics' as category UNION ALL SELECT 'PROD002', 'Office Chair', 'Furniture' UNION ALL SELECT 'PROD003', 'Coffee Maker', 'Appliances' UNION ALL SELECT 'PROD004', 'Desk Lamp', 'Furniture')`
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "`+createProductsSQL+`"}`), "")
|
||||
|
||||
createReviewsSQL := `CREATE TABLE files.test_reviews (SELECT 'PROD001' as product_id, 'Great laptop, very fast!' as review, 5 as rating UNION ALL SELECT 'PROD001', 'Good value for money', 4 UNION ALL SELECT 'PROD002', 'Very comfortable chair', 5 UNION ALL SELECT 'PROD002', 'Nice design but expensive', 3 UNION ALL SELECT 'PROD003', 'Makes excellent coffee', 5 UNION ALL SELECT 'PROD004', 'Bright light, perfect for reading', 4)`
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "`+createReviewsSQL+`"}`), "")
|
||||
|
||||
t.Run("query_created_tables", func(t *testing.T) {
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "SELECT * FROM files.test_products ORDER BY product_id"}`), "")
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "SELECT * FROM files.test_reviews ORDER BY product_id, rating DESC"}`), "")
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "SELECT category, COUNT(*) as product_count FROM files.test_products GROUP BY category ORDER BY category"}`), "")
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "SELECT product_id, AVG(rating) as avg_rating FROM files.test_reviews GROUP BY product_id ORDER BY avg_rating DESC"}`), "")
|
||||
})
|
||||
|
||||
t.Run("cross_database_join", func(t *testing.T) {
|
||||
joinSQL := `SELECT p.product_name, p.category, r.review, r.rating FROM files.test_products p JOIN files.test_reviews r ON p.product_id = r.product_id WHERE r.rating >= 4 ORDER BY p.product_name, r.rating DESC`
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "`+joinSQL+`"}`), "")
|
||||
|
||||
aggSQL := `SELECT p.category, COUNT(DISTINCT p.product_id) as product_count, COUNT(r.review) as review_count, AVG(r.rating) as avg_rating FROM files.test_products p LEFT JOIN files.test_reviews r ON p.product_id = r.product_id GROUP BY p.category ORDER BY avg_rating DESC`
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "`+aggSQL+`"}`), "")
|
||||
})
|
||||
|
||||
t.Run("advanced_sql_features", func(t *testing.T) {
|
||||
subquerySQL := `SELECT p.product_name, p.category, AVG(r.rating) as avg_rating FROM files.test_products p JOIN files.test_reviews r ON p.product_id = r.product_id GROUP BY p.product_id, p.product_name, p.category HAVING AVG(r.rating) >= 4 ORDER BY avg_rating DESC`
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "`+subquerySQL+`"}`), "")
|
||||
|
||||
caseSQL := `SELECT product_id, review, rating, CASE WHEN rating >= 5 THEN 'Excellent' WHEN rating >= 4 THEN 'Good' WHEN rating >= 3 THEN 'Average' ELSE 'Poor' END as rating_category FROM files.test_reviews ORDER BY rating DESC, product_id`
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "`+caseSQL+`"}`), "")
|
||||
})
|
||||
|
||||
t.Run("data_manipulation", func(t *testing.T) {
|
||||
summarySQL := `CREATE TABLE files.test_product_summary (SELECT p.product_id, p.product_name, p.category, COUNT(r.review) as total_reviews, AVG(r.rating) as avg_rating, MAX(r.rating) as max_rating, MIN(r.rating) as min_rating FROM files.test_products p LEFT JOIN files.test_reviews r ON p.product_id = r.product_id GROUP BY p.product_id, p.product_name, p.category)`
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "`+summarySQL+`"}`), "")
|
||||
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "SELECT * FROM files.test_product_summary ORDER BY avg_rating DESC"}`), "")
|
||||
})
|
||||
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "DROP TABLE IF EXISTS files.test_products"}`), "")
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "DROP TABLE IF EXISTS files.test_reviews"}`), "")
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "DROP TABLE IF EXISTS files.test_product_summary"}`), "")
|
||||
})
|
||||
|
||||
// Test database integration and cross-database joins
|
||||
t.Run("mindsdb_create_database_integration", func(t *testing.T) {
|
||||
showDBSQL := `SHOW DATABASES`
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "`+showDBSQL+`"}`), "")
|
||||
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "SHOW TABLES FROM files"}`), "")
|
||||
|
||||
createIntegrationTableSQL := `CREATE TABLE files.test_integration_data (SELECT 1 as id, 'Data from integration' as description, CURDATE() as created_at UNION ALL SELECT 2, 'Another record', CURDATE() UNION ALL SELECT 3, 'Third record', CURDATE())`
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "`+createIntegrationTableSQL+`"}`), "")
|
||||
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "SELECT * FROM files.test_integration_data ORDER BY id"}`), "")
|
||||
|
||||
createLocalTableSQL := `CREATE TABLE files.test_local_data (SELECT 1 as id, 'Local metadata' as metadata UNION ALL SELECT 2, 'More metadata')`
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "`+createLocalTableSQL+`"}`), "")
|
||||
|
||||
crossJoinSQL := `SELECT i.id, i.description, l.metadata FROM files.test_integration_data i LEFT JOIN files.test_local_data l ON i.id = l.id ORDER BY i.id`
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "`+crossJoinSQL+`"}`), "")
|
||||
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "DROP TABLE IF EXISTS files.test_integration_data"}`), "")
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "DROP TABLE IF EXISTS files.test_local_data"}`), "")
|
||||
})
|
||||
|
||||
// Test data transformation with customer order data
|
||||
t.Run("mindsdb_data_transformation", func(t *testing.T) {
|
||||
createOrdersSQL := `CREATE TABLE files.test_orders (SELECT 1 as order_id, 'CUST001' as customer_id, 100.50 as amount, '2024-01-15' as order_date UNION ALL SELECT 2, 'CUST001', 250.00, '2024-02-20' UNION ALL SELECT 3, 'CUST002', 75.25, '2024-01-18' UNION ALL SELECT 4, 'CUST003', 500.00, '2024-03-10' UNION ALL SELECT 5, 'CUST002', 150.00, '2024-02-25')`
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "`+createOrdersSQL+`"}`), "")
|
||||
|
||||
customerSummarySQL := `CREATE TABLE files.test_customer_summary (SELECT customer_id, COUNT(*) as total_orders, SUM(amount) as total_spent, AVG(amount) as avg_order_value, MIN(order_date) as first_order_date, MAX(order_date) as last_order_date FROM files.test_orders GROUP BY customer_id)`
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "`+customerSummarySQL+`"}`), "")
|
||||
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "SELECT * FROM files.test_customer_summary ORDER BY total_spent DESC"}`), "")
|
||||
|
||||
segmentSQL := `SELECT customer_id, total_spent, CASE WHEN total_spent >= 300 THEN 'High Value' WHEN total_spent >= 150 THEN 'Medium Value' ELSE 'Low Value' END as customer_segment FROM files.test_customer_summary ORDER BY total_spent DESC`
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "`+segmentSQL+`"}`), "")
|
||||
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "DROP TABLE IF EXISTS files.test_orders"}`), "")
|
||||
tests.RunToolInvokeParametersTest(t, "my-exec-sql-tool",
|
||||
[]byte(`{"sql": "DROP TABLE IF EXISTS files.test_customer_summary"}`), "")
|
||||
})
|
||||
|
||||
// Test error handling - these are expected to fail but exercise error paths
|
||||
t.Run("mindsdb_error_handling", func(t *testing.T) {
|
||||
// Test invalid SQL - expect this to fail with 400
|
||||
resp, err := http.Post("http://127.0.0.1:5000/api/tool/my-exec-sql-tool/invoke", "application/json", bytes.NewBuffer([]byte(`{"sql": "INVALID SQL QUERY"}`)))
|
||||
if err != nil {
|
||||
t.Fatalf("error when sending request: %s", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusBadRequest {
|
||||
t.Logf("Expected 400 for invalid SQL, got %d (this exercises error handling)", resp.StatusCode)
|
||||
}
|
||||
|
||||
// Test empty SQL - expect this to fail with 400
|
||||
resp2, err := http.Post("http://127.0.0.1:5000/api/tool/my-exec-sql-tool/invoke", "application/json", bytes.NewBuffer([]byte(`{"sql": ""}`)))
|
||||
if err != nil {
|
||||
t.Fatalf("error when sending request: %s", err)
|
||||
}
|
||||
defer resp2.Body.Close()
|
||||
if resp2.StatusCode != http.StatusBadRequest {
|
||||
t.Logf("Expected 400 for empty SQL, got %d (this exercises error handling)", resp2.StatusCode)
|
||||
}
|
||||
})
|
||||
|
||||
// Test authentication - these are expected to fail but exercise auth code paths
|
||||
t.Run("mindsdb_auth_tests", func(t *testing.T) {
|
||||
// Test auth-required tool without auth - expect this to fail with 401
|
||||
resp, err := http.Post("http://127.0.0.1:5000/api/tool/my-auth-exec-sql-tool/invoke", "application/json", bytes.NewBuffer([]byte(`{"sql": "SELECT 1"}`)))
|
||||
if err != nil {
|
||||
t.Fatalf("error when sending request: %s", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusUnauthorized {
|
||||
t.Logf("Expected 401 for missing auth, got %d (this exercises auth handling)", resp.StatusCode)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -373,6 +373,7 @@ func RunToolInvokeTest(t *testing.T, select1Want string, options ...InvokeTestOp
|
||||
enabled: configs.supportSelect1Auth,
|
||||
requestHeader: map[string]string{"my-google-auth_token": "INVALID_TOKEN"},
|
||||
requestBody: bytes.NewBuffer([]byte(`{}`)),
|
||||
wantBody: "",
|
||||
wantStatusCode: http.StatusUnauthorized,
|
||||
},
|
||||
{
|
||||
@@ -381,15 +382,15 @@ func RunToolInvokeTest(t *testing.T, select1Want string, options ...InvokeTestOp
|
||||
enabled: true,
|
||||
requestHeader: map[string]string{},
|
||||
requestBody: bytes.NewBuffer([]byte(`{}`)),
|
||||
wantBody: "",
|
||||
wantStatusCode: http.StatusUnauthorized,
|
||||
},
|
||||
{
|
||||
name: "Invoke my-auth-required-tool with auth token",
|
||||
api: "http://127.0.0.1:5000/api/tool/my-auth-required-tool/invoke",
|
||||
enabled: configs.supportSelect1Auth,
|
||||
requestHeader: map[string]string{"my-google-auth_token": idToken},
|
||||
requestBody: bytes.NewBuffer([]byte(`{}`)),
|
||||
|
||||
name: "Invoke my-auth-required-tool with auth token",
|
||||
api: "http://127.0.0.1:5000/api/tool/my-auth-required-tool/invoke",
|
||||
enabled: configs.supportSelect1Auth,
|
||||
requestHeader: map[string]string{"my-google-auth_token": idToken},
|
||||
requestBody: bytes.NewBuffer([]byte(`{}`)),
|
||||
wantBody: select1Want,
|
||||
wantStatusCode: http.StatusOK,
|
||||
},
|
||||
@@ -399,6 +400,7 @@ func RunToolInvokeTest(t *testing.T, select1Want string, options ...InvokeTestOp
|
||||
enabled: true,
|
||||
requestHeader: map[string]string{"my-google-auth_token": "INVALID_TOKEN"},
|
||||
requestBody: bytes.NewBuffer([]byte(`{}`)),
|
||||
wantBody: "",
|
||||
wantStatusCode: http.StatusUnauthorized,
|
||||
},
|
||||
{
|
||||
@@ -407,6 +409,7 @@ func RunToolInvokeTest(t *testing.T, select1Want string, options ...InvokeTestOp
|
||||
enabled: true,
|
||||
requestHeader: map[string]string{},
|
||||
requestBody: bytes.NewBuffer([]byte(`{}`)),
|
||||
wantBody: "",
|
||||
wantStatusCode: http.StatusUnauthorized,
|
||||
},
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user