mirror of
https://github.com/googleapis/genai-toolbox.git
synced 2026-01-09 15:38:08 -05:00
🚀 Add MindsDB Integration: Expand Toolbox to Hundreds of Datasources Overview This PR introduces comprehensive MindsDB integration to the Google GenAI Toolbox, enabling SQL queries across hundreds of datasources through a unified interface. MindsDB is the most widely adopted AI federated database that automatically translates MySQL queries into REST APIs, GraphQL, and native protocols. 🎯 Key Value for Google GenAI Toolbox Ecosystem 1. Massive Datasource Expansion Before: Toolbox limited to ~15 traditional databases After: Access to hundreds of datasources including Salesforce, Jira, GitHub, MongoDB, Gmail, Slack, and more Impact: Dramatically expands the toolbox's reach and utility for enterprise users 2. Cross-Datasource Analytics New Capability: Perform joins and analytics across different datasources seamlessly Example: Join Salesforce opportunities with GitHub activity to correlate sales with development activity Value: Enables comprehensive data analysis that was previously impossible 3. API Abstraction Layer Innovation: Write standard SQL queries that automatically translate to any API Benefit: Developers can query REST APIs, GraphQL, and native protocols using familiar SQL syntax Impact: Reduces complexity and learning curve for accessing diverse datasources 4. ML Model Integration Enhanced Capability: Use ML models as virtual tables for real-time predictions Example: Query customer churn predictions directly through SQL Value: Brings AI/ML capabilities into the standard SQL workflow 🔧 Technical Implementation Source Layer ✅ New MindsDB source implementation using MySQL wire protocol ✅ Comprehensive test coverage with integration tests ✅ Updated existing MySQL tools to support MindsDB sources ✅ Created dedicated MindsDB tools for enhanced functionality Tools Layer ✅ mindsdb-execute-sql: Direct SQL execution across federated datasources ✅ mindsdb-sql: Parameterized SQL queries with template support ✅ Backward compatibility with existing MySQL tools Documentation & Configuration ✅ Comprehensive documentation with real-world examples ✅ Prebuilt configuration for easy setup ✅ Updated CLI help text and command-line options 📊 Supported Datasources Business Applications Salesforce (leads, opportunities, accounts) Jira (issues, projects, workflows) GitHub (repositories, commits, PRs) Slack (channels, messages, teams) HubSpot (contacts, companies, deals) Databases & Storage MongoDB (NoSQL collections as structured tables) Redis (key-value stores) Elasticsearch (search and analytics) S3, filesystems, etc (file storage) Communication & Email Gmail/Outlook (emails, attachments) Microsoft Teams (communications, files) Discord (server data, messages) 🎯 Example Use Cases Cross-Datasource Analytics -- Join Salesforce opportunities with GitHub activity ``` SELECT s.opportunity_name, s.amount, g.repository_name, COUNT(g.commits) as commit_count FROM salesforce.opportunities s JOIN github.repositories g ON s.account_id = g.owner_id WHERE s.stage = 'Closed Won'; ``` Email & Communication Analysis ``` -- Analyze email patterns with Slack activity SELECT e.sender, e.subject, s.channel_name, COUNT(s.messages) as message_count FROM gmail.emails e JOIN slack.messages s ON e.sender = s.user_name WHERE e.date >= '2024-01-01'; ``` 🚀 Benefits for Google GenAI Toolbox Enterprise Adoption: Enables access to enterprise datasources (Salesforce, Jira, etc.) Developer Productivity: Familiar SQL interface for any datasource AI/ML Integration: Seamless integration of ML models into SQL workflows Scalability: Single interface for hundreds of datasources Competitive Advantage: Unique federated database capabilities in the toolbox ecosystem 📈 Impact Metrics Datasource Coverage: +1000% increase in supported datasources API Abstraction: Eliminates need to learn individual API syntaxes Cross-Platform Analytics: Enables previously impossible data correlations ML Integration: Brings AI capabilities into standard SQL workflows 🔗 Resources MindsDB Documentation MindsDB GitHub Updated Toolbox Documentation ✅ Testing ✅ Unit tests for MindsDB source implementation ✅ Integration tests with real datasource examples ✅ Backward compatibility with existing MySQL tools ✅ Documentation examples tested and verified This integration transforms the Google GenAI Toolbox from a traditional database tool into a comprehensive federated data platform, enabling users to query and analyze data across their entire technology stack through a unified SQL interface. --------- Co-authored-by: duwenxin <duwenxin@google.com> Co-authored-by: setohe0909 <setohe.09@gmail.com> Co-authored-by: Kurtis Van Gent <31518063+kurtisvg@users.noreply.github.com> Co-authored-by: Wenxin Du <117315983+duwenxin99@users.noreply.github.com> Co-authored-by: Yuan Teoh <45984206+Yuan325@users.noreply.github.com>
176 lines
5.1 KiB
Go
176 lines
5.1 KiB
Go
// Copyright 2025 Google LLC
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package mindsdbsql_test
|
|
|
|
import (
|
|
"testing"
|
|
|
|
yaml "github.com/goccy/go-yaml"
|
|
"github.com/google/go-cmp/cmp"
|
|
"github.com/googleapis/genai-toolbox/internal/server"
|
|
"github.com/googleapis/genai-toolbox/internal/testutils"
|
|
"github.com/googleapis/genai-toolbox/internal/tools"
|
|
"github.com/googleapis/genai-toolbox/internal/tools/mindsdb/mindsdbsql"
|
|
)
|
|
|
|
func TestParseFromYamlmindsdbsql(t *testing.T) {
|
|
ctx, err := testutils.ContextWithNewLogger()
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %s", err)
|
|
}
|
|
tcs := []struct {
|
|
desc string
|
|
in string
|
|
want server.ToolConfigs
|
|
}{
|
|
{
|
|
desc: "basic example",
|
|
in: `
|
|
tools:
|
|
example_tool:
|
|
kind: mindsdb-sql
|
|
source: my-mindsdbsql-instance
|
|
description: some description
|
|
statement: |
|
|
SELECT * FROM SQL_STATEMENT;
|
|
authRequired:
|
|
- my-google-auth-service
|
|
- other-auth-service
|
|
parameters:
|
|
- name: country
|
|
type: string
|
|
description: some description
|
|
authServices:
|
|
- name: my-google-auth-service
|
|
field: user_id
|
|
- name: other-auth-service
|
|
field: user_id
|
|
`,
|
|
want: server.ToolConfigs{
|
|
"example_tool": mindsdbsql.Config{
|
|
Name: "example_tool",
|
|
Kind: "mindsdb-sql",
|
|
Source: "my-mindsdbsql-instance",
|
|
Description: "some description",
|
|
Statement: "SELECT * FROM SQL_STATEMENT;\n",
|
|
AuthRequired: []string{"my-google-auth-service", "other-auth-service"},
|
|
Parameters: []tools.Parameter{
|
|
tools.NewStringParameterWithAuth("country", "some description",
|
|
[]tools.ParamAuthService{{Name: "my-google-auth-service", Field: "user_id"},
|
|
{Name: "other-auth-service", Field: "user_id"}}),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
for _, tc := range tcs {
|
|
t.Run(tc.desc, func(t *testing.T) {
|
|
got := struct {
|
|
Tools server.ToolConfigs `yaml:"tools"`
|
|
}{}
|
|
// Parse contents
|
|
err := yaml.UnmarshalContext(ctx, testutils.FormatYaml(tc.in), &got)
|
|
if err != nil {
|
|
t.Fatalf("unable to unmarshal: %s", err)
|
|
}
|
|
if diff := cmp.Diff(tc.want, got.Tools); diff != "" {
|
|
t.Fatalf("incorrect parse: diff %v", diff)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestParseFromYamlWithTemplateParamsmindsdbsql(t *testing.T) {
|
|
ctx, err := testutils.ContextWithNewLogger()
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %s", err)
|
|
}
|
|
tcs := []struct {
|
|
desc string
|
|
in string
|
|
want server.ToolConfigs
|
|
}{
|
|
{
|
|
desc: "basic example",
|
|
in: `
|
|
tools:
|
|
example_tool:
|
|
kind: mindsdb-sql
|
|
source: my-mindsdbsql-instance
|
|
description: some description
|
|
statement: |
|
|
SELECT * FROM SQL_STATEMENT;
|
|
authRequired:
|
|
- my-google-auth-service
|
|
- other-auth-service
|
|
parameters:
|
|
- name: country
|
|
type: string
|
|
description: some description
|
|
authServices:
|
|
- name: my-google-auth-service
|
|
field: user_id
|
|
- name: other-auth-service
|
|
field: user_id
|
|
templateParameters:
|
|
- name: tableName
|
|
type: string
|
|
description: The table to select hotels from.
|
|
- name: fieldArray
|
|
type: array
|
|
description: The columns to return for the query.
|
|
items:
|
|
name: column
|
|
type: string
|
|
description: A column name that will be returned from the query.
|
|
`,
|
|
want: server.ToolConfigs{
|
|
"example_tool": mindsdbsql.Config{
|
|
Name: "example_tool",
|
|
Kind: "mindsdb-sql",
|
|
Source: "my-mindsdbsql-instance",
|
|
Description: "some description",
|
|
Statement: "SELECT * FROM SQL_STATEMENT;\n",
|
|
AuthRequired: []string{"my-google-auth-service", "other-auth-service"},
|
|
Parameters: []tools.Parameter{
|
|
tools.NewStringParameterWithAuth("country", "some description",
|
|
[]tools.ParamAuthService{{Name: "my-google-auth-service", Field: "user_id"},
|
|
{Name: "other-auth-service", Field: "user_id"}}),
|
|
},
|
|
TemplateParameters: []tools.Parameter{
|
|
tools.NewStringParameter("tableName", "The table to select hotels from."),
|
|
tools.NewArrayParameter("fieldArray", "The columns to return for the query.", tools.NewStringParameter("column", "A column name that will be returned from the query.")),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
for _, tc := range tcs {
|
|
t.Run(tc.desc, func(t *testing.T) {
|
|
got := struct {
|
|
Tools server.ToolConfigs `yaml:"tools"`
|
|
}{}
|
|
// Parse contents
|
|
err := yaml.UnmarshalContext(ctx, testutils.FormatYaml(tc.in), &got)
|
|
if err != nil {
|
|
t.Fatalf("unable to unmarshal: %s", err)
|
|
}
|
|
if diff := cmp.Diff(tc.want, got.Tools); diff != "" {
|
|
t.Fatalf("incorrect parse: diff %v", diff)
|
|
}
|
|
})
|
|
}
|
|
}
|