mirror of
https://github.com/googleapis/genai-toolbox.git
synced 2026-01-09 15:38:08 -05:00
🚀 Add MindsDB Integration: Expand Toolbox to Hundreds of Datasources Overview This PR introduces comprehensive MindsDB integration to the Google GenAI Toolbox, enabling SQL queries across hundreds of datasources through a unified interface. MindsDB is the most widely adopted AI federated database that automatically translates MySQL queries into REST APIs, GraphQL, and native protocols. 🎯 Key Value for Google GenAI Toolbox Ecosystem 1. Massive Datasource Expansion Before: Toolbox limited to ~15 traditional databases After: Access to hundreds of datasources including Salesforce, Jira, GitHub, MongoDB, Gmail, Slack, and more Impact: Dramatically expands the toolbox's reach and utility for enterprise users 2. Cross-Datasource Analytics New Capability: Perform joins and analytics across different datasources seamlessly Example: Join Salesforce opportunities with GitHub activity to correlate sales with development activity Value: Enables comprehensive data analysis that was previously impossible 3. API Abstraction Layer Innovation: Write standard SQL queries that automatically translate to any API Benefit: Developers can query REST APIs, GraphQL, and native protocols using familiar SQL syntax Impact: Reduces complexity and learning curve for accessing diverse datasources 4. ML Model Integration Enhanced Capability: Use ML models as virtual tables for real-time predictions Example: Query customer churn predictions directly through SQL Value: Brings AI/ML capabilities into the standard SQL workflow 🔧 Technical Implementation Source Layer ✅ New MindsDB source implementation using MySQL wire protocol ✅ Comprehensive test coverage with integration tests ✅ Updated existing MySQL tools to support MindsDB sources ✅ Created dedicated MindsDB tools for enhanced functionality Tools Layer ✅ mindsdb-execute-sql: Direct SQL execution across federated datasources ✅ mindsdb-sql: Parameterized SQL queries with template support ✅ Backward compatibility with existing MySQL tools Documentation & Configuration ✅ Comprehensive documentation with real-world examples ✅ Prebuilt configuration for easy setup ✅ Updated CLI help text and command-line options 📊 Supported Datasources Business Applications Salesforce (leads, opportunities, accounts) Jira (issues, projects, workflows) GitHub (repositories, commits, PRs) Slack (channels, messages, teams) HubSpot (contacts, companies, deals) Databases & Storage MongoDB (NoSQL collections as structured tables) Redis (key-value stores) Elasticsearch (search and analytics) S3, filesystems, etc (file storage) Communication & Email Gmail/Outlook (emails, attachments) Microsoft Teams (communications, files) Discord (server data, messages) 🎯 Example Use Cases Cross-Datasource Analytics -- Join Salesforce opportunities with GitHub activity ``` SELECT s.opportunity_name, s.amount, g.repository_name, COUNT(g.commits) as commit_count FROM salesforce.opportunities s JOIN github.repositories g ON s.account_id = g.owner_id WHERE s.stage = 'Closed Won'; ``` Email & Communication Analysis ``` -- Analyze email patterns with Slack activity SELECT e.sender, e.subject, s.channel_name, COUNT(s.messages) as message_count FROM gmail.emails e JOIN slack.messages s ON e.sender = s.user_name WHERE e.date >= '2024-01-01'; ``` 🚀 Benefits for Google GenAI Toolbox Enterprise Adoption: Enables access to enterprise datasources (Salesforce, Jira, etc.) Developer Productivity: Familiar SQL interface for any datasource AI/ML Integration: Seamless integration of ML models into SQL workflows Scalability: Single interface for hundreds of datasources Competitive Advantage: Unique federated database capabilities in the toolbox ecosystem 📈 Impact Metrics Datasource Coverage: +1000% increase in supported datasources API Abstraction: Eliminates need to learn individual API syntaxes Cross-Platform Analytics: Enables previously impossible data correlations ML Integration: Brings AI capabilities into standard SQL workflows 🔗 Resources MindsDB Documentation MindsDB GitHub Updated Toolbox Documentation ✅ Testing ✅ Unit tests for MindsDB source implementation ✅ Integration tests with real datasource examples ✅ Backward compatibility with existing MySQL tools ✅ Documentation examples tested and verified This integration transforms the Google GenAI Toolbox from a traditional database tool into a comprehensive federated data platform, enabling users to query and analyze data across their entire technology stack through a unified SQL interface. --------- Co-authored-by: duwenxin <duwenxin@google.com> Co-authored-by: setohe0909 <setohe.09@gmail.com> Co-authored-by: Kurtis Van Gent <31518063+kurtisvg@users.noreply.github.com> Co-authored-by: Wenxin Du <117315983+duwenxin99@users.noreply.github.com> Co-authored-by: Yuan Teoh <45984206+Yuan325@users.noreply.github.com>
154 lines
3.7 KiB
Go
154 lines
3.7 KiB
Go
// Copyright 2025 Google LLC
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package mindsdb_test
|
|
|
|
import (
|
|
"testing"
|
|
|
|
yaml "github.com/goccy/go-yaml"
|
|
"github.com/google/go-cmp/cmp"
|
|
"github.com/googleapis/genai-toolbox/internal/server"
|
|
"github.com/googleapis/genai-toolbox/internal/sources/mindsdb"
|
|
"github.com/googleapis/genai-toolbox/internal/testutils"
|
|
)
|
|
|
|
func TestParseFromYamlMindsDB(t *testing.T) {
|
|
tcs := []struct {
|
|
desc string
|
|
in string
|
|
want server.SourceConfigs
|
|
}{
|
|
{
|
|
desc: "basic example",
|
|
in: `
|
|
sources:
|
|
my-mindsdb-instance:
|
|
kind: mindsdb
|
|
host: 0.0.0.0
|
|
port: my-port
|
|
database: my_db
|
|
user: my_user
|
|
password: my_pass
|
|
`,
|
|
want: server.SourceConfigs{
|
|
"my-mindsdb-instance": mindsdb.Config{
|
|
Name: "my-mindsdb-instance",
|
|
Kind: mindsdb.SourceKind,
|
|
Host: "0.0.0.0",
|
|
Port: "my-port",
|
|
Database: "my_db",
|
|
User: "my_user",
|
|
Password: "my_pass",
|
|
},
|
|
},
|
|
},
|
|
{
|
|
desc: "with query timeout",
|
|
in: `
|
|
sources:
|
|
my-mindsdb-instance:
|
|
kind: mindsdb
|
|
host: 0.0.0.0
|
|
port: my-port
|
|
database: my_db
|
|
user: my_user
|
|
password: my_pass
|
|
queryTimeout: 45s
|
|
`,
|
|
want: server.SourceConfigs{
|
|
"my-mindsdb-instance": mindsdb.Config{
|
|
Name: "my-mindsdb-instance",
|
|
Kind: mindsdb.SourceKind,
|
|
Host: "0.0.0.0",
|
|
Port: "my-port",
|
|
Database: "my_db",
|
|
User: "my_user",
|
|
Password: "my_pass",
|
|
QueryTimeout: "45s",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
for _, tc := range tcs {
|
|
t.Run(tc.desc, func(t *testing.T) {
|
|
got := struct {
|
|
Sources server.SourceConfigs `yaml:"sources"`
|
|
}{}
|
|
// Parse contents
|
|
err := yaml.Unmarshal(testutils.FormatYaml(tc.in), &got)
|
|
if err != nil {
|
|
t.Fatalf("unable to unmarshal: %s", err)
|
|
}
|
|
if !cmp.Equal(tc.want, got.Sources) {
|
|
t.Fatalf("incorrect parse: want %v, got %v", tc.want, got.Sources)
|
|
}
|
|
})
|
|
}
|
|
|
|
}
|
|
|
|
func TestFailParseFromYaml(t *testing.T) {
|
|
tcs := []struct {
|
|
desc string
|
|
in string
|
|
err string
|
|
}{
|
|
{
|
|
desc: "extra field",
|
|
in: `
|
|
sources:
|
|
my-mindsdb-instance:
|
|
kind: mindsdb
|
|
host: 0.0.0.0
|
|
port: my-port
|
|
database: my_db
|
|
user: my_user
|
|
password: my_pass
|
|
foo: bar
|
|
`,
|
|
err: "unable to parse source \"my-mindsdb-instance\" as \"mindsdb\": [2:1] unknown field \"foo\"\n 1 | database: my_db\n> 2 | foo: bar\n ^\n 3 | host: 0.0.0.0\n 4 | kind: mindsdb\n 5 | password: my_pass\n 6 | ",
|
|
},
|
|
{
|
|
desc: "missing required field",
|
|
in: `
|
|
sources:
|
|
my-mindsdb-instance:
|
|
kind: mindsdb
|
|
port: my-port
|
|
database: my_db
|
|
user: my_user
|
|
password: my_pass
|
|
`,
|
|
err: "unable to parse source \"my-mindsdb-instance\" as \"mindsdb\": Key: 'Config.Host' Error:Field validation for 'Host' failed on the 'required' tag",
|
|
},
|
|
}
|
|
for _, tc := range tcs {
|
|
t.Run(tc.desc, func(t *testing.T) {
|
|
got := struct {
|
|
Sources server.SourceConfigs `yaml:"sources"`
|
|
}{}
|
|
// Parse contents
|
|
err := yaml.Unmarshal(testutils.FormatYaml(tc.in), &got)
|
|
if err == nil {
|
|
t.Fatalf("expect parsing to fail")
|
|
}
|
|
errStr := err.Error()
|
|
if errStr != tc.err {
|
|
t.Fatalf("unexpected error: got %q, want %q", errStr, tc.err)
|
|
}
|
|
})
|
|
}
|
|
}
|