mirror of
https://github.com/googleapis/genai-toolbox.git
synced 2026-01-11 16:38:15 -05:00
Compare commits
17 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2c62647c0f | ||
|
|
19465c5359 | ||
|
|
6906fa98c2 | ||
|
|
b36fa6ef4b | ||
|
|
8b0b4e6391 | ||
|
|
e908dd204a | ||
|
|
5818cacfb2 | ||
|
|
84006f0dad | ||
|
|
9acd4492d1 | ||
|
|
5a1c495187 | ||
|
|
a044c468d0 | ||
|
|
8b0a18352e | ||
|
|
53dd247e6e | ||
|
|
648eede62b | ||
|
|
9b2dfcc553 | ||
|
|
cb514209b6 | ||
|
|
0a93b0482c |
1
.github/workflows/docs_preview_deploy.yaml
vendored
1
.github/workflows/docs_preview_deploy.yaml
vendored
@@ -51,6 +51,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
fetch-depth: 0 # Fetch all history for .GitInfo and .Lastmod
|
||||
|
||||
- name: Setup Hugo
|
||||
|
||||
155
README.md
155
README.md
@@ -151,6 +151,8 @@ execute `toolbox` to start the server:
|
||||
```sh
|
||||
./toolbox --tools-file "tools.yaml"
|
||||
```
|
||||
> [!NOTE]
|
||||
> Toolbox enables dynamic reloading by default. To disable, use the `--disable-reload` flag.
|
||||
|
||||
You can use `toolbox help` for a full list of flags! To stop the server, send a
|
||||
terminate signal (`ctrl+c` on most platforms).
|
||||
@@ -165,7 +167,12 @@ Once your server is up and running, you can load the tools into your
|
||||
application. See below the list of Client SDKs for using various frameworks:
|
||||
|
||||
<details open>
|
||||
<summary>Core</summary>
|
||||
<summary>Python</summary>
|
||||
<br>
|
||||
<blockquote>
|
||||
|
||||
<details open>
|
||||
<summary>Core</summary>
|
||||
|
||||
1. Install [Toolbox Core SDK][toolbox-core]:
|
||||
|
||||
@@ -191,9 +198,9 @@ For more detailed instructions on using the Toolbox Core SDK, see the
|
||||
[toolbox-core]: https://pypi.org/project/toolbox-core/
|
||||
[toolbox-core-readme]: https://github.com/googleapis/mcp-toolbox-sdk-python/tree/main/packages/toolbox-core/README.md
|
||||
|
||||
</details>
|
||||
<details>
|
||||
<summary>LangChain / LangGraph</summary>
|
||||
</details>
|
||||
<details>
|
||||
<summary>LangChain / LangGraph</summary>
|
||||
|
||||
1. Install [Toolbox LangChain SDK][toolbox-langchain]:
|
||||
|
||||
@@ -213,16 +220,15 @@ For more detailed instructions on using the Toolbox Core SDK, see the
|
||||
tools = client.load_toolset()
|
||||
```
|
||||
|
||||
For more detailed instructions on using the Toolbox LangChain SDK, see the
|
||||
[project's README][toolbox-langchain-readme].
|
||||
For more detailed instructions on using the Toolbox LangChain SDK, see the
|
||||
[project's README][toolbox-langchain-readme].
|
||||
|
||||
[toolbox-langchain]: https://pypi.org/project/toolbox-langchain/
|
||||
[toolbox-langchain-readme]: https://github.com/googleapis/mcp-toolbox-sdk-python/blob/main/packages/toolbox-langchain/README.md
|
||||
[toolbox-langchain]: https://pypi.org/project/toolbox-langchain/
|
||||
[toolbox-langchain-readme]: https://github.com/googleapis/mcp-toolbox-sdk-python/blob/main/packages/toolbox-langchain/README.md
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary>LlamaIndex</summary>
|
||||
</details>
|
||||
<details>
|
||||
<summary>LlamaIndex</summary>
|
||||
|
||||
1. Install [Toolbox Llamaindex SDK][toolbox-llamaindex]:
|
||||
|
||||
@@ -242,12 +248,129 @@ For more detailed instructions on using the Toolbox LangChain SDK, see the
|
||||
tools = client.load_toolset()
|
||||
```
|
||||
|
||||
For more detailed instructions on using the Toolbox Llamaindex SDK, see the
|
||||
[project's README][toolbox-llamaindex-readme].
|
||||
For more detailed instructions on using the Toolbox Llamaindex SDK, see the
|
||||
[project's README][toolbox-llamaindex-readme].
|
||||
|
||||
[toolbox-llamaindex]: https://pypi.org/project/toolbox-llamaindex/
|
||||
[toolbox-llamaindex-readme]: https://github.com/googleapis/genai-toolbox-llamaindex-python/blob/main/README.md
|
||||
[toolbox-llamaindex]: https://pypi.org/project/toolbox-llamaindex/
|
||||
[toolbox-llamaindex-readme]: https://github.com/googleapis/genai-toolbox-llamaindex-python/blob/main/README.md
|
||||
|
||||
</details>
|
||||
</details>
|
||||
</blockquote>
|
||||
<details>
|
||||
<summary>Javascript/Typescript</summary>
|
||||
<br>
|
||||
<blockquote>
|
||||
|
||||
<details open>
|
||||
<summary>Core</summary>
|
||||
|
||||
1. Install [Toolbox Core SDK][toolbox-core-js]:
|
||||
|
||||
```bash
|
||||
npm install @toolbox-sdk/core
|
||||
```
|
||||
|
||||
1. Load tools:
|
||||
|
||||
```javascript
|
||||
import { ToolboxClient } from '@toolbox-sdk/core';
|
||||
|
||||
// update the url to point to your server
|
||||
const URL = 'http://127.0.0.1:5000';
|
||||
let client = new ToolboxClient(URL);
|
||||
|
||||
// these tools can be passed to your application!
|
||||
const tools = await client.loadToolset('toolsetName');
|
||||
```
|
||||
|
||||
For more detailed instructions on using the Toolbox Core SDK, see the
|
||||
[project's README][toolbox-core-js-readme].
|
||||
|
||||
[toolbox-core-js]: https://www.npmjs.com/package/@toolbox-sdk/core
|
||||
[toolbox-core-js-readme]: https://github.com/googleapis/mcp-toolbox-sdk-js/blob/main/packages/toolbox-core/README.md
|
||||
|
||||
</details>
|
||||
<details>
|
||||
<summary>LangChain / LangGraph</summary>
|
||||
|
||||
1. Install [Toolbox Core SDK][toolbox-core-js]:
|
||||
|
||||
```bash
|
||||
npm install @toolbox-sdk/core
|
||||
```
|
||||
|
||||
2. Load tools:
|
||||
|
||||
```javascript
|
||||
import { ToolboxClient } from '@toolbox-sdk/core';
|
||||
|
||||
// update the url to point to your server
|
||||
const URL = 'http://127.0.0.1:5000';
|
||||
let client = new ToolboxClient(URL);
|
||||
|
||||
// these tools can be passed to your application!
|
||||
const toolboxTools = await client.loadToolset('toolsetName');
|
||||
|
||||
// Define the basics of the tool: name, description, schema and core logic
|
||||
const getTool = (toolboxTool) => tool(currTool, {
|
||||
name: toolboxTool.getName(),
|
||||
description: toolboxTool.getDescription(),
|
||||
schema: toolboxTool.getParamSchema()
|
||||
});
|
||||
|
||||
// Use these tools in your Langchain/Langraph applications
|
||||
const tools = toolboxTools.map(getTool);
|
||||
```
|
||||
|
||||
</details>
|
||||
<details>
|
||||
<summary>Genkit</summary>
|
||||
|
||||
1. Install [Toolbox Core SDK][toolbox-core-js]:
|
||||
|
||||
```bash
|
||||
npm install @toolbox-sdk/core
|
||||
```
|
||||
|
||||
2. Load tools:
|
||||
|
||||
```javascript
|
||||
import { ToolboxClient } from '@toolbox-sdk/core';
|
||||
import { genkit } from 'genkit';
|
||||
|
||||
// Initialise genkit
|
||||
const ai = genkit({
|
||||
plugins: [
|
||||
googleAI({
|
||||
apiKey: process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY
|
||||
})
|
||||
],
|
||||
model: googleAI.model('gemini-2.0-flash'),
|
||||
});
|
||||
|
||||
// update the url to point to your server
|
||||
const URL = 'http://127.0.0.1:5000';
|
||||
let client = new ToolboxClient(URL);
|
||||
|
||||
// these tools can be passed to your application!
|
||||
const toolboxTools = await client.loadToolset('toolsetName');
|
||||
|
||||
// Define the basics of the tool: name, description, schema and core logic
|
||||
const getTool = (toolboxTool) => ai.defineTool({
|
||||
name: toolboxTool.getName(),
|
||||
description: toolboxTool.getDescription(),
|
||||
schema: toolboxTool.getParamSchema()
|
||||
}, toolboxTool)
|
||||
|
||||
// Use these tools in your Genkit applications
|
||||
const tools = toolboxTools.map(getTool);
|
||||
```
|
||||
|
||||
</details>
|
||||
</details>
|
||||
</blockquote>
|
||||
|
||||
</details>
|
||||
|
||||
## Configuration
|
||||
|
||||
233
cmd/root.go
233
cmd/root.go
@@ -19,20 +19,26 @@ import (
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/fsnotify/fsnotify"
|
||||
yaml "github.com/goccy/go-yaml"
|
||||
"github.com/googleapis/genai-toolbox/internal/auth"
|
||||
"github.com/googleapis/genai-toolbox/internal/log"
|
||||
"github.com/googleapis/genai-toolbox/internal/prebuiltconfigs"
|
||||
"github.com/googleapis/genai-toolbox/internal/server"
|
||||
"github.com/googleapis/genai-toolbox/internal/sources"
|
||||
"github.com/googleapis/genai-toolbox/internal/telemetry"
|
||||
"github.com/googleapis/genai-toolbox/internal/tools"
|
||||
"github.com/googleapis/genai-toolbox/internal/util"
|
||||
|
||||
// Import tool packages for side effect of registration
|
||||
@@ -178,6 +184,7 @@ func NewCommand(opts ...Option) *Command {
|
||||
flags.StringVar(&cmd.cfg.TelemetryServiceName, "telemetry-service-name", "toolbox", "Sets the value of the service.name resource attribute for telemetry data.")
|
||||
flags.StringVar(&cmd.prebuiltConfig, "prebuilt", "", "Use a prebuilt tool configuration by source type. Cannot be used with --tools-file. Allowed: 'alloydb-postgres', 'bigquery', 'cloud-sql-mysql', 'cloud-sql-postgres', 'cloud-sql-mssql', 'postgres', 'spanner', 'spanner-postgres'.")
|
||||
flags.BoolVar(&cmd.cfg.Stdio, "stdio", false, "Listens via MCP STDIO instead of acting as a remote HTTP server.")
|
||||
flags.BoolVar(&cmd.cfg.DisableReload, "disable-reload", false, "Disables dynamic reloading of tools file.")
|
||||
|
||||
// wrap RunE command so that we have access to original Command object
|
||||
cmd.RunE = func(*cobra.Command, []string) error { return run(cmd) }
|
||||
@@ -347,7 +354,7 @@ func loadAndMergeToolsFolder(ctx context.Context, folderPath string) (ToolsFile,
|
||||
|
||||
// Combine both file lists
|
||||
allFiles := append(yamlFiles, ymlFiles...)
|
||||
|
||||
|
||||
if len(allFiles) == 0 {
|
||||
return ToolsFile{}, fmt.Errorf("no YAML files found in directory %q", folderPath)
|
||||
}
|
||||
@@ -356,6 +363,177 @@ func loadAndMergeToolsFolder(ctx context.Context, folderPath string) (ToolsFile,
|
||||
return loadAndMergeToolsFiles(ctx, allFiles)
|
||||
}
|
||||
|
||||
func handleDynamicReload(ctx context.Context, toolsFile ToolsFile, s *server.Server) error {
|
||||
logger, err := util.LoggerFromContext(ctx)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
sourcesMap, authServicesMap, toolsMap, toolsetsMap, err := validateReloadEdits(ctx, toolsFile)
|
||||
if err != nil {
|
||||
errMsg := fmt.Errorf("unable to validate reloaded edits: %w", err)
|
||||
logger.WarnContext(ctx, errMsg.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
s.ResourceMgr.SetResources(sourcesMap, authServicesMap, toolsMap, toolsetsMap)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateReloadEdits checks that the reloaded tools file configs can initialized without failing
|
||||
func validateReloadEdits(
|
||||
ctx context.Context, toolsFile ToolsFile,
|
||||
) (map[string]sources.Source, map[string]auth.AuthService, map[string]tools.Tool, map[string]tools.Toolset, error,
|
||||
) {
|
||||
logger, err := util.LoggerFromContext(ctx)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
instrumentation, err := util.InstrumentationFromContext(ctx)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
logger.DebugContext(ctx, "Attempting to parse and validate reloaded tools file.")
|
||||
|
||||
ctx, span := instrumentation.Tracer.Start(ctx, "toolbox/server/reload")
|
||||
defer span.End()
|
||||
|
||||
reloadedConfig := server.ServerConfig{
|
||||
Version: versionString,
|
||||
SourceConfigs: toolsFile.Sources,
|
||||
AuthServiceConfigs: toolsFile.AuthServices,
|
||||
ToolConfigs: toolsFile.Tools,
|
||||
ToolsetConfigs: toolsFile.Toolsets,
|
||||
}
|
||||
|
||||
sourcesMap, authServicesMap, toolsMap, toolsetsMap, err := server.InitializeConfigs(ctx, reloadedConfig)
|
||||
if err != nil {
|
||||
errMsg := fmt.Errorf("unable to initialize reloaded configs: %w", err)
|
||||
logger.WarnContext(ctx, errMsg.Error())
|
||||
return nil, nil, nil, nil, err
|
||||
}
|
||||
|
||||
return sourcesMap, authServicesMap, toolsMap, toolsetsMap, nil
|
||||
}
|
||||
|
||||
// watchChanges checks for changes in the provided yaml tools file(s) or folder.
|
||||
func watchChanges(ctx context.Context, watchDirs map[string]bool, watchedFiles map[string]bool, s *server.Server) {
|
||||
logger, err := util.LoggerFromContext(ctx)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
w, err := fsnotify.NewWatcher()
|
||||
if err != nil {
|
||||
logger.WarnContext(ctx, "error setting up new watcher %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
defer w.Close()
|
||||
|
||||
watchingFolder := false
|
||||
var folderToWatch string
|
||||
|
||||
// if watchedFiles is empty, indicates that user passed entire folder instead
|
||||
if len(watchedFiles) == 0 {
|
||||
watchingFolder = true
|
||||
|
||||
// validate that watchDirs only has single element
|
||||
if len(watchDirs) > 1 {
|
||||
logger.WarnContext(ctx, "error setting watcher, expected single tools folder if no file(s) are defined.")
|
||||
return
|
||||
}
|
||||
|
||||
for onlyKey := range watchDirs {
|
||||
folderToWatch = onlyKey
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
for dir := range watchDirs {
|
||||
err := w.Add(dir)
|
||||
if err != nil {
|
||||
logger.WarnContext(ctx, fmt.Sprintf("Error adding path %s to watcher: %s", dir, err))
|
||||
break
|
||||
}
|
||||
logger.DebugContext(ctx, fmt.Sprintf("Added directory %s to watcher.", dir))
|
||||
}
|
||||
|
||||
// debounce timer is used to prevent multiple writes triggering multiple reloads
|
||||
debounceDelay := 100 * time.Millisecond
|
||||
debounce := time.NewTimer(1 * time.Minute)
|
||||
debounce.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
logger.DebugContext(ctx, "file watcher context cancelled")
|
||||
return
|
||||
case err, ok := <-w.Errors:
|
||||
if !ok {
|
||||
logger.WarnContext(ctx, "file watcher was closed unexpectedly")
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
logger.WarnContext(ctx, "file watcher error %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
case e, ok := <-w.Events:
|
||||
if !ok {
|
||||
logger.WarnContext(ctx, "file watcher already closed")
|
||||
return
|
||||
}
|
||||
|
||||
// only check for write events which indicate user saved a new tools file
|
||||
if !e.Has(fsnotify.Write) {
|
||||
continue
|
||||
}
|
||||
|
||||
cleanedFilename := filepath.Clean(e.Name)
|
||||
logger.DebugContext(ctx, fmt.Sprintf("WRITE event detected in %s", cleanedFilename))
|
||||
|
||||
folderChanged := watchingFolder &&
|
||||
(strings.HasSuffix(cleanedFilename, ".yaml") || strings.HasSuffix(cleanedFilename, ".yml"))
|
||||
|
||||
if folderChanged || watchedFiles[cleanedFilename] {
|
||||
// indicates the write event is on a relevant file
|
||||
debounce.Reset(debounceDelay)
|
||||
}
|
||||
|
||||
case <-debounce.C:
|
||||
debounce.Stop()
|
||||
var reloadedToolsFile ToolsFile
|
||||
|
||||
if watchingFolder {
|
||||
logger.DebugContext(ctx, "Reloading tools folder.")
|
||||
reloadedToolsFile, err = loadAndMergeToolsFolder(ctx, folderToWatch)
|
||||
if err != nil {
|
||||
logger.WarnContext(ctx, "error loading tools folder %s", err)
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
logger.DebugContext(ctx, "Reloading tools file(s).")
|
||||
reloadedToolsFile, err = loadAndMergeToolsFiles(ctx, slices.Collect(maps.Keys(watchedFiles)))
|
||||
if err != nil {
|
||||
logger.WarnContext(ctx, "error loading tools files %s", err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
err = handleDynamicReload(ctx, reloadedToolsFile, s)
|
||||
if err != nil {
|
||||
errMsg := fmt.Errorf("unable to parse reloaded tools file at %q: %w", reloadedToolsFile, err)
|
||||
logger.WarnContext(ctx, errMsg.Error())
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// updateLogLevel checks if Toolbox have to update the existing log level set by users.
|
||||
// stdio doesn't support "debug" and "info" logs.
|
||||
func updateLogLevel(stdio bool, logLevel string) bool {
|
||||
@@ -370,6 +548,33 @@ func updateLogLevel(stdio bool, logLevel string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func resolveWatcherInputs(toolsFile string, toolsFiles []string, toolsFolder string) (map[string]bool, map[string]bool) {
|
||||
var relevantFiles []string
|
||||
|
||||
// map for efficiently checking if a file is relevant
|
||||
watchedFiles := make(map[string]bool)
|
||||
|
||||
// dirs that will be added to watcher (fsnotify prefers watching directory then filtering for file)
|
||||
watchDirs := make(map[string]bool)
|
||||
|
||||
if len(toolsFiles) > 0 {
|
||||
relevantFiles = toolsFiles
|
||||
} else if toolsFolder != "" {
|
||||
watchDirs[filepath.Clean(toolsFolder)] = true
|
||||
} else {
|
||||
relevantFiles = []string{toolsFile}
|
||||
}
|
||||
|
||||
// extract parent dir for relevant files and dedup
|
||||
for _, f := range relevantFiles {
|
||||
cleanFile := filepath.Clean(f)
|
||||
watchedFiles[cleanFile] = true
|
||||
watchDirs[filepath.Dir(cleanFile)] = true
|
||||
}
|
||||
|
||||
return watchDirs, watchedFiles
|
||||
}
|
||||
|
||||
func run(cmd *Command) error {
|
||||
if updateLogLevel(cmd.cfg.Stdio, cmd.cfg.LogLevel.String()) {
|
||||
cmd.cfg.LogLevel = server.StringLevel(log.Warn)
|
||||
@@ -466,6 +671,7 @@ func run(cmd *Command) error {
|
||||
cmd.logger.ErrorContext(ctx, errMsg.Error())
|
||||
return errMsg
|
||||
}
|
||||
|
||||
// Use multiple tools files
|
||||
cmd.logger.InfoContext(ctx, fmt.Sprintf("Loading and merging %d tool configuration files", len(cmd.tools_files)))
|
||||
var err error
|
||||
@@ -481,6 +687,7 @@ func run(cmd *Command) error {
|
||||
cmd.logger.ErrorContext(ctx, errMsg.Error())
|
||||
return errMsg
|
||||
}
|
||||
|
||||
// Use tools folder
|
||||
cmd.logger.InfoContext(ctx, fmt.Sprintf("Loading and merging all YAML files from directory: %s", cmd.tools_folder))
|
||||
var err error
|
||||
@@ -494,6 +701,7 @@ func run(cmd *Command) error {
|
||||
if cmd.tools_file == "" {
|
||||
cmd.tools_file = "tools.yaml"
|
||||
}
|
||||
|
||||
// Read single tool file contents
|
||||
buf, err := os.ReadFile(cmd.tools_file)
|
||||
if err != nil {
|
||||
@@ -516,9 +724,23 @@ func run(cmd *Command) error {
|
||||
cmd.logger.WarnContext(ctx, "`authSources` is deprecated, use `authServices` instead")
|
||||
cmd.cfg.AuthServiceConfigs = authSourceConfigs
|
||||
}
|
||||
if err != nil {
|
||||
errMsg := fmt.Errorf("unable to parse tool file at %q: %w", cmd.tools_file, err)
|
||||
cmd.logger.ErrorContext(ctx, errMsg.Error())
|
||||
return errMsg
|
||||
}
|
||||
|
||||
instrumentation, err := telemetry.CreateTelemetryInstrumentation(versionString)
|
||||
if err != nil {
|
||||
errMsg := fmt.Errorf("unable to create telemetry instrumentation: %w", err)
|
||||
cmd.logger.ErrorContext(ctx, errMsg.Error())
|
||||
return errMsg
|
||||
}
|
||||
|
||||
ctx = util.WithInstrumentation(ctx, instrumentation)
|
||||
|
||||
// start server
|
||||
s, err := server.NewServer(ctx, cmd.cfg, cmd.logger)
|
||||
s, err := server.NewServer(ctx, cmd.cfg)
|
||||
if err != nil {
|
||||
errMsg := fmt.Errorf("toolbox failed to initialize: %w", err)
|
||||
cmd.logger.ErrorContext(ctx, errMsg.Error())
|
||||
@@ -553,6 +775,13 @@ func run(cmd *Command) error {
|
||||
}()
|
||||
}
|
||||
|
||||
watchDirs, watchedFiles := resolveWatcherInputs(cmd.tools_file, cmd.tools_files, cmd.tools_folder)
|
||||
|
||||
if !cmd.cfg.DisableReload {
|
||||
// start watching the file(s) or folder for changes to trigger dynamic reloading
|
||||
go watchChanges(ctx, watchDirs, watchedFiles, s)
|
||||
}
|
||||
|
||||
// wait for either the server to error out or the command's context to be canceled
|
||||
select {
|
||||
case err := <-srvErr:
|
||||
|
||||
196
cmd/root_test.go
196
cmd/root_test.go
@@ -16,23 +16,33 @@ package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
|
||||
"github.com/googleapis/genai-toolbox/internal/auth/google"
|
||||
"github.com/googleapis/genai-toolbox/internal/log"
|
||||
"github.com/googleapis/genai-toolbox/internal/prebuiltconfigs"
|
||||
"github.com/googleapis/genai-toolbox/internal/server"
|
||||
cloudsqlpgsrc "github.com/googleapis/genai-toolbox/internal/sources/cloudsqlpg"
|
||||
httpsrc "github.com/googleapis/genai-toolbox/internal/sources/http"
|
||||
"github.com/googleapis/genai-toolbox/internal/telemetry"
|
||||
"github.com/googleapis/genai-toolbox/internal/testutils"
|
||||
"github.com/googleapis/genai-toolbox/internal/tools"
|
||||
"github.com/googleapis/genai-toolbox/internal/tools/http"
|
||||
"github.com/googleapis/genai-toolbox/internal/tools/postgres/postgressql"
|
||||
"github.com/googleapis/genai-toolbox/internal/util"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
@@ -174,6 +184,13 @@ func TestServerConfigFlags(t *testing.T) {
|
||||
Stdio: true,
|
||||
}),
|
||||
},
|
||||
{
|
||||
desc: "disable reload",
|
||||
args: []string{"--disable-reload"},
|
||||
want: withDefaults(server.ServerConfig{
|
||||
DisableReload: true,
|
||||
}),
|
||||
},
|
||||
}
|
||||
for _, tc := range tcs {
|
||||
t.Run(tc.desc, func(t *testing.T) {
|
||||
@@ -965,6 +982,185 @@ func TestEnvVarReplacement(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
// normalizeFilepaths is a helper function to allow same filepath formats for Mac and Windows.
|
||||
// this prevents needing multiple "want" cases for TestResolveWatcherInputs
|
||||
func normalizeFilepaths(m map[string]bool) map[string]bool {
|
||||
newMap := make(map[string]bool)
|
||||
for k, v := range m {
|
||||
newMap[filepath.ToSlash(k)] = v
|
||||
}
|
||||
return newMap
|
||||
}
|
||||
|
||||
func TestResolveWatcherInputs(t *testing.T) {
|
||||
tcs := []struct {
|
||||
description string
|
||||
toolsFile string
|
||||
toolsFiles []string
|
||||
toolsFolder string
|
||||
wantWatchDirs map[string]bool
|
||||
wantWatchedFiles map[string]bool
|
||||
}{
|
||||
{
|
||||
description: "single tools file",
|
||||
toolsFile: "tools_folder/example_tools.yaml",
|
||||
toolsFiles: []string{},
|
||||
toolsFolder: "",
|
||||
wantWatchDirs: map[string]bool{"tools_folder": true},
|
||||
wantWatchedFiles: map[string]bool{"tools_folder/example_tools.yaml": true},
|
||||
},
|
||||
{
|
||||
description: "default tools file (root dir)",
|
||||
toolsFile: "tools.yaml",
|
||||
toolsFiles: []string{},
|
||||
toolsFolder: "",
|
||||
wantWatchDirs: map[string]bool{".": true},
|
||||
wantWatchedFiles: map[string]bool{"tools.yaml": true},
|
||||
},
|
||||
{
|
||||
description: "multiple files in different folders",
|
||||
toolsFile: "",
|
||||
toolsFiles: []string{"tools_folder/example_tools.yaml", "tools_folder2/example_tools.yaml"},
|
||||
toolsFolder: "",
|
||||
wantWatchDirs: map[string]bool{"tools_folder": true, "tools_folder2": true},
|
||||
wantWatchedFiles: map[string]bool{
|
||||
"tools_folder/example_tools.yaml": true,
|
||||
"tools_folder2/example_tools.yaml": true,
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "multiple files in same folder",
|
||||
toolsFile: "",
|
||||
toolsFiles: []string{"tools_folder/example_tools.yaml", "tools_folder/example_tools2.yaml"},
|
||||
toolsFolder: "",
|
||||
wantWatchDirs: map[string]bool{"tools_folder": true},
|
||||
wantWatchedFiles: map[string]bool{
|
||||
"tools_folder/example_tools.yaml": true,
|
||||
"tools_folder/example_tools2.yaml": true,
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "multiple files in different levels",
|
||||
toolsFile: "",
|
||||
toolsFiles: []string{
|
||||
"tools_folder/example_tools.yaml",
|
||||
"tools_folder/special_tools/example_tools2.yaml"},
|
||||
toolsFolder: "",
|
||||
wantWatchDirs: map[string]bool{"tools_folder": true, "tools_folder/special_tools": true},
|
||||
wantWatchedFiles: map[string]bool{
|
||||
"tools_folder/example_tools.yaml": true,
|
||||
"tools_folder/special_tools/example_tools2.yaml": true,
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "tools folder",
|
||||
toolsFile: "",
|
||||
toolsFiles: []string{},
|
||||
toolsFolder: "tools_folder",
|
||||
wantWatchDirs: map[string]bool{"tools_folder": true},
|
||||
wantWatchedFiles: map[string]bool{},
|
||||
},
|
||||
}
|
||||
for _, tc := range tcs {
|
||||
t.Run(tc.description, func(t *testing.T) {
|
||||
gotWatchDirs, gotWatchedFiles := resolveWatcherInputs(tc.toolsFile, tc.toolsFiles, tc.toolsFolder)
|
||||
|
||||
normalizedGotWatchDirs := normalizeFilepaths(gotWatchDirs)
|
||||
normalizedGotWatchedFiles := normalizeFilepaths(gotWatchedFiles)
|
||||
|
||||
if diff := cmp.Diff(tc.wantWatchDirs, normalizedGotWatchDirs); diff != "" {
|
||||
t.Errorf("incorrect watchDirs: diff %v", diff)
|
||||
}
|
||||
if diff := cmp.Diff(tc.wantWatchedFiles, normalizedGotWatchedFiles); diff != "" {
|
||||
t.Errorf("incorrect watchedFiles: diff %v", diff)
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// helper function for testing file detection in dynamic reloading
|
||||
func tmpFileWithCleanup(content []byte) (string, func(), error) {
|
||||
f, err := os.CreateTemp("", "*")
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
cleanup := func() { os.Remove(f.Name()) }
|
||||
|
||||
if _, err := f.Write(content); err != nil {
|
||||
cleanup()
|
||||
return "", nil, err
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
cleanup()
|
||||
return "", nil, err
|
||||
}
|
||||
return f.Name(), cleanup, err
|
||||
}
|
||||
|
||||
func TestSingleEdit(t *testing.T) {
|
||||
ctx, cancelCtx := context.WithTimeout(context.Background(), time.Minute)
|
||||
defer cancelCtx()
|
||||
|
||||
pr, pw := io.Pipe()
|
||||
defer pw.Close()
|
||||
defer pr.Close()
|
||||
|
||||
fileToWatch, cleanup, err := tmpFileWithCleanup([]byte("initial content"))
|
||||
if err != nil {
|
||||
t.Fatalf("error editing tools file %s", err)
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
logger, err := log.NewStdLogger(pw, pw, "DEBUG")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to setup logger %s", err)
|
||||
}
|
||||
ctx = util.WithLogger(ctx, logger)
|
||||
|
||||
instrumentation, err := telemetry.CreateTelemetryInstrumentation(versionString)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to setup instrumentation %s", err)
|
||||
}
|
||||
ctx = util.WithInstrumentation(ctx, instrumentation)
|
||||
|
||||
mockServer := &server.Server{}
|
||||
|
||||
cleanFileToWatch := filepath.Clean(fileToWatch)
|
||||
watchDir := filepath.Dir(cleanFileToWatch)
|
||||
|
||||
watchedFiles := map[string]bool{cleanFileToWatch: true}
|
||||
watchDirs := map[string]bool{watchDir: true}
|
||||
|
||||
go watchChanges(ctx, watchDirs, watchedFiles, mockServer)
|
||||
|
||||
// escape backslash so regex doesn't fail on windows filepaths
|
||||
regexEscapedPathFile := strings.ReplaceAll(cleanFileToWatch, `\`, `\\\\*\\`)
|
||||
regexEscapedPathFile = path.Clean(regexEscapedPathFile)
|
||||
|
||||
regexEscapedPathDir := strings.ReplaceAll(watchDir, `\`, `\\\\*\\`)
|
||||
regexEscapedPathDir = path.Clean(regexEscapedPathDir)
|
||||
|
||||
begunWatchingDir := regexp.MustCompile(fmt.Sprintf(`DEBUG "Added directory %s to watcher."`, regexEscapedPathDir))
|
||||
res, err := testutils.WaitForString(ctx, begunWatchingDir, pr)
|
||||
t.Log("log result: ", res)
|
||||
if err != nil {
|
||||
t.Fatalf("timeout or error waiting for watcher to start: %s, actually got: ", err)
|
||||
}
|
||||
|
||||
err = os.WriteFile(fileToWatch, []byte("modification"), 0777)
|
||||
if err != nil {
|
||||
t.Fatalf("error writing to file: %v", err)
|
||||
}
|
||||
|
||||
detectedFileChange := regexp.MustCompile(fmt.Sprintf(`DEBUG "WRITE event detected in %s"`, regexEscapedPathFile))
|
||||
res, err = testutils.WaitForString(ctx, detectedFileChange, pr)
|
||||
t.Log("log result: ", res)
|
||||
if err != nil {
|
||||
t.Fatalf("timeout or error waiting for file to detect write: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrebuiltTools(t *testing.T) {
|
||||
alloydb_config, _ := prebuiltconfigs.Get("alloydb-postgres")
|
||||
bigquery_config, _ := prebuiltconfigs.Get("bigquery")
|
||||
|
||||
@@ -123,6 +123,9 @@ execute `toolbox` to start the server:
|
||||
```sh
|
||||
./toolbox --tools-file "tools.yaml"
|
||||
```
|
||||
{{< notice note >}}
|
||||
Toolbox enables dynamic reloading by default. To disable, use the `--disable-reload` flag.
|
||||
{{< /notice >}}
|
||||
|
||||
You can use `toolbox help` for a full list of flags! To stop the server, send a
|
||||
terminate signal (`ctrl+c` on most platforms).
|
||||
@@ -135,6 +138,7 @@ out the resources in the [How-to section](../../how-to/_index.md)
|
||||
Once your server is up and running, you can load the tools into your
|
||||
application. See below the list of Client SDKs for using various frameworks:
|
||||
|
||||
#### Python
|
||||
{{< tabpane text=true persist=header >}}
|
||||
{{% tab header="Core" lang="en" %}}
|
||||
|
||||
@@ -201,3 +205,115 @@ For more detailed instructions on using the Toolbox Llamaindex SDK, see the
|
||||
|
||||
{{% /tab %}}
|
||||
{{< /tabpane >}}
|
||||
|
||||
#### Javascript/Typescript
|
||||
|
||||
Once you've installed the [Toolbox Core
|
||||
SDK](https://www.npmjs.com/package/@toolbox-sdk/core), you can load
|
||||
tools:
|
||||
|
||||
{{< tabpane text=true persist=header >}}
|
||||
{{% tab header="Core" lang="en" %}}
|
||||
|
||||
{{< highlight javascript >}}
|
||||
import { ToolboxClient } from '@toolbox-sdk/core';
|
||||
|
||||
// update the url to point to your server
|
||||
const URL = 'http://127.0.0.1:5000';
|
||||
let client = new ToolboxClient(URL);
|
||||
|
||||
// these tools can be passed to your application!
|
||||
const toolboxTools = await client.loadToolset('toolsetName');
|
||||
{{< /highlight >}}
|
||||
|
||||
{{% /tab %}}
|
||||
{{% tab header="LangChain/Langraph" lang="en" %}}
|
||||
|
||||
{{< highlight javascript >}}
|
||||
import { ToolboxClient } from '@toolbox-sdk/core';
|
||||
|
||||
// update the url to point to your server
|
||||
const URL = 'http://127.0.0.1:5000';
|
||||
let client = new ToolboxClient(URL);
|
||||
|
||||
// these tools can be passed to your application!
|
||||
const toolboxTools = await client.loadToolset('toolsetName');
|
||||
|
||||
// Define the basics of the tool: name, description, schema and core logic
|
||||
const getTool = (toolboxTool) => tool(currTool, {
|
||||
name: toolboxTool.getName(),
|
||||
description: toolboxTool.getDescription(),
|
||||
schema: toolboxTool.getParamSchema()
|
||||
});
|
||||
|
||||
// Use these tools in your Langchain/Langraph applications
|
||||
const tools = toolboxTools.map(getTool);
|
||||
{{< /highlight >}}
|
||||
|
||||
{{% /tab %}}
|
||||
{{% tab header="Genkit" lang="en" %}}
|
||||
|
||||
{{< highlight javascript >}}
|
||||
import { ToolboxClient } from '@toolbox-sdk/core';
|
||||
import { genkit } from 'genkit';
|
||||
|
||||
// Initialise genkit
|
||||
const ai = genkit({
|
||||
plugins: [
|
||||
googleAI({
|
||||
apiKey: process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY
|
||||
})
|
||||
],
|
||||
model: googleAI.model('gemini-2.0-flash'),
|
||||
});
|
||||
|
||||
// update the url to point to your server
|
||||
const URL = 'http://127.0.0.1:5000';
|
||||
let client = new ToolboxClient(URL);
|
||||
|
||||
// these tools can be passed to your application!
|
||||
const toolboxTools = await client.loadToolset('toolsetName');
|
||||
|
||||
// Define the basics of the tool: name, description, schema and core logic
|
||||
const getTool = (toolboxTool) => ai.defineTool({
|
||||
name: toolboxTool.getName(),
|
||||
description: toolboxTool.getDescription(),
|
||||
schema: toolboxTool.getParamSchema()
|
||||
}, toolboxTool)
|
||||
|
||||
// Use these tools in your Genkit applications
|
||||
const tools = toolboxTools.map(getTool);
|
||||
{{< /highlight >}}
|
||||
|
||||
{{% /tab %}}
|
||||
{{% tab header="LlamaIndex" lang="en" %}}
|
||||
|
||||
{{< highlight javascript >}}
|
||||
import { ToolboxClient } from '@toolbox-sdk/core';
|
||||
import { tool } from "llamaindex";
|
||||
|
||||
// update the url to point to your server
|
||||
const URL = 'http://127.0.0.1:5000';
|
||||
let client = new ToolboxClient(URL);
|
||||
|
||||
// these tools can be passed to your application!
|
||||
const toolboxTools = await client.loadToolset('toolsetName');
|
||||
|
||||
// Define the basics of the tool: name, description, schema and core logic
|
||||
const getTool = (toolboxTool) => tool({
|
||||
name: toolboxTool.getName(),
|
||||
description: toolboxTool.getDescription(),
|
||||
parameters: toolboxTool.getParams(),
|
||||
execute: toolboxTool
|
||||
});;
|
||||
|
||||
// Use these tools in your LlamaIndex applications
|
||||
const tools = toolboxTools.map(getTool);
|
||||
|
||||
{{< /highlight >}}
|
||||
|
||||
{{% /tab %}}
|
||||
{{< /tabpane >}}
|
||||
|
||||
For more detailed instructions on using the Toolbox Core SDK, see the
|
||||
[project's README](https://github.com/googleapis/mcp-toolbox-sdk-js/blob/main/packages/toolbox-core/README.md).
|
||||
@@ -257,6 +257,9 @@ In this section, we will download Toolbox, configure our tools in a
|
||||
```bash
|
||||
./toolbox --tools-file "tools.yaml"
|
||||
```
|
||||
{{< notice note >}}
|
||||
Toolbox enables dynamic reloading by default. To disable, use the `--disable-reload` flag.
|
||||
{{< /notice >}}
|
||||
|
||||
## Step 3: Connect your agent to Toolbox
|
||||
|
||||
|
||||
@@ -63,6 +63,10 @@ When running with stdio, Toolbox will listen via stdio instead of acting as a
|
||||
remote HTTP server. Logs will be set to the `warn` level by default. `debug` and
|
||||
`info` logs are not supported with stdio.
|
||||
|
||||
{{< notice note >}}
|
||||
Toolbox enables dynamic reloading by default. To disable, use the `--disable-reload` flag.
|
||||
{{< /notice >}}
|
||||
|
||||
### Connecting via HTTP
|
||||
|
||||
Toolbox supports the HTTP transport protocol with and without SSE.
|
||||
|
||||
@@ -292,6 +292,9 @@ to use BigQuery, and then run the Toolbox server.
|
||||
```bash
|
||||
./toolbox --tools-file "tools.yaml"
|
||||
```
|
||||
{{< notice note >}}
|
||||
Toolbox enables dynamic reloading by default. To disable, use the `--disable-reload` flag.
|
||||
{{< /notice >}}
|
||||
|
||||
## Step 3: Connect your agent to Toolbox
|
||||
|
||||
|
||||
3
go.mod
3
go.mod
@@ -14,6 +14,7 @@ require (
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.29.0
|
||||
github.com/couchbase/gocb/v2 v2.10.0
|
||||
github.com/couchbase/tools-common/http v1.0.9
|
||||
github.com/fsnotify/fsnotify v1.9.0
|
||||
github.com/go-chi/chi/v5 v5.2.2
|
||||
github.com/go-chi/httplog/v2 v2.1.1
|
||||
github.com/go-chi/render v1.0.3
|
||||
@@ -38,7 +39,7 @@ require (
|
||||
go.opentelemetry.io/otel/sdk/metric v1.36.0
|
||||
go.opentelemetry.io/otel/trace v1.36.0
|
||||
golang.org/x/oauth2 v0.30.0
|
||||
google.golang.org/api v0.239.0
|
||||
google.golang.org/api v0.240.0
|
||||
modernc.org/sqlite v1.38.0
|
||||
)
|
||||
|
||||
|
||||
6
go.sum
6
go.sum
@@ -765,6 +765,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
|
||||
github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
|
||||
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||
github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM=
|
||||
github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
@@ -1605,8 +1607,8 @@ google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/
|
||||
google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI=
|
||||
google.golang.org/api v0.111.0/go.mod h1:qtFHvU9mhgTJegR31csQ+rwxyUTHOKFqCKWp1J0fdw0=
|
||||
google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg=
|
||||
google.golang.org/api v0.239.0 h1:2hZKUnFZEy81eugPs4e2XzIJ5SOwQg0G82bpXD65Puo=
|
||||
google.golang.org/api v0.239.0/go.mod h1:cOVEm2TpdAGHL2z+UwyS+kmlGr3bVWQQ6sYEqkKje50=
|
||||
google.golang.org/api v0.240.0 h1:PxG3AA2UIqT1ofIzWV2COM3j3JagKTKSwy7L6RHNXNU=
|
||||
google.golang.org/api v0.240.0/go.mod h1:cOVEm2TpdAGHL2z+UwyS+kmlGr3bVWQQ6sYEqkKje50=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
|
||||
@@ -75,7 +75,7 @@ func toolsetHandler(s *Server, w http.ResponseWriter, r *http.Request) {
|
||||
)
|
||||
}()
|
||||
|
||||
toolset, ok := s.toolsets[toolsetName]
|
||||
toolset, ok := s.ResourceMgr.GetToolset(toolsetName)
|
||||
if !ok {
|
||||
err = fmt.Errorf("toolset %q does not exist", toolsetName)
|
||||
s.logger.DebugContext(ctx, err.Error())
|
||||
@@ -111,7 +111,7 @@ func toolGetHandler(s *Server, w http.ResponseWriter, r *http.Request) {
|
||||
metric.WithAttributes(attribute.String("toolbox.operation.status", status)),
|
||||
)
|
||||
}()
|
||||
tool, ok := s.tools[toolName]
|
||||
tool, ok := s.ResourceMgr.GetTool(toolName)
|
||||
if !ok {
|
||||
err = fmt.Errorf("invalid tool name: tool with name %q does not exist", toolName)
|
||||
s.logger.DebugContext(ctx, err.Error())
|
||||
@@ -156,7 +156,7 @@ func toolInvokeHandler(s *Server, w http.ResponseWriter, r *http.Request) {
|
||||
)
|
||||
}()
|
||||
|
||||
tool, ok := s.tools[toolName]
|
||||
tool, ok := s.ResourceMgr.GetTool(toolName)
|
||||
if !ok {
|
||||
err = fmt.Errorf("invalid tool name: tool with name %q does not exist", toolName)
|
||||
s.logger.DebugContext(ctx, err.Error())
|
||||
@@ -167,7 +167,7 @@ func toolInvokeHandler(s *Server, w http.ResponseWriter, r *http.Request) {
|
||||
// Tool authentication
|
||||
// claimsFromAuth maps the name of the authservice to the claims retrieved from it.
|
||||
claimsFromAuth := make(map[string]map[string]any)
|
||||
for _, aS := range s.authServices {
|
||||
for _, aS := range s.ResourceMgr.GetAuthServiceMap() {
|
||||
claims, err := aS.GetClaimsFromHeader(ctx, r.Header)
|
||||
if err != nil {
|
||||
s.logger.DebugContext(ctx, err.Error())
|
||||
|
||||
@@ -147,14 +147,23 @@ func setUpServer(t *testing.T, router string, tools map[string]tools.Tool, tools
|
||||
t.Fatalf("unable to setup otel: %s", err)
|
||||
}
|
||||
|
||||
instrumentation, err := CreateTelemetryInstrumentation(fakeVersionString)
|
||||
instrumentation, err := telemetry.CreateTelemetryInstrumentation(fakeVersionString)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to create custom metrics: %s", err)
|
||||
}
|
||||
|
||||
sseManager := newSseManager(ctx)
|
||||
|
||||
server := Server{version: fakeVersionString, logger: testLogger, instrumentation: instrumentation, sseManager: sseManager, tools: tools, toolsets: toolsets}
|
||||
resourceManager := NewResourceManager(nil, nil, tools, toolsets)
|
||||
|
||||
server := Server{
|
||||
version: fakeVersionString,
|
||||
logger: testLogger,
|
||||
instrumentation: instrumentation,
|
||||
sseManager: sseManager,
|
||||
ResourceMgr: resourceManager,
|
||||
}
|
||||
|
||||
var r chi.Router
|
||||
switch router {
|
||||
case "api":
|
||||
|
||||
@@ -53,6 +53,8 @@ type ServerConfig struct {
|
||||
TelemetryServiceName string
|
||||
// Stdio indicates if Toolbox is listening via MCP stdio.
|
||||
Stdio bool
|
||||
// DisableReload indicates if the user has disabled dynamic reloading for Toolbox.
|
||||
DisableReload bool
|
||||
}
|
||||
|
||||
type logFormat string
|
||||
|
||||
@@ -479,12 +479,12 @@ func processMcpMessage(ctx context.Context, body []byte, s *Server, protocolVers
|
||||
}
|
||||
return v, res, err
|
||||
default:
|
||||
toolset, ok := s.toolsets[toolsetName]
|
||||
toolset, ok := s.ResourceMgr.GetToolset(toolsetName)
|
||||
if !ok {
|
||||
err = fmt.Errorf("toolset does not exist")
|
||||
return "", jsonrpc.NewError(baseMessage.Id, jsonrpc.INVALID_REQUEST, err.Error(), nil), err
|
||||
}
|
||||
res, err := mcp.ProcessMethod(ctx, protocolVersion, baseMessage.Id, baseMessage.Method, toolset, s.tools, body)
|
||||
res, err := mcp.ProcessMethod(ctx, protocolVersion, baseMessage.Id, baseMessage.Method, toolset, s.ResourceMgr.GetToolsMap(), body)
|
||||
return "", res, err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -693,14 +693,22 @@ func TestStdioSession(t *testing.T) {
|
||||
}
|
||||
}()
|
||||
|
||||
instrumentation, err := CreateTelemetryInstrumentation(fakeVersionString)
|
||||
instrumentation, err := telemetry.CreateTelemetryInstrumentation(fakeVersionString)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to create custom metrics: %s", err)
|
||||
}
|
||||
|
||||
sseManager := newSseManager(ctx)
|
||||
|
||||
server := &Server{version: fakeVersionString, logger: testLogger, instrumentation: instrumentation, sseManager: sseManager, tools: toolsMap, toolsets: toolsets}
|
||||
resourceManager := NewResourceManager(nil, nil, toolsMap, toolsets)
|
||||
|
||||
server := &Server{
|
||||
version: fakeVersionString,
|
||||
logger: testLogger,
|
||||
instrumentation: instrumentation,
|
||||
sseManager: sseManager,
|
||||
ResourceMgr: resourceManager,
|
||||
}
|
||||
|
||||
in := bufio.NewReader(pr)
|
||||
stdioSession := NewStdioSession(server, in, pw)
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
@@ -29,6 +30,7 @@ import (
|
||||
"github.com/googleapis/genai-toolbox/internal/auth"
|
||||
"github.com/googleapis/genai-toolbox/internal/log"
|
||||
"github.com/googleapis/genai-toolbox/internal/sources"
|
||||
"github.com/googleapis/genai-toolbox/internal/telemetry"
|
||||
"github.com/googleapis/genai-toolbox/internal/tools"
|
||||
"github.com/googleapis/genai-toolbox/internal/util"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
@@ -42,26 +44,225 @@ type Server struct {
|
||||
listener net.Listener
|
||||
root chi.Router
|
||||
logger log.Logger
|
||||
instrumentation *Instrumentation
|
||||
instrumentation *telemetry.Instrumentation
|
||||
sseManager *sseManager
|
||||
ResourceMgr *ResourceManager
|
||||
}
|
||||
|
||||
// ResourceManager contains available resources for the server. Should be initialized with NewResourceManager().
|
||||
type ResourceManager struct {
|
||||
mu sync.RWMutex
|
||||
sources map[string]sources.Source
|
||||
authServices map[string]auth.AuthService
|
||||
tools map[string]tools.Tool
|
||||
toolsets map[string]tools.Toolset
|
||||
}
|
||||
|
||||
// NewServer returns a Server object based on provided Config.
|
||||
func NewServer(ctx context.Context, cfg ServerConfig, l log.Logger) (*Server, error) {
|
||||
instrumentation, err := CreateTelemetryInstrumentation(cfg.Version)
|
||||
func NewResourceManager(
|
||||
sourcesMap map[string]sources.Source,
|
||||
authServicesMap map[string]auth.AuthService,
|
||||
toolsMap map[string]tools.Tool, toolsetsMap map[string]tools.Toolset,
|
||||
) *ResourceManager {
|
||||
resourceMgr := &ResourceManager{
|
||||
mu: sync.RWMutex{},
|
||||
sources: sourcesMap,
|
||||
authServices: authServicesMap,
|
||||
tools: toolsMap,
|
||||
toolsets: toolsetsMap,
|
||||
}
|
||||
|
||||
return resourceMgr
|
||||
}
|
||||
|
||||
func (r *ResourceManager) GetSource(sourceName string) (sources.Source, bool) {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
source, ok := r.sources[sourceName]
|
||||
return source, ok
|
||||
}
|
||||
|
||||
func (r *ResourceManager) GetAuthService(authServiceName string) (auth.AuthService, bool) {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
authService, ok := r.authServices[authServiceName]
|
||||
return authService, ok
|
||||
}
|
||||
|
||||
func (r *ResourceManager) GetTool(toolName string) (tools.Tool, bool) {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
tool, ok := r.tools[toolName]
|
||||
return tool, ok
|
||||
}
|
||||
|
||||
func (r *ResourceManager) GetToolset(toolsetName string) (tools.Toolset, bool) {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
toolset, ok := r.toolsets[toolsetName]
|
||||
return toolset, ok
|
||||
}
|
||||
|
||||
func (r *ResourceManager) SetResources(sourcesMap map[string]sources.Source, authServicesMap map[string]auth.AuthService, toolsMap map[string]tools.Tool, toolsetsMap map[string]tools.Toolset) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.sources = sourcesMap
|
||||
r.authServices = authServicesMap
|
||||
r.tools = toolsMap
|
||||
r.toolsets = toolsetsMap
|
||||
}
|
||||
|
||||
func (r *ResourceManager) GetAuthServiceMap() map[string]auth.AuthService {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
return r.authServices
|
||||
}
|
||||
|
||||
func (r *ResourceManager) GetToolsMap() map[string]tools.Tool {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
return r.tools
|
||||
}
|
||||
|
||||
func InitializeConfigs(ctx context.Context, cfg ServerConfig) (
|
||||
map[string]sources.Source,
|
||||
map[string]auth.AuthService,
|
||||
map[string]tools.Tool,
|
||||
map[string]tools.Toolset,
|
||||
error,
|
||||
) {
|
||||
ctx = util.WithUserAgent(ctx, cfg.Version)
|
||||
instrumentation, err := util.InstrumentationFromContext(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to create telemetry instrumentation: %w", err)
|
||||
panic(err)
|
||||
}
|
||||
|
||||
l, err := util.LoggerFromContext(ctx)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// initialize and validate the sources from configs
|
||||
sourcesMap := make(map[string]sources.Source)
|
||||
for name, sc := range cfg.SourceConfigs {
|
||||
s, err := func() (sources.Source, error) {
|
||||
childCtx, span := instrumentation.Tracer.Start(
|
||||
ctx,
|
||||
"toolbox/server/source/init",
|
||||
trace.WithAttributes(attribute.String("source_kind", sc.SourceConfigKind())),
|
||||
trace.WithAttributes(attribute.String("source_name", name)),
|
||||
)
|
||||
defer span.End()
|
||||
s, err := sc.Initialize(childCtx, instrumentation.Tracer)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to initialize source %q: %w", name, err)
|
||||
}
|
||||
return s, nil
|
||||
}()
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, err
|
||||
}
|
||||
sourcesMap[name] = s
|
||||
}
|
||||
l.InfoContext(ctx, fmt.Sprintf("Initialized %d sources.", len(sourcesMap)))
|
||||
|
||||
// initialize and validate the auth services from configs
|
||||
authServicesMap := make(map[string]auth.AuthService)
|
||||
for name, sc := range cfg.AuthServiceConfigs {
|
||||
a, err := func() (auth.AuthService, error) {
|
||||
_, span := instrumentation.Tracer.Start(
|
||||
ctx,
|
||||
"toolbox/server/auth/init",
|
||||
trace.WithAttributes(attribute.String("auth_kind", sc.AuthServiceConfigKind())),
|
||||
trace.WithAttributes(attribute.String("auth_name", name)),
|
||||
)
|
||||
defer span.End()
|
||||
a, err := sc.Initialize()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to initialize auth service %q: %w", name, err)
|
||||
}
|
||||
return a, nil
|
||||
}()
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, err
|
||||
}
|
||||
authServicesMap[name] = a
|
||||
}
|
||||
l.InfoContext(ctx, fmt.Sprintf("Initialized %d authServices.", len(authServicesMap)))
|
||||
|
||||
// initialize and validate the tools from configs
|
||||
toolsMap := make(map[string]tools.Tool)
|
||||
for name, tc := range cfg.ToolConfigs {
|
||||
t, err := func() (tools.Tool, error) {
|
||||
_, span := instrumentation.Tracer.Start(
|
||||
ctx,
|
||||
"toolbox/server/tool/init",
|
||||
trace.WithAttributes(attribute.String("tool_kind", tc.ToolConfigKind())),
|
||||
trace.WithAttributes(attribute.String("tool_name", name)),
|
||||
)
|
||||
defer span.End()
|
||||
t, err := tc.Initialize(sourcesMap)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to initialize tool %q: %w", name, err)
|
||||
}
|
||||
return t, nil
|
||||
}()
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, err
|
||||
}
|
||||
toolsMap[name] = t
|
||||
}
|
||||
l.InfoContext(ctx, fmt.Sprintf("Initialized %d tools.", len(toolsMap)))
|
||||
|
||||
// create a default toolset that contains all tools
|
||||
allToolNames := make([]string, 0, len(toolsMap))
|
||||
for name := range toolsMap {
|
||||
allToolNames = append(allToolNames, name)
|
||||
}
|
||||
if cfg.ToolsetConfigs == nil {
|
||||
cfg.ToolsetConfigs = make(ToolsetConfigs)
|
||||
}
|
||||
cfg.ToolsetConfigs[""] = tools.ToolsetConfig{Name: "", ToolNames: allToolNames}
|
||||
|
||||
// initialize and validate the toolsets from configs
|
||||
toolsetsMap := make(map[string]tools.Toolset)
|
||||
for name, tc := range cfg.ToolsetConfigs {
|
||||
t, err := func() (tools.Toolset, error) {
|
||||
_, span := instrumentation.Tracer.Start(
|
||||
ctx,
|
||||
"toolbox/server/toolset/init",
|
||||
trace.WithAttributes(attribute.String("toolset_name", name)),
|
||||
)
|
||||
defer span.End()
|
||||
t, err := tc.Initialize(cfg.Version, toolsMap)
|
||||
if err != nil {
|
||||
return tools.Toolset{}, fmt.Errorf("unable to initialize toolset %q: %w", name, err)
|
||||
}
|
||||
return t, err
|
||||
}()
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, err
|
||||
}
|
||||
toolsetsMap[name] = t
|
||||
}
|
||||
l.InfoContext(ctx, fmt.Sprintf("Initialized %d toolsets.", len(toolsetsMap)))
|
||||
|
||||
return sourcesMap, authServicesMap, toolsMap, toolsetsMap, nil
|
||||
}
|
||||
|
||||
// NewServer returns a Server object based on provided Config.
|
||||
func NewServer(ctx context.Context, cfg ServerConfig) (*Server, error) {
|
||||
instrumentation, err := util.InstrumentationFromContext(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ctx, span := instrumentation.Tracer.Start(ctx, "toolbox/server/init")
|
||||
defer span.End()
|
||||
|
||||
ctx = util.WithUserAgent(ctx, cfg.Version)
|
||||
l, err := util.LoggerFromContext(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// set up http serving
|
||||
r := chi.NewRouter()
|
||||
@@ -97,116 +298,18 @@ func NewServer(ctx context.Context, cfg ServerConfig, l log.Logger) (*Server, er
|
||||
httpLogger := httplog.NewLogger("httplog", httpOpts)
|
||||
r.Use(httplog.RequestLogger(httpLogger))
|
||||
|
||||
// initialize and validate the sources from configs
|
||||
sourcesMap := make(map[string]sources.Source)
|
||||
for name, sc := range cfg.SourceConfigs {
|
||||
s, err := func() (sources.Source, error) {
|
||||
childCtx, span := instrumentation.Tracer.Start(
|
||||
ctx,
|
||||
"toolbox/server/source/init",
|
||||
trace.WithAttributes(attribute.String("source_kind", sc.SourceConfigKind())),
|
||||
trace.WithAttributes(attribute.String("source_name", name)),
|
||||
)
|
||||
defer span.End()
|
||||
s, err := sc.Initialize(childCtx, instrumentation.Tracer)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to initialize source %q: %w", name, err)
|
||||
}
|
||||
return s, nil
|
||||
}()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sourcesMap[name] = s
|
||||
sourcesMap, authServicesMap, toolsMap, toolsetsMap, err := InitializeConfigs(ctx, cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to initialize configs: %w", err)
|
||||
}
|
||||
l.InfoContext(ctx, fmt.Sprintf("Initialized %d sources.", len(sourcesMap)))
|
||||
|
||||
// initialize and validate the auth services from configs
|
||||
authServicesMap := make(map[string]auth.AuthService)
|
||||
for name, sc := range cfg.AuthServiceConfigs {
|
||||
a, err := func() (auth.AuthService, error) {
|
||||
_, span := instrumentation.Tracer.Start(
|
||||
ctx,
|
||||
"toolbox/server/auth/init",
|
||||
trace.WithAttributes(attribute.String("auth_kind", sc.AuthServiceConfigKind())),
|
||||
trace.WithAttributes(attribute.String("auth_name", name)),
|
||||
)
|
||||
defer span.End()
|
||||
a, err := sc.Initialize()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to initialize auth service %q: %w", name, err)
|
||||
}
|
||||
return a, nil
|
||||
}()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
authServicesMap[name] = a
|
||||
}
|
||||
l.InfoContext(ctx, fmt.Sprintf("Initialized %d authServices.", len(authServicesMap)))
|
||||
|
||||
// initialize and validate the tools from configs
|
||||
toolsMap := make(map[string]tools.Tool)
|
||||
for name, tc := range cfg.ToolConfigs {
|
||||
t, err := func() (tools.Tool, error) {
|
||||
_, span := instrumentation.Tracer.Start(
|
||||
ctx,
|
||||
"toolbox/server/tool/init",
|
||||
trace.WithAttributes(attribute.String("tool_kind", tc.ToolConfigKind())),
|
||||
trace.WithAttributes(attribute.String("tool_name", name)),
|
||||
)
|
||||
defer span.End()
|
||||
t, err := tc.Initialize(sourcesMap)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to initialize tool %q: %w", name, err)
|
||||
}
|
||||
return t, nil
|
||||
}()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
toolsMap[name] = t
|
||||
}
|
||||
l.InfoContext(ctx, fmt.Sprintf("Initialized %d tools.", len(toolsMap)))
|
||||
|
||||
// create a default toolset that contains all tools
|
||||
allToolNames := make([]string, 0, len(toolsMap))
|
||||
for name := range toolsMap {
|
||||
allToolNames = append(allToolNames, name)
|
||||
}
|
||||
if cfg.ToolsetConfigs == nil {
|
||||
cfg.ToolsetConfigs = make(ToolsetConfigs)
|
||||
}
|
||||
cfg.ToolsetConfigs[""] = tools.ToolsetConfig{Name: "", ToolNames: allToolNames}
|
||||
|
||||
// initialize and validate the toolsets from configs
|
||||
toolsetsMap := make(map[string]tools.Toolset)
|
||||
for name, tc := range cfg.ToolsetConfigs {
|
||||
t, err := func() (tools.Toolset, error) {
|
||||
_, span := instrumentation.Tracer.Start(
|
||||
ctx,
|
||||
"toolbox/server/toolset/init",
|
||||
trace.WithAttributes(attribute.String("toolset_name", name)),
|
||||
)
|
||||
defer span.End()
|
||||
t, err := tc.Initialize(cfg.Version, toolsMap)
|
||||
if err != nil {
|
||||
return tools.Toolset{}, fmt.Errorf("unable to initialize toolset %q: %w", name, err)
|
||||
}
|
||||
return t, err
|
||||
}()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
toolsetsMap[name] = t
|
||||
}
|
||||
l.InfoContext(ctx, fmt.Sprintf("Initialized %d toolsets.", len(toolsetsMap)))
|
||||
|
||||
addr := net.JoinHostPort(cfg.Address, strconv.Itoa(cfg.Port))
|
||||
srv := &http.Server{Addr: addr, Handler: r}
|
||||
|
||||
sseManager := newSseManager(ctx)
|
||||
|
||||
resourceManager := NewResourceManager(sourcesMap, authServicesMap, toolsMap, toolsetsMap)
|
||||
|
||||
s := &Server{
|
||||
version: cfg.Version,
|
||||
srv: srv,
|
||||
@@ -214,11 +317,7 @@ func NewServer(ctx context.Context, cfg ServerConfig, l log.Logger) (*Server, er
|
||||
logger: l,
|
||||
instrumentation: instrumentation,
|
||||
sseManager: sseManager,
|
||||
|
||||
sources: sourcesMap,
|
||||
authServices: authServicesMap,
|
||||
tools: toolsMap,
|
||||
toolsets: toolsetsMap,
|
||||
ResourceMgr: resourceManager,
|
||||
}
|
||||
// control plane
|
||||
apiR, err := apiRouter(s)
|
||||
|
||||
@@ -23,9 +23,16 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/googleapis/genai-toolbox/internal/auth"
|
||||
"github.com/googleapis/genai-toolbox/internal/log"
|
||||
"github.com/googleapis/genai-toolbox/internal/server"
|
||||
"github.com/googleapis/genai-toolbox/internal/sources"
|
||||
"github.com/googleapis/genai-toolbox/internal/sources/alloydbpg"
|
||||
"github.com/googleapis/genai-toolbox/internal/telemetry"
|
||||
"github.com/googleapis/genai-toolbox/internal/testutils"
|
||||
"github.com/googleapis/genai-toolbox/internal/tools"
|
||||
"github.com/googleapis/genai-toolbox/internal/util"
|
||||
)
|
||||
|
||||
func TestServe(t *testing.T) {
|
||||
@@ -54,8 +61,16 @@ func TestServe(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
ctx = util.WithLogger(ctx, testLogger)
|
||||
|
||||
s, err := server.NewServer(ctx, cfg, testLogger)
|
||||
instrumentation, err := telemetry.CreateTelemetryInstrumentation(cfg.Version)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
|
||||
ctx = util.WithInstrumentation(ctx, instrumentation)
|
||||
|
||||
s, err := server.NewServer(ctx, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to initialize server: %v", err)
|
||||
}
|
||||
@@ -93,3 +108,67 @@ func TestServe(t *testing.T) {
|
||||
t.Fatalf("version missing from output: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateServer(t *testing.T) {
|
||||
ctx, err := testutils.ContextWithNewLogger()
|
||||
if err != nil {
|
||||
t.Fatalf("error setting up logger: %s", err)
|
||||
}
|
||||
|
||||
addr, port := "127.0.0.1", 5000
|
||||
cfg := server.ServerConfig{
|
||||
Version: "0.0.0",
|
||||
Address: addr,
|
||||
Port: port,
|
||||
}
|
||||
|
||||
instrumentation, err := telemetry.CreateTelemetryInstrumentation(cfg.Version)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
|
||||
ctx = util.WithInstrumentation(ctx, instrumentation)
|
||||
|
||||
s, err := server.NewServer(ctx, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("error setting up server: %s", err)
|
||||
}
|
||||
|
||||
newSources := map[string]sources.Source{
|
||||
"example-source": &alloydbpg.Source{
|
||||
Name: "example-alloydb-source",
|
||||
Kind: "alloydb-postgres",
|
||||
},
|
||||
}
|
||||
newAuth := map[string]auth.AuthService{"example-auth": nil}
|
||||
newTools := map[string]tools.Tool{"example-tool": nil}
|
||||
newToolsets := map[string]tools.Toolset{
|
||||
"example-toolset": {
|
||||
Name: "example-toolset", Tools: []*tools.Tool{},
|
||||
},
|
||||
}
|
||||
s.ResourceMgr.SetResources(newSources, newAuth, newTools, newToolsets)
|
||||
if err != nil {
|
||||
t.Errorf("error updating server: %s", err)
|
||||
}
|
||||
|
||||
gotSource, _ := s.ResourceMgr.GetSource("example-source")
|
||||
if diff := cmp.Diff(gotSource, newSources["example-source"]); diff != "" {
|
||||
t.Errorf("error updating server, sources (-want +got):\n%s", diff)
|
||||
}
|
||||
|
||||
gotAuthService, _ := s.ResourceMgr.GetAuthService("example-auth")
|
||||
if diff := cmp.Diff(gotAuthService, newAuth["example-auth"]); diff != "" {
|
||||
t.Errorf("error updating server, authServices (-want +got):\n%s", diff)
|
||||
}
|
||||
|
||||
gotTool, _ := s.ResourceMgr.GetTool("example-tool")
|
||||
if diff := cmp.Diff(gotTool, newTools["example-tool"]); diff != "" {
|
||||
t.Errorf("error updating server, tools (-want +got):\n%s", diff)
|
||||
}
|
||||
|
||||
gotToolset, _ := s.ResourceMgr.GetToolset("example-toolset")
|
||||
if diff := cmp.Diff(gotToolset, newToolsets["example-toolset"]); diff != "" {
|
||||
t.Errorf("error updating server, toolset (-want +got):\n%s", diff)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package server
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
@@ -15,9 +15,12 @@
|
||||
package testutils
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/googleapis/genai-toolbox/internal/log"
|
||||
@@ -42,3 +45,63 @@ func ContextWithNewLogger() (context.Context, error) {
|
||||
}
|
||||
return util.WithLogger(ctx, logger), nil
|
||||
}
|
||||
|
||||
// WaitForString waits until the server logs a single line that matches the provided regex.
|
||||
// returns the output of whatever the server sent so far.
|
||||
func WaitForString(ctx context.Context, re *regexp.Regexp, pr io.ReadCloser) (string, error) {
|
||||
in := bufio.NewReader(pr)
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
// read lines in background, sending result of each read over a channel
|
||||
// this allows us to use in.ReadString without blocking
|
||||
type result struct {
|
||||
s string
|
||||
err error
|
||||
}
|
||||
output := make(chan result)
|
||||
go func() {
|
||||
defer close(output)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// if the context is canceled, the orig thread will send back the error
|
||||
// so we can just exit the goroutine here
|
||||
return
|
||||
default:
|
||||
// otherwise read a line from the output
|
||||
s, err := in.ReadString('\n')
|
||||
if err != nil {
|
||||
output <- result{err: err}
|
||||
return
|
||||
}
|
||||
output <- result{s: s}
|
||||
// if that last string matched, exit the goroutine
|
||||
if re.MatchString(s) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// collect the output until the ctx is canceled, an error was hit,
|
||||
// or match was found (which is indicated the channel is closed)
|
||||
var sb strings.Builder
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// if ctx is done, return that error
|
||||
return sb.String(), ctx.Err()
|
||||
case o, ok := <-output:
|
||||
if !ok {
|
||||
// match was found!
|
||||
return sb.String(), nil
|
||||
}
|
||||
if o.err != nil {
|
||||
// error was found!
|
||||
return sb.String(), o.err
|
||||
}
|
||||
sb.WriteString(o.s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
"github.com/go-playground/validator/v10"
|
||||
yaml "github.com/goccy/go-yaml"
|
||||
"github.com/googleapis/genai-toolbox/internal/log"
|
||||
"github.com/googleapis/genai-toolbox/internal/telemetry"
|
||||
)
|
||||
|
||||
// DecodeJSON decodes a given reader into an interface using the json decoder.
|
||||
@@ -98,10 +99,25 @@ func WithLogger(ctx context.Context, logger log.Logger) context.Context {
|
||||
return context.WithValue(ctx, loggerKey, logger)
|
||||
}
|
||||
|
||||
// LoggerFromContext retreives the logger or return an error
|
||||
// LoggerFromContext retrieves the logger or return an error
|
||||
func LoggerFromContext(ctx context.Context) (log.Logger, error) {
|
||||
if logger, ok := ctx.Value(loggerKey).(log.Logger); ok {
|
||||
return logger, nil
|
||||
}
|
||||
return nil, fmt.Errorf("unable to retrieve logger")
|
||||
}
|
||||
|
||||
const instrumentationKey contextKey = "instrumentation"
|
||||
|
||||
// WithInstrumentation adds an instrumentation into the context as a value
|
||||
func WithInstrumentation(ctx context.Context, instrumentation *telemetry.Instrumentation) context.Context {
|
||||
return context.WithValue(ctx, instrumentationKey, instrumentation)
|
||||
}
|
||||
|
||||
// InstrumentationFromContext retrieves the instrumentation or return an error
|
||||
func InstrumentationFromContext(ctx context.Context) (*telemetry.Instrumentation, error) {
|
||||
if instrumentation, ok := ctx.Value(instrumentationKey).(*telemetry.Instrumentation); ok {
|
||||
return instrumentation, nil
|
||||
}
|
||||
return nil, fmt.Errorf("unable to retrieve instrumentation")
|
||||
}
|
||||
|
||||
@@ -28,59 +28,60 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/googleapis/genai-toolbox/internal/server/mcp/jsonrpc"
|
||||
"github.com/googleapis/genai-toolbox/internal/testutils"
|
||||
"github.com/googleapis/genai-toolbox/tests"
|
||||
)
|
||||
|
||||
var (
|
||||
ALLOYDB_AI_NL_SOURCE_KIND = "alloydb-postgres"
|
||||
ALLOYDB_AI_NL_TOOL_KIND = "alloydb-ai-nl"
|
||||
ALLOYDB_AI_NL_PROJECT = os.Getenv("ALLOYDB_AI_NL_PROJECT")
|
||||
ALLOYDB_AI_NL_REGION = os.Getenv("ALLOYDB_AI_NL_REGION")
|
||||
ALLOYDB_AI_NL_CLUSTER = os.Getenv("ALLOYDB_AI_NL_CLUSTER")
|
||||
ALLOYDB_AI_NL_INSTANCE = os.Getenv("ALLOYDB_AI_NL_INSTANCE")
|
||||
ALLOYDB_AI_NL_DATABASE = os.Getenv("ALLOYDB_AI_NL_DATABASE")
|
||||
ALLOYDB_AI_NL_USER = os.Getenv("ALLOYDB_AI_NL_USER")
|
||||
ALLOYDB_AI_NL_PASS = os.Getenv("ALLOYDB_AI_NL_PASS")
|
||||
AlloyDBAINLSourceKind = "alloydb-postgres"
|
||||
AlloyDBAINLToolKind = "alloydb-ai-nl"
|
||||
AlloyDBAINLProject = os.Getenv("ALLOYDB_AI_NL_PROJECT")
|
||||
AlloyDBAINLRegion = os.Getenv("ALLOYDB_AI_NL_REGION")
|
||||
AlloyDBAINLCluster = os.Getenv("ALLOYDB_AI_NL_CLUSTER")
|
||||
AlloyDBAINLInstance = os.Getenv("ALLOYDB_AI_NL_INSTANCE")
|
||||
AlloyDBAINLDatabase = os.Getenv("ALLOYDB_AI_NL_DATABASE")
|
||||
AlloyDBAINLUser = os.Getenv("ALLOYDB_AI_NL_USER")
|
||||
AlloyDBAINLPass = os.Getenv("ALLOYDB_AI_NL_PASS")
|
||||
)
|
||||
|
||||
func getAlloyDBAiNlVars(t *testing.T) map[string]any {
|
||||
func getAlloyDBAINLVars(t *testing.T) map[string]any {
|
||||
switch "" {
|
||||
case ALLOYDB_AI_NL_PROJECT:
|
||||
case AlloyDBAINLProject:
|
||||
t.Fatal("'ALLOYDB_AI_NL_PROJECT' not set")
|
||||
case ALLOYDB_AI_NL_REGION:
|
||||
case AlloyDBAINLRegion:
|
||||
t.Fatal("'ALLOYDB_AI_NL_REGION' not set")
|
||||
case ALLOYDB_AI_NL_CLUSTER:
|
||||
case AlloyDBAINLCluster:
|
||||
t.Fatal("'ALLOYDB_AI_NL_CLUSTER' not set")
|
||||
case ALLOYDB_AI_NL_INSTANCE:
|
||||
case AlloyDBAINLInstance:
|
||||
t.Fatal("'ALLOYDB_AI_NL_INSTANCE' not set")
|
||||
case ALLOYDB_AI_NL_DATABASE:
|
||||
case AlloyDBAINLDatabase:
|
||||
t.Fatal("'ALLOYDB_AI_NL_DATABASE' not set")
|
||||
case ALLOYDB_AI_NL_USER:
|
||||
case AlloyDBAINLUser:
|
||||
t.Fatal("'ALLOYDB_AI_NL_USER' not set")
|
||||
case ALLOYDB_AI_NL_PASS:
|
||||
case AlloyDBAINLPass:
|
||||
t.Fatal("'ALLOYDB_AI_NL_PASS' not set")
|
||||
}
|
||||
return map[string]any{
|
||||
"kind": ALLOYDB_AI_NL_SOURCE_KIND,
|
||||
"project": ALLOYDB_AI_NL_PROJECT,
|
||||
"cluster": ALLOYDB_AI_NL_CLUSTER,
|
||||
"instance": ALLOYDB_AI_NL_INSTANCE,
|
||||
"region": ALLOYDB_AI_NL_REGION,
|
||||
"database": ALLOYDB_AI_NL_DATABASE,
|
||||
"user": ALLOYDB_AI_NL_USER,
|
||||
"password": ALLOYDB_AI_NL_PASS,
|
||||
"kind": AlloyDBAINLSourceKind,
|
||||
"project": AlloyDBAINLProject,
|
||||
"cluster": AlloyDBAINLCluster,
|
||||
"instance": AlloyDBAINLInstance,
|
||||
"region": AlloyDBAINLRegion,
|
||||
"database": AlloyDBAINLDatabase,
|
||||
"user": AlloyDBAINLUser,
|
||||
"password": AlloyDBAINLPass,
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlloyDBAiNlToolEndpoints(t *testing.T) {
|
||||
sourceConfig := getAlloyDBAiNlVars(t)
|
||||
func TestAlloyDBAINLToolEndpoints(t *testing.T) {
|
||||
sourceConfig := getAlloyDBAINLVars(t)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||
defer cancel()
|
||||
|
||||
var args []string
|
||||
|
||||
// Write config into a file and pass it to command
|
||||
toolsFile := getAiNlToolsConfig(sourceConfig)
|
||||
toolsFile := getAINLToolsConfig(sourceConfig)
|
||||
|
||||
cmd, cleanup, err := tests.StartCmd(ctx, toolsFile, args...)
|
||||
if err != nil {
|
||||
@@ -90,18 +91,18 @@ func TestAlloyDBAiNlToolEndpoints(t *testing.T) {
|
||||
|
||||
waitCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := cmd.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`))
|
||||
out, err := testutils.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`), cmd.Out)
|
||||
if err != nil {
|
||||
t.Logf("toolbox command logs: \n%s", out)
|
||||
t.Fatalf("toolbox didn't start successfully: %s", err)
|
||||
}
|
||||
|
||||
runAiNlToolGetTest(t)
|
||||
runAiNlToolInvokeTest(t)
|
||||
runAiNlMCPToolCallMethod(t)
|
||||
runAINLToolGetTest(t)
|
||||
runAINLToolInvokeTest(t)
|
||||
runAINLMCPToolCallMethod(t)
|
||||
}
|
||||
|
||||
func runAiNlToolGetTest(t *testing.T) {
|
||||
func runAINLToolGetTest(t *testing.T) {
|
||||
// Test tool get endpoint
|
||||
tcs := []struct {
|
||||
name string
|
||||
@@ -156,7 +157,7 @@ func runAiNlToolGetTest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func runAiNlToolInvokeTest(t *testing.T) {
|
||||
func runAINLToolInvokeTest(t *testing.T) {
|
||||
// Get ID token
|
||||
idToken, err := tests.GetGoogleIdToken(tests.ClientId)
|
||||
if err != nil {
|
||||
@@ -276,7 +277,7 @@ func runAiNlToolInvokeTest(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
func getAiNlToolsConfig(sourceConfig map[string]any) map[string]any {
|
||||
func getAINLToolsConfig(sourceConfig map[string]any) map[string]any {
|
||||
// Write config into a file and pass it to command
|
||||
toolsFile := map[string]any{
|
||||
"sources": map[string]any{
|
||||
@@ -290,13 +291,13 @@ func getAiNlToolsConfig(sourceConfig map[string]any) map[string]any {
|
||||
},
|
||||
"tools": map[string]any{
|
||||
"my-simple-tool": map[string]any{
|
||||
"kind": ALLOYDB_AI_NL_TOOL_KIND,
|
||||
"kind": AlloyDBAINLToolKind,
|
||||
"source": "my-instance",
|
||||
"description": "Simple tool to test end to end functionality.",
|
||||
"nlConfig": "my_nl_config",
|
||||
},
|
||||
"my-auth-tool": map[string]any{
|
||||
"kind": ALLOYDB_AI_NL_TOOL_KIND,
|
||||
"kind": AlloyDBAINLToolKind,
|
||||
"source": "my-instance",
|
||||
"description": "Tool to test authenticated parameters.",
|
||||
"nlConfig": "my_nl_config",
|
||||
@@ -315,7 +316,7 @@ func getAiNlToolsConfig(sourceConfig map[string]any) map[string]any {
|
||||
},
|
||||
},
|
||||
"my-auth-required-tool": map[string]any{
|
||||
"kind": ALLOYDB_AI_NL_TOOL_KIND,
|
||||
"kind": AlloyDBAINLToolKind,
|
||||
"source": "my-instance",
|
||||
"description": "Tool to test auth required invocation.",
|
||||
"nlConfig": "my_nl_config",
|
||||
@@ -329,7 +330,7 @@ func getAiNlToolsConfig(sourceConfig map[string]any) map[string]any {
|
||||
return toolsFile
|
||||
}
|
||||
|
||||
func runAiNlMCPToolCallMethod(t *testing.T) {
|
||||
func runAINLMCPToolCallMethod(t *testing.T) {
|
||||
sessionId := tests.RunInitialize(t, "2024-11-05")
|
||||
header := map[string]string{}
|
||||
if sessionId != "" {
|
||||
|
||||
@@ -26,65 +26,66 @@ import (
|
||||
|
||||
"cloud.google.com/go/alloydbconn"
|
||||
"github.com/google/uuid"
|
||||
"github.com/googleapis/genai-toolbox/internal/testutils"
|
||||
"github.com/googleapis/genai-toolbox/tests"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
var (
|
||||
ALLOYDB_POSTGRES_SOURCE_KIND = "alloydb-postgres"
|
||||
ALLOYDB_POSTGRES_TOOL_KIND = "postgres-sql"
|
||||
ALLOYDB_POSTGRES_PROJECT = os.Getenv("ALLOYDB_POSTGRES_PROJECT")
|
||||
ALLOYDB_POSTGRES_REGION = os.Getenv("ALLOYDB_POSTGRES_REGION")
|
||||
ALLOYDB_POSTGRES_CLUSTER = os.Getenv("ALLOYDB_POSTGRES_CLUSTER")
|
||||
ALLOYDB_POSTGRES_INSTANCE = os.Getenv("ALLOYDB_POSTGRES_INSTANCE")
|
||||
ALLOYDB_POSTGRES_DATABASE = os.Getenv("ALLOYDB_POSTGRES_DATABASE")
|
||||
ALLOYDB_POSTGRES_USER = os.Getenv("ALLOYDB_POSTGRES_USER")
|
||||
ALLOYDB_POSTGRES_PASS = os.Getenv("ALLOYDB_POSTGRES_PASS")
|
||||
AlloyDBPostgresSourceKind = "alloydb-postgres"
|
||||
AlloyDBPostgresToolKind = "postgres-sql"
|
||||
AlloyDBPostgresProject = os.Getenv("ALLOYDB_POSTGRES_PROJECT")
|
||||
AlloyDBPostgresRegion = os.Getenv("ALLOYDB_POSTGRES_REGION")
|
||||
AlloyDBPostgresCluster = os.Getenv("ALLOYDB_POSTGRES_CLUSTER")
|
||||
AlloyDBPostgresInstance = os.Getenv("ALLOYDB_POSTGRES_INSTANCE")
|
||||
AlloyDBPostgresDatabase = os.Getenv("ALLOYDB_POSTGRES_DATABASE")
|
||||
AlloyDBPostgresUser = os.Getenv("ALLOYDB_POSTGRES_USER")
|
||||
AlloyDBPostgresPass = os.Getenv("ALLOYDB_POSTGRES_PASS")
|
||||
)
|
||||
|
||||
func getAlloyDBPgVars(t *testing.T) map[string]any {
|
||||
switch "" {
|
||||
case ALLOYDB_POSTGRES_PROJECT:
|
||||
case AlloyDBPostgresProject:
|
||||
t.Fatal("'ALLOYDB_POSTGRES_PROJECT' not set")
|
||||
case ALLOYDB_POSTGRES_REGION:
|
||||
case AlloyDBPostgresRegion:
|
||||
t.Fatal("'ALLOYDB_POSTGRES_REGION' not set")
|
||||
case ALLOYDB_POSTGRES_CLUSTER:
|
||||
case AlloyDBPostgresCluster:
|
||||
t.Fatal("'ALLOYDB_POSTGRES_CLUSTER' not set")
|
||||
case ALLOYDB_POSTGRES_INSTANCE:
|
||||
case AlloyDBPostgresInstance:
|
||||
t.Fatal("'ALLOYDB_POSTGRES_INSTANCE' not set")
|
||||
case ALLOYDB_POSTGRES_DATABASE:
|
||||
case AlloyDBPostgresDatabase:
|
||||
t.Fatal("'ALLOYDB_POSTGRES_DATABASE' not set")
|
||||
case ALLOYDB_POSTGRES_USER:
|
||||
case AlloyDBPostgresUser:
|
||||
t.Fatal("'ALLOYDB_POSTGRES_USER' not set")
|
||||
case ALLOYDB_POSTGRES_PASS:
|
||||
case AlloyDBPostgresPass:
|
||||
t.Fatal("'ALLOYDB_POSTGRES_PASS' not set")
|
||||
}
|
||||
return map[string]any{
|
||||
"kind": ALLOYDB_POSTGRES_SOURCE_KIND,
|
||||
"project": ALLOYDB_POSTGRES_PROJECT,
|
||||
"cluster": ALLOYDB_POSTGRES_CLUSTER,
|
||||
"instance": ALLOYDB_POSTGRES_INSTANCE,
|
||||
"region": ALLOYDB_POSTGRES_REGION,
|
||||
"database": ALLOYDB_POSTGRES_DATABASE,
|
||||
"user": ALLOYDB_POSTGRES_USER,
|
||||
"password": ALLOYDB_POSTGRES_PASS,
|
||||
"kind": AlloyDBPostgresSourceKind,
|
||||
"project": AlloyDBPostgresProject,
|
||||
"cluster": AlloyDBPostgresCluster,
|
||||
"instance": AlloyDBPostgresInstance,
|
||||
"region": AlloyDBPostgresRegion,
|
||||
"database": AlloyDBPostgresDatabase,
|
||||
"user": AlloyDBPostgresUser,
|
||||
"password": AlloyDBPostgresPass,
|
||||
}
|
||||
}
|
||||
|
||||
// Copied over from alloydb_pg.go
|
||||
func getAlloyDBDialOpts(ip_type string) ([]alloydbconn.DialOption, error) {
|
||||
switch strings.ToLower(ip_type) {
|
||||
func getAlloyDBDialOpts(ipType string) ([]alloydbconn.DialOption, error) {
|
||||
switch strings.ToLower(ipType) {
|
||||
case "private":
|
||||
return []alloydbconn.DialOption{alloydbconn.WithPrivateIP()}, nil
|
||||
case "public":
|
||||
return []alloydbconn.DialOption{alloydbconn.WithPublicIP()}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("invalid ip_type %s", ip_type)
|
||||
return nil, fmt.Errorf("invalid ipType %s", ipType)
|
||||
}
|
||||
}
|
||||
|
||||
// Copied over from alloydb_pg.go
|
||||
func initAlloyDBPgConnectionPool(project, region, cluster, instance, ip_type, user, pass, dbname string) (*pgxpool.Pool, error) {
|
||||
func initAlloyDBPgConnectionPool(project, region, cluster, instance, ipType, user, pass, dbname string) (*pgxpool.Pool, error) {
|
||||
// Configure the driver to connect to the database
|
||||
dsn := fmt.Sprintf("user=%s password=%s dbname=%s sslmode=disable", user, pass, dbname)
|
||||
config, err := pgxpool.ParseConfig(dsn)
|
||||
@@ -93,7 +94,7 @@ func initAlloyDBPgConnectionPool(project, region, cluster, instance, ip_type, us
|
||||
}
|
||||
|
||||
// Create a new dialer with options
|
||||
dialOpts, err := getAlloyDBDialOpts(ip_type)
|
||||
dialOpts, err := getAlloyDBDialOpts(ipType)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -123,7 +124,7 @@ func TestAlloyDBPgToolEndpoints(t *testing.T) {
|
||||
|
||||
var args []string
|
||||
|
||||
pool, err := initAlloyDBPgConnectionPool(ALLOYDB_POSTGRES_PROJECT, ALLOYDB_POSTGRES_REGION, ALLOYDB_POSTGRES_CLUSTER, ALLOYDB_POSTGRES_INSTANCE, "public", ALLOYDB_POSTGRES_USER, ALLOYDB_POSTGRES_PASS, ALLOYDB_POSTGRES_DATABASE)
|
||||
pool, err := initAlloyDBPgConnectionPool(AlloyDBPostgresProject, AlloyDBPostgresRegion, AlloyDBPostgresCluster, AlloyDBPostgresInstance, "public", AlloyDBPostgresUser, AlloyDBPostgresPass, AlloyDBPostgresDatabase)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to create AlloyDB connection pool: %s", err)
|
||||
}
|
||||
@@ -134,20 +135,20 @@ func TestAlloyDBPgToolEndpoints(t *testing.T) {
|
||||
tableNameTemplateParam := "template_param_table_" + strings.ReplaceAll(uuid.New().String(), "-", "")
|
||||
|
||||
// set up data for param tool
|
||||
create_statement1, insert_statement1, tool_statement1, params1 := tests.GetPostgresSQLParamToolInfo(tableNameParam)
|
||||
teardownTable1 := tests.SetupPostgresSQLTable(t, ctx, pool, create_statement1, insert_statement1, tableNameParam, params1)
|
||||
createStatement1, insertStatement1, toolStatement1, params1 := tests.GetPostgresSQLParamToolInfo(tableNameParam)
|
||||
teardownTable1 := tests.SetupPostgresSQLTable(t, ctx, pool, createStatement1, insertStatement1, tableNameParam, params1)
|
||||
defer teardownTable1(t)
|
||||
|
||||
// set up data for auth tool
|
||||
create_statement2, insert_statement2, tool_statement2, params2 := tests.GetPostgresSQLAuthToolInfo(tableNameAuth)
|
||||
teardownTable2 := tests.SetupPostgresSQLTable(t, ctx, pool, create_statement2, insert_statement2, tableNameAuth, params2)
|
||||
createStatement2, insertStatement2, toolStatement2, params2 := tests.GetPostgresSQLAuthToolInfo(tableNameAuth)
|
||||
teardownTable2 := tests.SetupPostgresSQLTable(t, ctx, pool, createStatement2, insertStatement2, tableNameAuth, params2)
|
||||
defer teardownTable2(t)
|
||||
|
||||
// Write config into a file and pass it to command
|
||||
toolsFile := tests.GetToolsConfig(sourceConfig, ALLOYDB_POSTGRES_TOOL_KIND, tool_statement1, tool_statement2)
|
||||
toolsFile := tests.GetToolsConfig(sourceConfig, AlloyDBPostgresToolKind, toolStatement1, toolStatement2)
|
||||
toolsFile = tests.AddPgExecuteSqlConfig(t, toolsFile)
|
||||
tmplSelectCombined, tmplSelectFilterCombined := tests.GetPostgresSQLTmplToolStatement()
|
||||
toolsFile = tests.AddTemplateParamConfig(t, toolsFile, ALLOYDB_POSTGRES_TOOL_KIND, tmplSelectCombined, tmplSelectFilterCombined, "")
|
||||
toolsFile = tests.AddTemplateParamConfig(t, toolsFile, AlloyDBPostgresToolKind, tmplSelectCombined, tmplSelectFilterCombined, "")
|
||||
|
||||
cmd, cleanup, err := tests.StartCmd(ctx, toolsFile, args...)
|
||||
if err != nil {
|
||||
@@ -157,7 +158,7 @@ func TestAlloyDBPgToolEndpoints(t *testing.T) {
|
||||
|
||||
waitCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := cmd.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`))
|
||||
out, err := testutils.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`), cmd.Out)
|
||||
if err != nil {
|
||||
t.Logf("toolbox command logs: \n%s", out)
|
||||
t.Fatalf("toolbox didn't start successfully: %s", err)
|
||||
@@ -193,7 +194,7 @@ func TestAlloyDBPgIpConnection(t *testing.T) {
|
||||
for _, tc := range tcs {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
sourceConfig["ipType"] = tc.ipType
|
||||
err := tests.RunSourceConnectionTest(t, sourceConfig, ALLOYDB_POSTGRES_TOOL_KIND)
|
||||
err := tests.RunSourceConnectionTest(t, sourceConfig, AlloyDBPostgresToolKind)
|
||||
if err != nil {
|
||||
t.Fatalf("Connection test failure: %s", err)
|
||||
}
|
||||
@@ -205,35 +206,35 @@ func TestAlloyDBPgIpConnection(t *testing.T) {
|
||||
func TestAlloyDBPgIAMConnection(t *testing.T) {
|
||||
getAlloyDBPgVars(t)
|
||||
// service account email used for IAM should trim the suffix
|
||||
serviceAccountEmail := strings.TrimSuffix(tests.SERVICE_ACCOUNT_EMAIL, ".gserviceaccount.com")
|
||||
serviceAccountEmail := strings.TrimSuffix(tests.ServiceAccountEmail, ".gserviceaccount.com")
|
||||
|
||||
noPassSourceConfig := map[string]any{
|
||||
"kind": ALLOYDB_POSTGRES_SOURCE_KIND,
|
||||
"project": ALLOYDB_POSTGRES_PROJECT,
|
||||
"cluster": ALLOYDB_POSTGRES_CLUSTER,
|
||||
"instance": ALLOYDB_POSTGRES_INSTANCE,
|
||||
"region": ALLOYDB_POSTGRES_REGION,
|
||||
"database": ALLOYDB_POSTGRES_DATABASE,
|
||||
"kind": AlloyDBPostgresSourceKind,
|
||||
"project": AlloyDBPostgresProject,
|
||||
"cluster": AlloyDBPostgresCluster,
|
||||
"instance": AlloyDBPostgresInstance,
|
||||
"region": AlloyDBPostgresRegion,
|
||||
"database": AlloyDBPostgresDatabase,
|
||||
"user": serviceAccountEmail,
|
||||
}
|
||||
|
||||
noUserSourceConfig := map[string]any{
|
||||
"kind": ALLOYDB_POSTGRES_SOURCE_KIND,
|
||||
"project": ALLOYDB_POSTGRES_PROJECT,
|
||||
"cluster": ALLOYDB_POSTGRES_CLUSTER,
|
||||
"instance": ALLOYDB_POSTGRES_INSTANCE,
|
||||
"region": ALLOYDB_POSTGRES_REGION,
|
||||
"database": ALLOYDB_POSTGRES_DATABASE,
|
||||
"kind": AlloyDBPostgresSourceKind,
|
||||
"project": AlloyDBPostgresProject,
|
||||
"cluster": AlloyDBPostgresCluster,
|
||||
"instance": AlloyDBPostgresInstance,
|
||||
"region": AlloyDBPostgresRegion,
|
||||
"database": AlloyDBPostgresDatabase,
|
||||
"password": "random",
|
||||
}
|
||||
|
||||
noUserNoPassSourceConfig := map[string]any{
|
||||
"kind": ALLOYDB_POSTGRES_SOURCE_KIND,
|
||||
"project": ALLOYDB_POSTGRES_PROJECT,
|
||||
"cluster": ALLOYDB_POSTGRES_CLUSTER,
|
||||
"instance": ALLOYDB_POSTGRES_INSTANCE,
|
||||
"region": ALLOYDB_POSTGRES_REGION,
|
||||
"database": ALLOYDB_POSTGRES_DATABASE,
|
||||
"kind": AlloyDBPostgresSourceKind,
|
||||
"project": AlloyDBPostgresProject,
|
||||
"cluster": AlloyDBPostgresCluster,
|
||||
"instance": AlloyDBPostgresInstance,
|
||||
"region": AlloyDBPostgresRegion,
|
||||
"database": AlloyDBPostgresDatabase,
|
||||
}
|
||||
tcs := []struct {
|
||||
name string
|
||||
@@ -258,7 +259,7 @@ func TestAlloyDBPgIAMConnection(t *testing.T) {
|
||||
}
|
||||
for _, tc := range tcs {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := tests.RunSourceConnectionTest(t, tc.sourceConfig, ALLOYDB_POSTGRES_TOOL_KIND)
|
||||
err := tests.RunSourceConnectionTest(t, tc.sourceConfig, AlloyDBPostgresToolKind)
|
||||
if err != nil {
|
||||
if tc.isErr {
|
||||
return
|
||||
|
||||
@@ -23,7 +23,7 @@ import (
|
||||
"google.golang.org/api/idtoken"
|
||||
)
|
||||
|
||||
var SERVICE_ACCOUNT_EMAIL = os.Getenv("SERVICE_ACCOUNT_EMAIL")
|
||||
var ServiceAccountEmail = os.Getenv("SERVICE_ACCOUNT_EMAIL")
|
||||
var ClientId = os.Getenv("CLIENT_ID")
|
||||
|
||||
// GetGoogleIdToken retrieve and return the Google ID token
|
||||
|
||||
@@ -29,6 +29,7 @@ import (
|
||||
|
||||
bigqueryapi "cloud.google.com/go/bigquery"
|
||||
"github.com/google/uuid"
|
||||
"github.com/googleapis/genai-toolbox/internal/testutils"
|
||||
"github.com/googleapis/genai-toolbox/tests"
|
||||
"golang.org/x/oauth2/google"
|
||||
"google.golang.org/api/googleapi"
|
||||
@@ -37,20 +38,20 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
BIGQUERY_SOURCE_KIND = "bigquery"
|
||||
BIGQUERY_TOOL_KIND = "bigquery-sql"
|
||||
BIGQUERY_PROJECT = os.Getenv("BIGQUERY_PROJECT")
|
||||
BigquerySourceKind = "bigquery"
|
||||
BigqueryToolKind = "bigquery-sql"
|
||||
BigqueryProject = os.Getenv("BIGQUERY_PROJECT")
|
||||
)
|
||||
|
||||
func getBigQueryVars(t *testing.T) map[string]any {
|
||||
switch "" {
|
||||
case BIGQUERY_PROJECT:
|
||||
case BigqueryProject:
|
||||
t.Fatal("'BIGQUERY_PROJECT' not set")
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"kind": BIGQUERY_SOURCE_KIND,
|
||||
"project": BIGQUERY_PROJECT,
|
||||
"kind": BigquerySourceKind,
|
||||
"project": BigqueryProject,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -76,7 +77,7 @@ func TestBigQueryToolEndpoints(t *testing.T) {
|
||||
|
||||
var args []string
|
||||
|
||||
client, err := initBigQueryConnection(BIGQUERY_PROJECT)
|
||||
client, err := initBigQueryConnection(BigqueryProject)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to create Cloud SQL connection pool: %s", err)
|
||||
}
|
||||
@@ -85,36 +86,36 @@ func TestBigQueryToolEndpoints(t *testing.T) {
|
||||
datasetName := fmt.Sprintf("temp_toolbox_test_%s", strings.ReplaceAll(uuid.New().String(), "-", ""))
|
||||
tableName := fmt.Sprintf("param_table_%s", strings.ReplaceAll(uuid.New().String(), "-", ""))
|
||||
tableNameParam := fmt.Sprintf("`%s.%s.%s`",
|
||||
BIGQUERY_PROJECT,
|
||||
BigqueryProject,
|
||||
datasetName,
|
||||
tableName,
|
||||
)
|
||||
tableNameAuth := fmt.Sprintf("`%s.%s.auth_table_%s`",
|
||||
BIGQUERY_PROJECT,
|
||||
BigqueryProject,
|
||||
datasetName,
|
||||
strings.ReplaceAll(uuid.New().String(), "-", ""),
|
||||
)
|
||||
tableNameTemplateParam := fmt.Sprintf("`%s.%s.template_param_table_%s`",
|
||||
BIGQUERY_PROJECT,
|
||||
BigqueryProject,
|
||||
datasetName,
|
||||
strings.ReplaceAll(uuid.New().String(), "-", ""),
|
||||
)
|
||||
|
||||
// set up data for param tool
|
||||
create_statement1, insert_statement1, tool_statement1, params1 := getBigQueryParamToolInfo(tableNameParam)
|
||||
teardownTable1 := setupBigQueryTable(t, ctx, client, create_statement1, insert_statement1, datasetName, tableNameParam, params1)
|
||||
createStatement1, insertStatement1, toolStatement1, params1 := getBigQueryParamToolInfo(tableNameParam)
|
||||
teardownTable1 := setupBigQueryTable(t, ctx, client, createStatement1, insertStatement1, datasetName, tableNameParam, params1)
|
||||
defer teardownTable1(t)
|
||||
|
||||
// set up data for auth tool
|
||||
create_statement2, insert_statement2, tool_statement2, params2 := getBigQueryAuthToolInfo(tableNameAuth)
|
||||
teardownTable2 := setupBigQueryTable(t, ctx, client, create_statement2, insert_statement2, datasetName, tableNameAuth, params2)
|
||||
createStatement2, insertStatement2, toolStatement2, params2 := getBigQueryAuthToolInfo(tableNameAuth)
|
||||
teardownTable2 := setupBigQueryTable(t, ctx, client, createStatement2, insertStatement2, datasetName, tableNameAuth, params2)
|
||||
defer teardownTable2(t)
|
||||
|
||||
// Write config into a file and pass it to command
|
||||
toolsFile := tests.GetToolsConfig(sourceConfig, BIGQUERY_TOOL_KIND, tool_statement1, tool_statement2)
|
||||
toolsFile := tests.GetToolsConfig(sourceConfig, BigqueryToolKind, toolStatement1, toolStatement2)
|
||||
toolsFile = addBigQueryPrebuiltToolsConfig(t, toolsFile)
|
||||
tmplSelectCombined, tmplSelectFilterCombined := getBigQueryTmplToolStatement()
|
||||
toolsFile = tests.AddTemplateParamConfig(t, toolsFile, BIGQUERY_TOOL_KIND, tmplSelectCombined, tmplSelectFilterCombined, "")
|
||||
toolsFile = tests.AddTemplateParamConfig(t, toolsFile, BigqueryToolKind, tmplSelectCombined, tmplSelectFilterCombined, "")
|
||||
|
||||
cmd, cleanup, err := tests.StartCmd(ctx, toolsFile, args...)
|
||||
if err != nil {
|
||||
@@ -124,7 +125,7 @@ func TestBigQueryToolEndpoints(t *testing.T) {
|
||||
|
||||
waitCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := cmd.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`))
|
||||
out, err := testutils.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`), cmd.Out)
|
||||
if err != nil {
|
||||
t.Logf("toolbox command logs: \n%s", out)
|
||||
t.Fatalf("toolbox didn't start successfully: %s", err)
|
||||
@@ -176,7 +177,7 @@ func getBigQueryAuthToolInfo(tableName string) (string, string, string, []bigque
|
||||
toolStatement := fmt.Sprintf(`
|
||||
SELECT name FROM %s WHERE email = ?`, tableName)
|
||||
params := []bigqueryapi.QueryParameter{
|
||||
{Value: int64(1)}, {Value: "Alice"}, {Value: tests.SERVICE_ACCOUNT_EMAIL},
|
||||
{Value: int64(1)}, {Value: "Alice"}, {Value: tests.ServiceAccountEmail},
|
||||
{Value: int64(2)}, {Value: "Jane"}, {Value: "janedoe@gmail.com"},
|
||||
}
|
||||
return createStatement, insertStatement, toolStatement, params
|
||||
@@ -189,7 +190,7 @@ func getBigQueryTmplToolStatement() (string, string) {
|
||||
return tmplSelectCombined, tmplSelectFilterCombined
|
||||
}
|
||||
|
||||
func setupBigQueryTable(t *testing.T, ctx context.Context, client *bigqueryapi.Client, create_statement, insert_statement, datasetName string, tableName string, params []bigqueryapi.QueryParameter) func(*testing.T) {
|
||||
func setupBigQueryTable(t *testing.T, ctx context.Context, client *bigqueryapi.Client, createStatement, insertStatement, datasetName string, tableName string, params []bigqueryapi.QueryParameter) func(*testing.T) {
|
||||
// Create dataset
|
||||
dataset := client.Dataset(datasetName)
|
||||
_, err := dataset.Metadata(ctx)
|
||||
@@ -206,7 +207,7 @@ func setupBigQueryTable(t *testing.T, ctx context.Context, client *bigqueryapi.C
|
||||
}
|
||||
|
||||
// Create table
|
||||
createJob, err := client.Query(create_statement).Run(ctx)
|
||||
createJob, err := client.Query(createStatement).Run(ctx)
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to start create table job for %s: %v", tableName, err)
|
||||
@@ -220,7 +221,7 @@ func setupBigQueryTable(t *testing.T, ctx context.Context, client *bigqueryapi.C
|
||||
}
|
||||
|
||||
// Insert test data
|
||||
insertQuery := client.Query(insert_statement)
|
||||
insertQuery := client.Query(insertStatement)
|
||||
insertQuery.Parameters = params
|
||||
insertJob, err := insertQuery.Run(ctx)
|
||||
if err != nil {
|
||||
@@ -340,7 +341,7 @@ func addBigQueryPrebuiltToolsConfig(t *testing.T, config map[string]any) map[str
|
||||
return config
|
||||
}
|
||||
|
||||
func runBigQueryExecuteSqlToolInvokeTest(t *testing.T, select_1_want, invokeParamWant, tableNameParam string) {
|
||||
func runBigQueryExecuteSqlToolInvokeTest(t *testing.T, select1Want, invokeParamWant, tableNameParam string) {
|
||||
// Get ID token
|
||||
idToken, err := tests.GetGoogleIdToken(tests.ClientId)
|
||||
if err != nil {
|
||||
@@ -368,7 +369,7 @@ func runBigQueryExecuteSqlToolInvokeTest(t *testing.T, select_1_want, invokePara
|
||||
api: "http://127.0.0.1:5000/api/tool/my-exec-sql-tool/invoke",
|
||||
requestHeader: map[string]string{},
|
||||
requestBody: bytes.NewBuffer([]byte(`{"sql":"SELECT 1"}`)),
|
||||
want: select_1_want,
|
||||
want: select1Want,
|
||||
isErr: false,
|
||||
},
|
||||
{
|
||||
@@ -414,7 +415,7 @@ func runBigQueryExecuteSqlToolInvokeTest(t *testing.T, select_1_want, invokePara
|
||||
requestHeader: map[string]string{"my-google-auth_token": idToken},
|
||||
requestBody: bytes.NewBuffer([]byte(`{"sql":"SELECT 1"}`)),
|
||||
isErr: false,
|
||||
want: select_1_want,
|
||||
want: select1Want,
|
||||
},
|
||||
{
|
||||
name: "Invoke my-auth-exec-sql-tool with invalid auth token",
|
||||
|
||||
@@ -29,29 +29,30 @@ import (
|
||||
|
||||
"cloud.google.com/go/bigtable"
|
||||
"github.com/google/uuid"
|
||||
"github.com/googleapis/genai-toolbox/internal/testutils"
|
||||
"github.com/googleapis/genai-toolbox/internal/tools"
|
||||
"github.com/googleapis/genai-toolbox/tests"
|
||||
)
|
||||
|
||||
var (
|
||||
BIGTABLE_SOURCE_KIND = "bigtable"
|
||||
BIGTABLE_TOOL_KIND = "bigtable-sql"
|
||||
BIGTABLE_PROJECT = os.Getenv("BIGTABLE_PROJECT")
|
||||
BIGTABLE_INSTANCE = os.Getenv("BIGTABLE_INSTANCE")
|
||||
BigtableSourceKind = "bigtable"
|
||||
BigtableToolKind = "bigtable-sql"
|
||||
BigtableProject = os.Getenv("BIGTABLE_PROJECT")
|
||||
BigtableInstance = os.Getenv("BIGTABLE_INSTANCE")
|
||||
)
|
||||
|
||||
func getBigtableVars(t *testing.T) map[string]any {
|
||||
switch "" {
|
||||
case BIGTABLE_PROJECT:
|
||||
case BigtableProject:
|
||||
t.Fatal("'BIGTABLE_PROJECT' not set")
|
||||
case BIGTABLE_INSTANCE:
|
||||
case BigtableInstance:
|
||||
t.Fatal("'BIGTABLE_INSTANCE' not set")
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"kind": BIGTABLE_SOURCE_KIND,
|
||||
"project": BIGTABLE_PROJECT,
|
||||
"instance": BIGTABLE_INSTANCE,
|
||||
"kind": BigtableSourceKind,
|
||||
"project": BigtableProject,
|
||||
"instance": BigtableInstance,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -77,13 +78,13 @@ func TestBigtableToolEndpoints(t *testing.T) {
|
||||
|
||||
// Do not change the shape of statement without checking tests/common_test.go.
|
||||
// The structure and value of seed data has to match https://github.com/googleapis/genai-toolbox/blob/4dba0df12dc438eca3cb476ef52aa17cdf232c12/tests/common_test.go#L200-L251
|
||||
param_test_statement := fmt.Sprintf("SELECT TO_INT64(cf['id']) as id, CAST(cf['name'] AS string) as name, FROM %s WHERE TO_INT64(cf['id']) = @id OR CAST(cf['name'] AS string) = @name;", tableName)
|
||||
paramTestStatement := fmt.Sprintf("SELECT TO_INT64(cf['id']) as id, CAST(cf['name'] AS string) as name, FROM %s WHERE TO_INT64(cf['id']) = @id OR CAST(cf['name'] AS string) = @name;", tableName)
|
||||
teardownTable1 := setupBtTable(t, ctx, sourceConfig["project"].(string), sourceConfig["instance"].(string), tableName, columnFamilyName, muts, rowKeys)
|
||||
defer teardownTable1(t)
|
||||
|
||||
// Do not change the shape of statement without checking tests/common_test.go.
|
||||
// The structure and value of seed data has to match https://github.com/googleapis/genai-toolbox/blob/4dba0df12dc438eca3cb476ef52aa17cdf232c12/tests/common_test.go#L200-L251
|
||||
auth_tool_statement := fmt.Sprintf("SELECT CAST(cf['name'] AS string) as name FROM %s WHERE CAST(cf['email'] AS string) = @email;", tableNameAuth)
|
||||
authToolStatement := fmt.Sprintf("SELECT CAST(cf['name'] AS string) as name FROM %s WHERE CAST(cf['email'] AS string) = @email;", tableNameAuth)
|
||||
teardownTable2 := setupBtTable(t, ctx, sourceConfig["project"].(string), sourceConfig["instance"].(string), tableNameAuth, columnFamilyName, muts, rowKeys)
|
||||
defer teardownTable2(t)
|
||||
|
||||
@@ -92,7 +93,7 @@ func TestBigtableToolEndpoints(t *testing.T) {
|
||||
defer teardownTableTmpl(t)
|
||||
|
||||
// Write config into a file and pass it to command
|
||||
toolsFile := tests.GetToolsConfig(sourceConfig, BIGTABLE_TOOL_KIND, param_test_statement, auth_tool_statement)
|
||||
toolsFile := tests.GetToolsConfig(sourceConfig, BigtableToolKind, paramTestStatement, authToolStatement)
|
||||
toolsFile = addTemplateParamConfig(t, toolsFile)
|
||||
|
||||
cmd, cleanup, err := tests.StartCmd(ctx, toolsFile, args...)
|
||||
@@ -103,7 +104,7 @@ func TestBigtableToolEndpoints(t *testing.T) {
|
||||
|
||||
waitCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := cmd.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`))
|
||||
out, err := testutils.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`), cmd.Out)
|
||||
if err != nil {
|
||||
t.Logf("toolbox command logs: \n%s", out)
|
||||
t.Fatalf("toolbox didn't start successfully: %s", err)
|
||||
@@ -151,7 +152,7 @@ func getTestData(columnFamilyName string) ([]*bigtable.Mutation, []string) {
|
||||
// Expected values are defined in https://github.com/googleapis/genai-toolbox/blob/52b09a67cb40ac0c5f461598b4673136699a3089/tests/tool_test.go#L229-L310
|
||||
"row-01": {
|
||||
"name": []byte("Alice"),
|
||||
"email": []byte(tests.SERVICE_ACCOUNT_EMAIL),
|
||||
"email": []byte(tests.ServiceAccountEmail),
|
||||
"id": ids[0],
|
||||
},
|
||||
"row-02": {
|
||||
|
||||
@@ -29,53 +29,54 @@ import (
|
||||
"cloud.google.com/go/cloudsqlconn"
|
||||
"cloud.google.com/go/cloudsqlconn/sqlserver/mssql"
|
||||
"github.com/google/uuid"
|
||||
"github.com/googleapis/genai-toolbox/internal/testutils"
|
||||
"github.com/googleapis/genai-toolbox/tests"
|
||||
)
|
||||
|
||||
var (
|
||||
CLOUD_SQL_MSSQL_SOURCE_KIND = "cloud-sql-mssql"
|
||||
CLOUD_SQL_MSSQL_TOOL_KIND = "mssql-sql"
|
||||
CLOUD_SQL_MSSQL_PROJECT = os.Getenv("CLOUD_SQL_MSSQL_PROJECT")
|
||||
CLOUD_SQL_MSSQL_REGION = os.Getenv("CLOUD_SQL_MSSQL_REGION")
|
||||
CLOUD_SQL_MSSQL_INSTANCE = os.Getenv("CLOUD_SQL_MSSQL_INSTANCE")
|
||||
CLOUD_SQL_MSSQL_DATABASE = os.Getenv("CLOUD_SQL_MSSQL_DATABASE")
|
||||
CLOUD_SQL_MSSQL_IP = os.Getenv("CLOUD_SQL_MSSQL_IP")
|
||||
CLOUD_SQL_MSSQL_USER = os.Getenv("CLOUD_SQL_MSSQL_USER")
|
||||
CLOUD_SQL_MSSQL_PASS = os.Getenv("CLOUD_SQL_MSSQL_PASS")
|
||||
CloudSQLMSSQLSourceKind = "cloud-sql-mssql"
|
||||
CloudSQLMSSQLToolKind = "mssql-sql"
|
||||
CloudSQLMSSQLProject = os.Getenv("CLOUD_SQL_MSSQL_PROJECT")
|
||||
CloudSQLMSSQLRegion = os.Getenv("CLOUD_SQL_MSSQL_REGION")
|
||||
CloudSQLMSSQLInstance = os.Getenv("CLOUD_SQL_MSSQL_INSTANCE")
|
||||
CloudSQLMSSQLDatabase = os.Getenv("CLOUD_SQL_MSSQL_DATABASE")
|
||||
CloudSQLMSSQLIp = os.Getenv("CLOUD_SQL_MSSQL_IP")
|
||||
CloudSQLMSSQLUser = os.Getenv("CLOUD_SQL_MSSQL_USER")
|
||||
CloudSQLMSSQLPass = os.Getenv("CLOUD_SQL_MSSQL_PASS")
|
||||
)
|
||||
|
||||
func getCloudSQLMssqlVars(t *testing.T) map[string]any {
|
||||
func getCloudSQLMSSQLVars(t *testing.T) map[string]any {
|
||||
switch "" {
|
||||
case CLOUD_SQL_MSSQL_PROJECT:
|
||||
case CloudSQLMSSQLProject:
|
||||
t.Fatal("'CLOUD_SQL_MSSQL_PROJECT' not set")
|
||||
case CLOUD_SQL_MSSQL_REGION:
|
||||
case CloudSQLMSSQLRegion:
|
||||
t.Fatal("'CLOUD_SQL_MSSQL_REGION' not set")
|
||||
case CLOUD_SQL_MSSQL_INSTANCE:
|
||||
case CloudSQLMSSQLInstance:
|
||||
t.Fatal("'CLOUD_SQL_MSSQL_INSTANCE' not set")
|
||||
case CLOUD_SQL_MSSQL_IP:
|
||||
case CloudSQLMSSQLIp:
|
||||
t.Fatal("'CLOUD_SQL_MSSQL_IP' not set")
|
||||
case CLOUD_SQL_MSSQL_DATABASE:
|
||||
case CloudSQLMSSQLDatabase:
|
||||
t.Fatal("'CLOUD_SQL_MSSQL_DATABASE' not set")
|
||||
case CLOUD_SQL_MSSQL_USER:
|
||||
case CloudSQLMSSQLUser:
|
||||
t.Fatal("'CLOUD_SQL_MSSQL_USER' not set")
|
||||
case CLOUD_SQL_MSSQL_PASS:
|
||||
case CloudSQLMSSQLPass:
|
||||
t.Fatal("'CLOUD_SQL_MSSQL_PASS' not set")
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"kind": CLOUD_SQL_MSSQL_SOURCE_KIND,
|
||||
"project": CLOUD_SQL_MSSQL_PROJECT,
|
||||
"instance": CLOUD_SQL_MSSQL_INSTANCE,
|
||||
"ipAddress": CLOUD_SQL_MSSQL_IP,
|
||||
"region": CLOUD_SQL_MSSQL_REGION,
|
||||
"database": CLOUD_SQL_MSSQL_DATABASE,
|
||||
"user": CLOUD_SQL_MSSQL_USER,
|
||||
"password": CLOUD_SQL_MSSQL_PASS,
|
||||
"kind": CloudSQLMSSQLSourceKind,
|
||||
"project": CloudSQLMSSQLProject,
|
||||
"instance": CloudSQLMSSQLInstance,
|
||||
"ipAddress": CloudSQLMSSQLIp,
|
||||
"region": CloudSQLMSSQLRegion,
|
||||
"database": CloudSQLMSSQLDatabase,
|
||||
"user": CloudSQLMSSQLUser,
|
||||
"password": CloudSQLMSSQLPass,
|
||||
}
|
||||
}
|
||||
|
||||
// Copied over from cloud_sql_mssql.go
|
||||
func initCloudSQLMssqlConnection(project, region, instance, ipAddress, ipType, user, pass, dbname string) (*sql.DB, error) {
|
||||
func initCloudSQLMSSQLConnection(project, region, instance, ipAddress, ipType, user, pass, dbname string) (*sql.DB, error) {
|
||||
// Create dsn
|
||||
query := fmt.Sprintf("database=%s&cloudsql=%s:%s:%s", dbname, project, region, instance)
|
||||
url := &url.URL{
|
||||
@@ -110,14 +111,14 @@ func initCloudSQLMssqlConnection(project, region, instance, ipAddress, ipType, u
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func TestCloudSQLMssqlToolEndpoints(t *testing.T) {
|
||||
sourceConfig := getCloudSQLMssqlVars(t)
|
||||
func TestCloudSQLMSSQLToolEndpoints(t *testing.T) {
|
||||
sourceConfig := getCloudSQLMSSQLVars(t)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||
defer cancel()
|
||||
|
||||
var args []string
|
||||
|
||||
db, err := initCloudSQLMssqlConnection(CLOUD_SQL_MSSQL_PROJECT, CLOUD_SQL_MSSQL_REGION, CLOUD_SQL_MSSQL_INSTANCE, CLOUD_SQL_MSSQL_IP, "public", CLOUD_SQL_MSSQL_USER, CLOUD_SQL_MSSQL_PASS, CLOUD_SQL_MSSQL_DATABASE)
|
||||
db, err := initCloudSQLMSSQLConnection(CloudSQLMSSQLProject, CloudSQLMSSQLRegion, CloudSQLMSSQLInstance, CloudSQLMSSQLIp, "public", CloudSQLMSSQLUser, CloudSQLMSSQLPass, CloudSQLMSSQLDatabase)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to create Cloud SQL connection pool: %s", err)
|
||||
}
|
||||
@@ -128,20 +129,20 @@ func TestCloudSQLMssqlToolEndpoints(t *testing.T) {
|
||||
tableNameTemplateParam := "template_param_table_" + strings.ReplaceAll(uuid.New().String(), "-", "")
|
||||
|
||||
// set up data for param tool
|
||||
create_statement1, insert_statement1, tool_statement1, params1 := tests.GetMssqlParamToolInfo(tableNameParam)
|
||||
teardownTable1 := tests.SetupMsSQLTable(t, ctx, db, create_statement1, insert_statement1, tableNameParam, params1)
|
||||
createStatement1, insertStatement1, toolStatement1, params1 := tests.GetMSSQLParamToolInfo(tableNameParam)
|
||||
teardownTable1 := tests.SetupMsSQLTable(t, ctx, db, createStatement1, insertStatement1, tableNameParam, params1)
|
||||
defer teardownTable1(t)
|
||||
|
||||
// set up data for auth tool
|
||||
create_statement2, insert_statement2, tool_statement2, params2 := tests.GetMssqlAuthToolInfo(tableNameAuth)
|
||||
teardownTable2 := tests.SetupMsSQLTable(t, ctx, db, create_statement2, insert_statement2, tableNameAuth, params2)
|
||||
createStatement2, insertStatement2, toolStatement2, params2 := tests.GetMSSQLAuthToolInfo(tableNameAuth)
|
||||
teardownTable2 := tests.SetupMsSQLTable(t, ctx, db, createStatement2, insertStatement2, tableNameAuth, params2)
|
||||
defer teardownTable2(t)
|
||||
|
||||
// Write config into a file and pass it to command
|
||||
toolsFile := tests.GetToolsConfig(sourceConfig, CLOUD_SQL_MSSQL_TOOL_KIND, tool_statement1, tool_statement2)
|
||||
toolsFile = tests.AddMssqlExecuteSqlConfig(t, toolsFile)
|
||||
tmplSelectCombined, tmplSelectFilterCombined := tests.GetMssqlTmplToolStatement()
|
||||
toolsFile = tests.AddTemplateParamConfig(t, toolsFile, CLOUD_SQL_MSSQL_TOOL_KIND, tmplSelectCombined, tmplSelectFilterCombined, "")
|
||||
toolsFile := tests.GetToolsConfig(sourceConfig, CloudSQLMSSQLToolKind, toolStatement1, toolStatement2)
|
||||
toolsFile = tests.AddMSSQLExecuteSqlConfig(t, toolsFile)
|
||||
tmplSelectCombined, tmplSelectFilterCombined := tests.GetMSSQLTmplToolStatement()
|
||||
toolsFile = tests.AddTemplateParamConfig(t, toolsFile, CloudSQLMSSQLToolKind, tmplSelectCombined, tmplSelectFilterCombined, "")
|
||||
|
||||
cmd, cleanup, err := tests.StartCmd(ctx, toolsFile, args...)
|
||||
if err != nil {
|
||||
@@ -151,7 +152,7 @@ func TestCloudSQLMssqlToolEndpoints(t *testing.T) {
|
||||
|
||||
waitCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := cmd.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`))
|
||||
out, err := testutils.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`), cmd.Out)
|
||||
if err != nil {
|
||||
t.Logf("toolbox command logs: \n%s", out)
|
||||
t.Fatalf("toolbox didn't start successfully: %s", err)
|
||||
@@ -159,7 +160,7 @@ func TestCloudSQLMssqlToolEndpoints(t *testing.T) {
|
||||
|
||||
tests.RunToolGetTest(t)
|
||||
|
||||
select1Want, failInvocationWant, createTableStatement := tests.GetMssqlWants()
|
||||
select1Want, failInvocationWant, createTableStatement := tests.GetMSSQLWants()
|
||||
invokeParamWant, mcpInvokeParamWant := tests.GetNonSpannerInvokeParamWant()
|
||||
tests.RunToolInvokeTest(t, select1Want, invokeParamWant)
|
||||
tests.RunExecuteSqlToolInvokeTest(t, createTableStatement, select1Want)
|
||||
@@ -168,8 +169,8 @@ func TestCloudSQLMssqlToolEndpoints(t *testing.T) {
|
||||
}
|
||||
|
||||
// Test connection with different IP type
|
||||
func TestCloudSQLMssqlIpConnection(t *testing.T) {
|
||||
sourceConfig := getCloudSQLMssqlVars(t)
|
||||
func TestCloudSQLMSSQLIpConnection(t *testing.T) {
|
||||
sourceConfig := getCloudSQLMSSQLVars(t)
|
||||
|
||||
tcs := []struct {
|
||||
name string
|
||||
@@ -187,7 +188,7 @@ func TestCloudSQLMssqlIpConnection(t *testing.T) {
|
||||
for _, tc := range tcs {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
sourceConfig["ipType"] = tc.ipType
|
||||
err := tests.RunSourceConnectionTest(t, sourceConfig, CLOUD_SQL_MSSQL_TOOL_KIND)
|
||||
err := tests.RunSourceConnectionTest(t, sourceConfig, CloudSQLMSSQLToolKind)
|
||||
if err != nil {
|
||||
t.Fatalf("Connection test failure: %s", err)
|
||||
}
|
||||
|
||||
@@ -28,44 +28,45 @@ import (
|
||||
"cloud.google.com/go/cloudsqlconn"
|
||||
"cloud.google.com/go/cloudsqlconn/mysql/mysql"
|
||||
"github.com/google/uuid"
|
||||
"github.com/googleapis/genai-toolbox/internal/testutils"
|
||||
"github.com/googleapis/genai-toolbox/tests"
|
||||
)
|
||||
|
||||
var (
|
||||
CLOUD_SQL_MYSQL_SOURCE_KIND = "cloud-sql-mysql"
|
||||
CLOUD_SQL_MYSQL_TOOL_KIND = "mysql-sql"
|
||||
CLOUD_SQL_MYSQL_PROJECT = os.Getenv("CLOUD_SQL_MYSQL_PROJECT")
|
||||
CLOUD_SQL_MYSQL_REGION = os.Getenv("CLOUD_SQL_MYSQL_REGION")
|
||||
CLOUD_SQL_MYSQL_INSTANCE = os.Getenv("CLOUD_SQL_MYSQL_INSTANCE")
|
||||
CLOUD_SQL_MYSQL_DATABASE = os.Getenv("CLOUD_SQL_MYSQL_DATABASE")
|
||||
CLOUD_SQL_MYSQL_USER = os.Getenv("CLOUD_SQL_MYSQL_USER")
|
||||
CLOUD_SQL_MYSQL_PASS = os.Getenv("CLOUD_SQL_MYSQL_PASS")
|
||||
CloudSQLMySQLSourceKind = "cloud-sql-mysql"
|
||||
CloudSQLMySQLToolKind = "mysql-sql"
|
||||
CloudSQLMySQLProject = os.Getenv("CLOUD_SQL_MYSQL_PROJECT")
|
||||
CloudSQLMySQLRegion = os.Getenv("CLOUD_SQL_MYSQL_REGION")
|
||||
CloudSQLMySQLInstance = os.Getenv("CLOUD_SQL_MYSQL_INSTANCE")
|
||||
CloudSQLMySQLDatabase = os.Getenv("CLOUD_SQL_MYSQL_DATABASE")
|
||||
CloudSQLMySQLUser = os.Getenv("CLOUD_SQL_MYSQL_USER")
|
||||
CloudSQLMySQLPass = os.Getenv("CLOUD_SQL_MYSQL_PASS")
|
||||
)
|
||||
|
||||
func getCloudSQLMySQLVars(t *testing.T) map[string]any {
|
||||
switch "" {
|
||||
case CLOUD_SQL_MYSQL_PROJECT:
|
||||
case CloudSQLMySQLProject:
|
||||
t.Fatal("'CLOUD_SQL_MYSQL_PROJECT' not set")
|
||||
case CLOUD_SQL_MYSQL_REGION:
|
||||
case CloudSQLMySQLRegion:
|
||||
t.Fatal("'CLOUD_SQL_MYSQL_REGION' not set")
|
||||
case CLOUD_SQL_MYSQL_INSTANCE:
|
||||
case CloudSQLMySQLInstance:
|
||||
t.Fatal("'CLOUD_SQL_MYSQL_INSTANCE' not set")
|
||||
case CLOUD_SQL_MYSQL_DATABASE:
|
||||
case CloudSQLMySQLDatabase:
|
||||
t.Fatal("'CLOUD_SQL_MYSQL_DATABASE' not set")
|
||||
case CLOUD_SQL_MYSQL_USER:
|
||||
case CloudSQLMySQLUser:
|
||||
t.Fatal("'CLOUD_SQL_MYSQL_USER' not set")
|
||||
case CLOUD_SQL_MYSQL_PASS:
|
||||
case CloudSQLMySQLPass:
|
||||
t.Fatal("'CLOUD_SQL_MYSQL_PASS' not set")
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"kind": CLOUD_SQL_MYSQL_SOURCE_KIND,
|
||||
"project": CLOUD_SQL_MYSQL_PROJECT,
|
||||
"instance": CLOUD_SQL_MYSQL_INSTANCE,
|
||||
"region": CLOUD_SQL_MYSQL_REGION,
|
||||
"database": CLOUD_SQL_MYSQL_DATABASE,
|
||||
"user": CLOUD_SQL_MYSQL_USER,
|
||||
"password": CLOUD_SQL_MYSQL_PASS,
|
||||
"kind": CloudSQLMySQLSourceKind,
|
||||
"project": CloudSQLMySQLProject,
|
||||
"instance": CloudSQLMySQLInstance,
|
||||
"region": CloudSQLMySQLRegion,
|
||||
"database": CloudSQLMySQLDatabase,
|
||||
"user": CloudSQLMySQLUser,
|
||||
"password": CloudSQLMySQLPass,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,14 +98,14 @@ func initCloudSQLMySQLConnectionPool(project, region, instance, ipType, user, pa
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func TestCloudSQLMysqlToolEndpoints(t *testing.T) {
|
||||
func TestCloudSQLMySQLToolEndpoints(t *testing.T) {
|
||||
sourceConfig := getCloudSQLMySQLVars(t)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||
defer cancel()
|
||||
|
||||
var args []string
|
||||
|
||||
pool, err := initCloudSQLMySQLConnectionPool(CLOUD_SQL_MYSQL_PROJECT, CLOUD_SQL_MYSQL_REGION, CLOUD_SQL_MYSQL_INSTANCE, "public", CLOUD_SQL_MYSQL_USER, CLOUD_SQL_MYSQL_PASS, CLOUD_SQL_MYSQL_DATABASE)
|
||||
pool, err := initCloudSQLMySQLConnectionPool(CloudSQLMySQLProject, CloudSQLMySQLRegion, CloudSQLMySQLInstance, "public", CloudSQLMySQLUser, CloudSQLMySQLPass, CloudSQLMySQLDatabase)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to create Cloud SQL connection pool: %s", err)
|
||||
}
|
||||
@@ -115,20 +116,20 @@ func TestCloudSQLMysqlToolEndpoints(t *testing.T) {
|
||||
tableNameTemplateParam := "template_param_table_" + strings.ReplaceAll(uuid.New().String(), "-", "")
|
||||
|
||||
// set up data for param tool
|
||||
create_statement1, insert_statement1, tool_statement1, params1 := tests.GetMysqlParamToolInfo(tableNameParam)
|
||||
teardownTable1 := tests.SetupMySQLTable(t, ctx, pool, create_statement1, insert_statement1, tableNameParam, params1)
|
||||
createStatement1, insertStatement1, toolStatement1, params1 := tests.GetMySQLParamToolInfo(tableNameParam)
|
||||
teardownTable1 := tests.SetupMySQLTable(t, ctx, pool, createStatement1, insertStatement1, tableNameParam, params1)
|
||||
defer teardownTable1(t)
|
||||
|
||||
// set up data for auth tool
|
||||
create_statement2, insert_statement2, tool_statement2, params2 := tests.GetMysqlAuthToolInfo(tableNameAuth)
|
||||
teardownTable2 := tests.SetupMySQLTable(t, ctx, pool, create_statement2, insert_statement2, tableNameAuth, params2)
|
||||
createStatement2, insertStatement2, toolStatement2, params2 := tests.GetMySQLAuthToolInfo(tableNameAuth)
|
||||
teardownTable2 := tests.SetupMySQLTable(t, ctx, pool, createStatement2, insertStatement2, tableNameAuth, params2)
|
||||
defer teardownTable2(t)
|
||||
|
||||
// Write config into a file and pass it to command
|
||||
toolsFile := tests.GetToolsConfig(sourceConfig, CLOUD_SQL_MYSQL_TOOL_KIND, tool_statement1, tool_statement2)
|
||||
toolsFile := tests.GetToolsConfig(sourceConfig, CloudSQLMySQLToolKind, toolStatement1, toolStatement2)
|
||||
toolsFile = tests.AddMySqlExecuteSqlConfig(t, toolsFile)
|
||||
tmplSelectCombined, tmplSelectFilterCombined := tests.GetMysqlTmplToolStatement()
|
||||
toolsFile = tests.AddTemplateParamConfig(t, toolsFile, CLOUD_SQL_MYSQL_TOOL_KIND, tmplSelectCombined, tmplSelectFilterCombined, "")
|
||||
tmplSelectCombined, tmplSelectFilterCombined := tests.GetMySQLTmplToolStatement()
|
||||
toolsFile = tests.AddTemplateParamConfig(t, toolsFile, CloudSQLMySQLToolKind, tmplSelectCombined, tmplSelectFilterCombined, "")
|
||||
|
||||
cmd, cleanup, err := tests.StartCmd(ctx, toolsFile, args...)
|
||||
if err != nil {
|
||||
@@ -138,7 +139,7 @@ func TestCloudSQLMysqlToolEndpoints(t *testing.T) {
|
||||
|
||||
waitCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := cmd.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`))
|
||||
out, err := testutils.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`), cmd.Out)
|
||||
if err != nil {
|
||||
t.Logf("toolbox command logs: \n%s", out)
|
||||
t.Fatalf("toolbox didn't start successfully: %s", err)
|
||||
@@ -146,7 +147,7 @@ func TestCloudSQLMysqlToolEndpoints(t *testing.T) {
|
||||
|
||||
tests.RunToolGetTest(t)
|
||||
|
||||
select1Want, failInvocationWant, createTableStatement := tests.GetMysqlWants()
|
||||
select1Want, failInvocationWant, createTableStatement := tests.GetMySQLWants()
|
||||
invokeParamWant, mcpInvokeParamWant := tests.GetNonSpannerInvokeParamWant()
|
||||
tests.RunToolInvokeTest(t, select1Want, invokeParamWant)
|
||||
tests.RunExecuteSqlToolInvokeTest(t, createTableStatement, select1Want)
|
||||
@@ -155,7 +156,7 @@ func TestCloudSQLMysqlToolEndpoints(t *testing.T) {
|
||||
}
|
||||
|
||||
// Test connection with different IP type
|
||||
func TestCloudSQLMysqlIpConnection(t *testing.T) {
|
||||
func TestCloudSQLMySQLIpConnection(t *testing.T) {
|
||||
sourceConfig := getCloudSQLMySQLVars(t)
|
||||
|
||||
tcs := []struct {
|
||||
@@ -174,7 +175,7 @@ func TestCloudSQLMysqlIpConnection(t *testing.T) {
|
||||
for _, tc := range tcs {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
sourceConfig["ipType"] = tc.ipType
|
||||
err := tests.RunSourceConnectionTest(t, sourceConfig, CLOUD_SQL_MYSQL_TOOL_KIND)
|
||||
err := tests.RunSourceConnectionTest(t, sourceConfig, CloudSQLMySQLToolKind)
|
||||
if err != nil {
|
||||
t.Fatalf("Connection test failure: %s", err)
|
||||
}
|
||||
|
||||
@@ -26,45 +26,46 @@ import (
|
||||
|
||||
"cloud.google.com/go/cloudsqlconn"
|
||||
"github.com/google/uuid"
|
||||
"github.com/googleapis/genai-toolbox/internal/testutils"
|
||||
"github.com/googleapis/genai-toolbox/tests"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
var (
|
||||
CLOUD_SQL_POSTGRES_SOURCE_KIND = "cloud-sql-postgres"
|
||||
CLOUD_SQL_POSTGRES_TOOL_KIND = "postgres-sql"
|
||||
CLOUD_SQL_POSTGRES_PROJECT = os.Getenv("CLOUD_SQL_POSTGRES_PROJECT")
|
||||
CLOUD_SQL_POSTGRES_REGION = os.Getenv("CLOUD_SQL_POSTGRES_REGION")
|
||||
CLOUD_SQL_POSTGRES_INSTANCE = os.Getenv("CLOUD_SQL_POSTGRES_INSTANCE")
|
||||
CLOUD_SQL_POSTGRES_DATABASE = os.Getenv("CLOUD_SQL_POSTGRES_DATABASE")
|
||||
CLOUD_SQL_POSTGRES_USER = os.Getenv("CLOUD_SQL_POSTGRES_USER")
|
||||
CLOUD_SQL_POSTGRES_PASS = os.Getenv("CLOUD_SQL_POSTGRES_PASS")
|
||||
CloudSQLPostgresSourceKind = "cloud-sql-postgres"
|
||||
CloudSQLPostgresToolKind = "postgres-sql"
|
||||
CloudSQLPostgresProject = os.Getenv("CLOUD_SQL_POSTGRES_PROJECT")
|
||||
CloudSQLPostgresRegion = os.Getenv("CLOUD_SQL_POSTGRES_REGION")
|
||||
CloudSQLPostgresInstance = os.Getenv("CLOUD_SQL_POSTGRES_INSTANCE")
|
||||
CloudSQLPostgresDatabase = os.Getenv("CLOUD_SQL_POSTGRES_DATABASE")
|
||||
CloudSQLPostgresUser = os.Getenv("CLOUD_SQL_POSTGRES_USER")
|
||||
CloudSQLPostgresPass = os.Getenv("CLOUD_SQL_POSTGRES_PASS")
|
||||
)
|
||||
|
||||
func getCloudSQLPgVars(t *testing.T) map[string]any {
|
||||
switch "" {
|
||||
case CLOUD_SQL_POSTGRES_PROJECT:
|
||||
case CloudSQLPostgresProject:
|
||||
t.Fatal("'CLOUD_SQL_POSTGRES_PROJECT' not set")
|
||||
case CLOUD_SQL_POSTGRES_REGION:
|
||||
case CloudSQLPostgresRegion:
|
||||
t.Fatal("'CLOUD_SQL_POSTGRES_REGION' not set")
|
||||
case CLOUD_SQL_POSTGRES_INSTANCE:
|
||||
case CloudSQLPostgresInstance:
|
||||
t.Fatal("'CLOUD_SQL_POSTGRES_INSTANCE' not set")
|
||||
case CLOUD_SQL_POSTGRES_DATABASE:
|
||||
case CloudSQLPostgresDatabase:
|
||||
t.Fatal("'CLOUD_SQL_POSTGRES_DATABASE' not set")
|
||||
case CLOUD_SQL_POSTGRES_USER:
|
||||
case CloudSQLPostgresUser:
|
||||
t.Fatal("'CLOUD_SQL_POSTGRES_USER' not set")
|
||||
case CLOUD_SQL_POSTGRES_PASS:
|
||||
case CloudSQLPostgresPass:
|
||||
t.Fatal("'CLOUD_SQL_POSTGRES_PASS' not set")
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"kind": CLOUD_SQL_POSTGRES_SOURCE_KIND,
|
||||
"project": CLOUD_SQL_POSTGRES_PROJECT,
|
||||
"instance": CLOUD_SQL_POSTGRES_INSTANCE,
|
||||
"region": CLOUD_SQL_POSTGRES_REGION,
|
||||
"database": CLOUD_SQL_POSTGRES_DATABASE,
|
||||
"user": CLOUD_SQL_POSTGRES_USER,
|
||||
"password": CLOUD_SQL_POSTGRES_PASS,
|
||||
"kind": CloudSQLPostgresSourceKind,
|
||||
"project": CloudSQLPostgresProject,
|
||||
"instance": CloudSQLPostgresInstance,
|
||||
"region": CloudSQLPostgresRegion,
|
||||
"database": CloudSQLPostgresDatabase,
|
||||
"user": CloudSQLPostgresUser,
|
||||
"password": CloudSQLPostgresPass,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -108,7 +109,7 @@ func TestCloudSQLPgSimpleToolEndpoints(t *testing.T) {
|
||||
|
||||
var args []string
|
||||
|
||||
pool, err := initCloudSQLPgConnectionPool(CLOUD_SQL_POSTGRES_PROJECT, CLOUD_SQL_POSTGRES_REGION, CLOUD_SQL_POSTGRES_INSTANCE, "public", CLOUD_SQL_POSTGRES_USER, CLOUD_SQL_POSTGRES_PASS, CLOUD_SQL_POSTGRES_DATABASE)
|
||||
pool, err := initCloudSQLPgConnectionPool(CloudSQLPostgresProject, CloudSQLPostgresRegion, CloudSQLPostgresInstance, "public", CloudSQLPostgresUser, CloudSQLPostgresPass, CloudSQLPostgresDatabase)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to create Cloud SQL connection pool: %s", err)
|
||||
}
|
||||
@@ -119,20 +120,20 @@ func TestCloudSQLPgSimpleToolEndpoints(t *testing.T) {
|
||||
tableNameTemplateParam := "template_param_table_" + strings.ReplaceAll(uuid.New().String(), "-", "")
|
||||
|
||||
// set up data for param tool
|
||||
create_statement1, insert_statement1, tool_statement1, params1 := tests.GetPostgresSQLParamToolInfo(tableNameParam)
|
||||
teardownTable1 := tests.SetupPostgresSQLTable(t, ctx, pool, create_statement1, insert_statement1, tableNameParam, params1)
|
||||
createStatement1, insertStatement1, toolStatement1, params1 := tests.GetPostgresSQLParamToolInfo(tableNameParam)
|
||||
teardownTable1 := tests.SetupPostgresSQLTable(t, ctx, pool, createStatement1, insertStatement1, tableNameParam, params1)
|
||||
defer teardownTable1(t)
|
||||
|
||||
// set up data for auth tool
|
||||
create_statement2, insert_statement2, tool_statement2, params2 := tests.GetPostgresSQLAuthToolInfo(tableNameAuth)
|
||||
teardownTable2 := tests.SetupPostgresSQLTable(t, ctx, pool, create_statement2, insert_statement2, tableNameAuth, params2)
|
||||
createStatement2, insertStatement2, toolStatement2, params2 := tests.GetPostgresSQLAuthToolInfo(tableNameAuth)
|
||||
teardownTable2 := tests.SetupPostgresSQLTable(t, ctx, pool, createStatement2, insertStatement2, tableNameAuth, params2)
|
||||
defer teardownTable2(t)
|
||||
|
||||
// Write config into a file and pass it to command
|
||||
toolsFile := tests.GetToolsConfig(sourceConfig, CLOUD_SQL_POSTGRES_TOOL_KIND, tool_statement1, tool_statement2)
|
||||
toolsFile := tests.GetToolsConfig(sourceConfig, CloudSQLPostgresToolKind, toolStatement1, toolStatement2)
|
||||
toolsFile = tests.AddPgExecuteSqlConfig(t, toolsFile)
|
||||
tmplSelectCombined, tmplSelectFilterCombined := tests.GetPostgresSQLTmplToolStatement()
|
||||
toolsFile = tests.AddTemplateParamConfig(t, toolsFile, CLOUD_SQL_POSTGRES_TOOL_KIND, tmplSelectCombined, tmplSelectFilterCombined, "")
|
||||
toolsFile = tests.AddTemplateParamConfig(t, toolsFile, CloudSQLPostgresToolKind, tmplSelectCombined, tmplSelectFilterCombined, "")
|
||||
|
||||
cmd, cleanup, err := tests.StartCmd(ctx, toolsFile, args...)
|
||||
if err != nil {
|
||||
@@ -142,7 +143,7 @@ func TestCloudSQLPgSimpleToolEndpoints(t *testing.T) {
|
||||
|
||||
waitCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := cmd.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`))
|
||||
out, err := testutils.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`), cmd.Out)
|
||||
if err != nil {
|
||||
t.Logf("toolbox command logs: \n%s", out)
|
||||
t.Fatalf("toolbox didn't start successfully: %s", err)
|
||||
@@ -178,7 +179,7 @@ func TestCloudSQLPgIpConnection(t *testing.T) {
|
||||
for _, tc := range tcs {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
sourceConfig["ipType"] = tc.ipType
|
||||
err := tests.RunSourceConnectionTest(t, sourceConfig, CLOUD_SQL_POSTGRES_TOOL_KIND)
|
||||
err := tests.RunSourceConnectionTest(t, sourceConfig, CloudSQLPostgresToolKind)
|
||||
if err != nil {
|
||||
t.Fatalf("Connection test failure: %s", err)
|
||||
}
|
||||
@@ -189,32 +190,32 @@ func TestCloudSQLPgIpConnection(t *testing.T) {
|
||||
func TestCloudSQLPgIAMConnection(t *testing.T) {
|
||||
getCloudSQLPgVars(t)
|
||||
// service account email used for IAM should trim the suffix
|
||||
serviceAccountEmail := strings.TrimSuffix(tests.SERVICE_ACCOUNT_EMAIL, ".gserviceaccount.com")
|
||||
serviceAccountEmail := strings.TrimSuffix(tests.ServiceAccountEmail, ".gserviceaccount.com")
|
||||
|
||||
noPassSourceConfig := map[string]any{
|
||||
"kind": CLOUD_SQL_POSTGRES_SOURCE_KIND,
|
||||
"project": CLOUD_SQL_POSTGRES_PROJECT,
|
||||
"instance": CLOUD_SQL_POSTGRES_INSTANCE,
|
||||
"region": CLOUD_SQL_POSTGRES_REGION,
|
||||
"database": CLOUD_SQL_POSTGRES_DATABASE,
|
||||
"kind": CloudSQLPostgresSourceKind,
|
||||
"project": CloudSQLPostgresProject,
|
||||
"instance": CloudSQLPostgresInstance,
|
||||
"region": CloudSQLPostgresRegion,
|
||||
"database": CloudSQLPostgresDatabase,
|
||||
"user": serviceAccountEmail,
|
||||
}
|
||||
|
||||
noUserSourceConfig := map[string]any{
|
||||
"kind": CLOUD_SQL_POSTGRES_SOURCE_KIND,
|
||||
"project": CLOUD_SQL_POSTGRES_PROJECT,
|
||||
"instance": CLOUD_SQL_POSTGRES_INSTANCE,
|
||||
"region": CLOUD_SQL_POSTGRES_REGION,
|
||||
"database": CLOUD_SQL_POSTGRES_DATABASE,
|
||||
"kind": CloudSQLPostgresSourceKind,
|
||||
"project": CloudSQLPostgresProject,
|
||||
"instance": CloudSQLPostgresInstance,
|
||||
"region": CloudSQLPostgresRegion,
|
||||
"database": CloudSQLPostgresDatabase,
|
||||
"password": "random",
|
||||
}
|
||||
|
||||
noUserNoPassSourceConfig := map[string]any{
|
||||
"kind": CLOUD_SQL_POSTGRES_SOURCE_KIND,
|
||||
"project": CLOUD_SQL_POSTGRES_PROJECT,
|
||||
"instance": CLOUD_SQL_POSTGRES_INSTANCE,
|
||||
"region": CLOUD_SQL_POSTGRES_REGION,
|
||||
"database": CLOUD_SQL_POSTGRES_DATABASE,
|
||||
"kind": CloudSQLPostgresSourceKind,
|
||||
"project": CloudSQLPostgresProject,
|
||||
"instance": CloudSQLPostgresInstance,
|
||||
"region": CloudSQLPostgresRegion,
|
||||
"database": CloudSQLPostgresDatabase,
|
||||
}
|
||||
tcs := []struct {
|
||||
name string
|
||||
@@ -239,7 +240,7 @@ func TestCloudSQLPgIAMConnection(t *testing.T) {
|
||||
}
|
||||
for _, tc := range tcs {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := tests.RunSourceConnectionTest(t, tc.sourceConfig, CLOUD_SQL_POSTGRES_TOOL_KIND)
|
||||
err := tests.RunSourceConnectionTest(t, tc.sourceConfig, CloudSQLPostgresToolKind)
|
||||
if err != nil {
|
||||
if tc.isErr {
|
||||
return
|
||||
|
||||
114
tests/common.go
114
tests/common.go
@@ -28,7 +28,7 @@ import (
|
||||
)
|
||||
|
||||
// GetToolsConfig returns a mock tools config file
|
||||
func GetToolsConfig(sourceConfig map[string]any, toolKind, param_tool_statement, auth_tool_statement string) map[string]any {
|
||||
func GetToolsConfig(sourceConfig map[string]any, toolKind, paramToolStatement, authToolStatement string) map[string]any {
|
||||
// Write config into a file and pass it to command
|
||||
toolsFile := map[string]any{
|
||||
"sources": map[string]any{
|
||||
@@ -51,7 +51,7 @@ func GetToolsConfig(sourceConfig map[string]any, toolKind, param_tool_statement,
|
||||
"kind": toolKind,
|
||||
"source": "my-instance",
|
||||
"description": "Tool to test invocation with params.",
|
||||
"statement": param_tool_statement,
|
||||
"statement": paramToolStatement,
|
||||
"parameters": []any{
|
||||
map[string]any{
|
||||
"name": "id",
|
||||
@@ -70,7 +70,7 @@ func GetToolsConfig(sourceConfig map[string]any, toolKind, param_tool_statement,
|
||||
"source": "my-instance",
|
||||
"description": "Tool to test authenticated parameters.",
|
||||
// statement to auto-fill authenticated parameter
|
||||
"statement": auth_tool_statement,
|
||||
"statement": authToolStatement,
|
||||
"parameters": []map[string]any{
|
||||
{
|
||||
"name": "email",
|
||||
@@ -237,8 +237,8 @@ func AddMySqlExecuteSqlConfig(t *testing.T, config map[string]any) map[string]an
|
||||
return config
|
||||
}
|
||||
|
||||
// AddMssqlExecuteSqlConfig gets the tools config for `mssql-execute-sql`
|
||||
func AddMssqlExecuteSqlConfig(t *testing.T, config map[string]any) map[string]any {
|
||||
// AddMSSQLExecuteSqlConfig gets the tools config for `mssql-execute-sql`
|
||||
func AddMSSQLExecuteSqlConfig(t *testing.T, config map[string]any) map[string]any {
|
||||
tools, ok := config["tools"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("unable to get tools from config")
|
||||
@@ -262,20 +262,20 @@ func AddMssqlExecuteSqlConfig(t *testing.T, config map[string]any) map[string]an
|
||||
|
||||
// GetPostgresSQLParamToolInfo returns statements and param for my-param-tool postgres-sql kind
|
||||
func GetPostgresSQLParamToolInfo(tableName string) (string, string, string, []any) {
|
||||
create_statement := fmt.Sprintf("CREATE TABLE %s (id SERIAL PRIMARY KEY, name TEXT);", tableName)
|
||||
insert_statement := fmt.Sprintf("INSERT INTO %s (name) VALUES ($1), ($2), ($3);", tableName)
|
||||
tool_statement := fmt.Sprintf("SELECT * FROM %s WHERE id = $1 OR name = $2;", tableName)
|
||||
createStatement := fmt.Sprintf("CREATE TABLE %s (id SERIAL PRIMARY KEY, name TEXT);", tableName)
|
||||
insertStatement := fmt.Sprintf("INSERT INTO %s (name) VALUES ($1), ($2), ($3);", tableName)
|
||||
toolStatement := fmt.Sprintf("SELECT * FROM %s WHERE id = $1 OR name = $2;", tableName)
|
||||
params := []any{"Alice", "Jane", "Sid"}
|
||||
return create_statement, insert_statement, tool_statement, params
|
||||
return createStatement, insertStatement, toolStatement, params
|
||||
}
|
||||
|
||||
// GetPostgresSQLAuthToolInfo returns statements and param of my-auth-tool for postgres-sql kind
|
||||
func GetPostgresSQLAuthToolInfo(tableName string) (string, string, string, []any) {
|
||||
create_statement := fmt.Sprintf("CREATE TABLE %s (id SERIAL PRIMARY KEY, name TEXT, email TEXT);", tableName)
|
||||
insert_statement := fmt.Sprintf("INSERT INTO %s (name, email) VALUES ($1, $2), ($3, $4)", tableName)
|
||||
tool_statement := fmt.Sprintf("SELECT name FROM %s WHERE email = $1;", tableName)
|
||||
params := []any{"Alice", SERVICE_ACCOUNT_EMAIL, "Jane", "janedoe@gmail.com"}
|
||||
return create_statement, insert_statement, tool_statement, params
|
||||
createStatement := fmt.Sprintf("CREATE TABLE %s (id SERIAL PRIMARY KEY, name TEXT, email TEXT);", tableName)
|
||||
insertStatement := fmt.Sprintf("INSERT INTO %s (name, email) VALUES ($1, $2), ($3, $4)", tableName)
|
||||
toolStatement := fmt.Sprintf("SELECT name FROM %s WHERE email = $1;", tableName)
|
||||
params := []any{"Alice", ServiceAccountEmail, "Jane", "janedoe@gmail.com"}
|
||||
return createStatement, insertStatement, toolStatement, params
|
||||
}
|
||||
|
||||
// GetPostgresSQLTmplToolStatement returns statements and param for template parameter test cases for postgres-sql kind
|
||||
@@ -285,51 +285,51 @@ func GetPostgresSQLTmplToolStatement() (string, string) {
|
||||
return tmplSelectCombined, tmplSelectFilterCombined
|
||||
}
|
||||
|
||||
// GetMssqlParamToolInfo returns statements and param for my-param-tool mssql-sql kind
|
||||
func GetMssqlParamToolInfo(tableName string) (string, string, string, []any) {
|
||||
create_statement := fmt.Sprintf("CREATE TABLE %s (id INT IDENTITY(1,1) PRIMARY KEY, name VARCHAR(255));", tableName)
|
||||
insert_statement := fmt.Sprintf("INSERT INTO %s (name) VALUES (@alice), (@jane), (@sid);", tableName)
|
||||
tool_statement := fmt.Sprintf("SELECT * FROM %s WHERE id = @id OR name = @p2;", tableName)
|
||||
// GetMSSQLParamToolInfo returns statements and param for my-param-tool mssql-sql kind
|
||||
func GetMSSQLParamToolInfo(tableName string) (string, string, string, []any) {
|
||||
createStatement := fmt.Sprintf("CREATE TABLE %s (id INT IDENTITY(1,1) PRIMARY KEY, name VARCHAR(255));", tableName)
|
||||
insertStatement := fmt.Sprintf("INSERT INTO %s (name) VALUES (@alice), (@jane), (@sid);", tableName)
|
||||
toolStatement := fmt.Sprintf("SELECT * FROM %s WHERE id = @id OR name = @p2;", tableName)
|
||||
params := []any{sql.Named("alice", "Alice"), sql.Named("jane", "Jane"), sql.Named("sid", "Sid")}
|
||||
return create_statement, insert_statement, tool_statement, params
|
||||
return createStatement, insertStatement, toolStatement, params
|
||||
}
|
||||
|
||||
// GetMssqlAuthToolInfo returns statements and param of my-auth-tool for mssql-sql kind
|
||||
func GetMssqlAuthToolInfo(tableName string) (string, string, string, []any) {
|
||||
create_statement := fmt.Sprintf("CREATE TABLE %s (id INT IDENTITY(1,1) PRIMARY KEY, name VARCHAR(255), email VARCHAR(255));", tableName)
|
||||
insert_statement := fmt.Sprintf("INSERT INTO %s (name, email) VALUES (@alice, @aliceemail), (@jane, @janeemail);", tableName)
|
||||
tool_statement := fmt.Sprintf("SELECT name FROM %s WHERE email = @email;", tableName)
|
||||
params := []any{sql.Named("alice", "Alice"), sql.Named("aliceemail", SERVICE_ACCOUNT_EMAIL), sql.Named("jane", "Jane"), sql.Named("janeemail", "janedoe@gmail.com")}
|
||||
return create_statement, insert_statement, tool_statement, params
|
||||
// GetMSSQLAuthToolInfo returns statements and param of my-auth-tool for mssql-sql kind
|
||||
func GetMSSQLAuthToolInfo(tableName string) (string, string, string, []any) {
|
||||
createStatement := fmt.Sprintf("CREATE TABLE %s (id INT IDENTITY(1,1) PRIMARY KEY, name VARCHAR(255), email VARCHAR(255));", tableName)
|
||||
insertStatement := fmt.Sprintf("INSERT INTO %s (name, email) VALUES (@alice, @aliceemail), (@jane, @janeemail);", tableName)
|
||||
toolStatement := fmt.Sprintf("SELECT name FROM %s WHERE email = @email;", tableName)
|
||||
params := []any{sql.Named("alice", "Alice"), sql.Named("aliceemail", ServiceAccountEmail), sql.Named("jane", "Jane"), sql.Named("janeemail", "janedoe@gmail.com")}
|
||||
return createStatement, insertStatement, toolStatement, params
|
||||
}
|
||||
|
||||
// GetMssqlTmplToolStatement returns statements and param for template parameter test cases for mysql-sql kind
|
||||
func GetMssqlTmplToolStatement() (string, string) {
|
||||
// GetMSSQLTmplToolStatement returns statements and param for template parameter test cases for mysql-sql kind
|
||||
func GetMSSQLTmplToolStatement() (string, string) {
|
||||
tmplSelectCombined := "SELECT * FROM {{.tableName}} WHERE id = @id"
|
||||
tmplSelectFilterCombined := "SELECT * FROM {{.tableName}} WHERE {{.columnFilter}} = @name"
|
||||
return tmplSelectCombined, tmplSelectFilterCombined
|
||||
}
|
||||
|
||||
// GetMysqlParamToolInfo returns statements and param for my-param-tool mysql-sql kind
|
||||
func GetMysqlParamToolInfo(tableName string) (string, string, string, []any) {
|
||||
create_statement := fmt.Sprintf("CREATE TABLE %s (id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, name VARCHAR(255));", tableName)
|
||||
insert_statement := fmt.Sprintf("INSERT INTO %s (name) VALUES (?), (?), (?);", tableName)
|
||||
tool_statement := fmt.Sprintf("SELECT * FROM %s WHERE id = ? OR name = ?;", tableName)
|
||||
// GetMySQLParamToolInfo returns statements and param for my-param-tool mysql-sql kind
|
||||
func GetMySQLParamToolInfo(tableName string) (string, string, string, []any) {
|
||||
createStatement := fmt.Sprintf("CREATE TABLE %s (id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, name VARCHAR(255));", tableName)
|
||||
insertStatement := fmt.Sprintf("INSERT INTO %s (name) VALUES (?), (?), (?);", tableName)
|
||||
toolStatement := fmt.Sprintf("SELECT * FROM %s WHERE id = ? OR name = ?;", tableName)
|
||||
params := []any{"Alice", "Jane", "Sid"}
|
||||
return create_statement, insert_statement, tool_statement, params
|
||||
return createStatement, insertStatement, toolStatement, params
|
||||
}
|
||||
|
||||
// GetMysqlAuthToolInfo returns statements and param of my-auth-tool for mysql-sql kind
|
||||
func GetMysqlAuthToolInfo(tableName string) (string, string, string, []any) {
|
||||
create_statement := fmt.Sprintf("CREATE TABLE %s (id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, name VARCHAR(255), email VARCHAR(255));", tableName)
|
||||
insert_statement := fmt.Sprintf("INSERT INTO %s (name, email) VALUES (?, ?), (?, ?)", tableName)
|
||||
tool_statement := fmt.Sprintf("SELECT name FROM %s WHERE email = ?;", tableName)
|
||||
params := []any{"Alice", SERVICE_ACCOUNT_EMAIL, "Jane", "janedoe@gmail.com"}
|
||||
return create_statement, insert_statement, tool_statement, params
|
||||
// GetMySQLAuthToolInfo returns statements and param of my-auth-tool for mysql-sql kind
|
||||
func GetMySQLAuthToolInfo(tableName string) (string, string, string, []any) {
|
||||
createStatement := fmt.Sprintf("CREATE TABLE %s (id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, name VARCHAR(255), email VARCHAR(255));", tableName)
|
||||
insertStatement := fmt.Sprintf("INSERT INTO %s (name, email) VALUES (?, ?), (?, ?)", tableName)
|
||||
toolStatement := fmt.Sprintf("SELECT name FROM %s WHERE email = ?;", tableName)
|
||||
params := []any{"Alice", ServiceAccountEmail, "Jane", "janedoe@gmail.com"}
|
||||
return createStatement, insertStatement, toolStatement, params
|
||||
}
|
||||
|
||||
// GetMysqlTmplToolStatement returns statements and param for template parameter test cases for mysql-sql kind
|
||||
func GetMysqlTmplToolStatement() (string, string) {
|
||||
// GetMySQLTmplToolStatement returns statements and param for template parameter test cases for mysql-sql kind
|
||||
func GetMySQLTmplToolStatement() (string, string) {
|
||||
tmplSelectCombined := "SELECT * FROM {{.tableName}} WHERE id = ?"
|
||||
tmplSelectFilterCombined := "SELECT * FROM {{.tableName}} WHERE {{.columnFilter}} = ?"
|
||||
return tmplSelectCombined, tmplSelectFilterCombined
|
||||
@@ -349,16 +349,16 @@ func GetPostgresWants() (string, string, string) {
|
||||
return select1Want, failInvocationWant, createTableStatement
|
||||
}
|
||||
|
||||
// GetMssqlWants return the expected wants for mssql
|
||||
func GetMssqlWants() (string, string, string) {
|
||||
// GetMSSQLWants return the expected wants for mssql
|
||||
func GetMSSQLWants() (string, string, string) {
|
||||
select1Want := "[{\"\":1}]"
|
||||
failInvocationWant := `{"jsonrpc":"2.0","id":"invoke-fail-tool","result":{"content":[{"type":"text","text":"unable to execute query: mssql: Could not find stored procedure 'SELEC'."}],"isError":true}}`
|
||||
createTableStatement := `"CREATE TABLE t (id INT IDENTITY(1,1) PRIMARY KEY, name NVARCHAR(MAX))"`
|
||||
return select1Want, failInvocationWant, createTableStatement
|
||||
}
|
||||
|
||||
// GetMysqlWants return the expected wants for mysql
|
||||
func GetMysqlWants() (string, string, string) {
|
||||
// GetMySQLWants return the expected wants for mysql
|
||||
func GetMySQLWants() (string, string, string) {
|
||||
select1Want := "[{\"1\":1}]"
|
||||
failInvocationWant := `{"jsonrpc":"2.0","id":"invoke-fail-tool","result":{"content":[{"type":"text","text":"unable to execute query: Error 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'SELEC 1' at line 1"}],"isError":true}}`
|
||||
createTableStatement := `"CREATE TABLE t (id SERIAL PRIMARY KEY, name TEXT)"`
|
||||
@@ -367,20 +367,20 @@ func GetMysqlWants() (string, string, string) {
|
||||
|
||||
// SetupPostgresSQLTable creates and inserts data into a table of tool
|
||||
// compatible with postgres-sql tool
|
||||
func SetupPostgresSQLTable(t *testing.T, ctx context.Context, pool *pgxpool.Pool, create_statement, insert_statement, tableName string, params []any) func(*testing.T) {
|
||||
func SetupPostgresSQLTable(t *testing.T, ctx context.Context, pool *pgxpool.Pool, createStatement, insertStatement, tableName string, params []any) func(*testing.T) {
|
||||
err := pool.Ping(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to connect to test database: %s", err)
|
||||
}
|
||||
|
||||
// Create table
|
||||
_, err = pool.Query(ctx, create_statement)
|
||||
_, err = pool.Query(ctx, createStatement)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to create test table %s: %s", tableName, err)
|
||||
}
|
||||
|
||||
// Insert test data
|
||||
_, err = pool.Query(ctx, insert_statement, params...)
|
||||
_, err = pool.Query(ctx, insertStatement, params...)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to insert test data: %s", err)
|
||||
}
|
||||
@@ -396,20 +396,20 @@ func SetupPostgresSQLTable(t *testing.T, ctx context.Context, pool *pgxpool.Pool
|
||||
|
||||
// SetupMsSQLTable creates and inserts data into a table of tool
|
||||
// compatible with mssql-sql tool
|
||||
func SetupMsSQLTable(t *testing.T, ctx context.Context, pool *sql.DB, create_statement, insert_statement, tableName string, params []any) func(*testing.T) {
|
||||
func SetupMsSQLTable(t *testing.T, ctx context.Context, pool *sql.DB, createStatement, insertStatement, tableName string, params []any) func(*testing.T) {
|
||||
err := pool.PingContext(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to connect to test database: %s", err)
|
||||
}
|
||||
|
||||
// Create table
|
||||
_, err = pool.QueryContext(ctx, create_statement)
|
||||
_, err = pool.QueryContext(ctx, createStatement)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to create test table %s: %s", tableName, err)
|
||||
}
|
||||
|
||||
// Insert test data
|
||||
_, err = pool.QueryContext(ctx, insert_statement, params...)
|
||||
_, err = pool.QueryContext(ctx, insertStatement, params...)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to insert test data: %s", err)
|
||||
}
|
||||
@@ -425,20 +425,20 @@ func SetupMsSQLTable(t *testing.T, ctx context.Context, pool *sql.DB, create_sta
|
||||
|
||||
// SetupMySQLTable creates and inserts data into a table of tool
|
||||
// compatible with mysql-sql tool
|
||||
func SetupMySQLTable(t *testing.T, ctx context.Context, pool *sql.DB, create_statement, insert_statement, tableName string, params []any) func(*testing.T) {
|
||||
func SetupMySQLTable(t *testing.T, ctx context.Context, pool *sql.DB, createStatement, insertStatement, tableName string, params []any) func(*testing.T) {
|
||||
err := pool.PingContext(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to connect to test database: %s", err)
|
||||
}
|
||||
|
||||
// Create table
|
||||
_, err = pool.QueryContext(ctx, create_statement)
|
||||
_, err = pool.QueryContext(ctx, createStatement)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to create test table %s: %s", tableName, err)
|
||||
}
|
||||
|
||||
// Insert test data
|
||||
_, err = pool.QueryContext(ctx, insert_statement, params...)
|
||||
_, err = pool.QueryContext(ctx, insertStatement, params...)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to insert test data: %s", err)
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tests
|
||||
package couchbase
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
|
||||
"github.com/couchbase/gocb/v2"
|
||||
"github.com/google/uuid"
|
||||
"github.com/googleapis/genai-toolbox/internal/testutils"
|
||||
"github.com/googleapis/genai-toolbox/tests"
|
||||
)
|
||||
|
||||
@@ -34,12 +35,11 @@ const (
|
||||
)
|
||||
|
||||
var (
|
||||
couchbaseConnection = os.Getenv("COUCHBASE_CONNECTION")
|
||||
couchbaseBucket = os.Getenv("COUCHBASE_BUCKET")
|
||||
couchbaseScope = os.Getenv("COUCHBASE_SCOPE")
|
||||
couchbaseUser = os.Getenv("COUCHBASE_USER")
|
||||
couchbasePass = os.Getenv("COUCHBASE_PASS")
|
||||
SERVICE_ACCOUNT_EMAIL = os.Getenv("SERVICE_ACCOUNT_EMAIL")
|
||||
couchbaseConnection = os.Getenv("COUCHBASE_CONNECTION")
|
||||
couchbaseBucket = os.Getenv("COUCHBASE_BUCKET")
|
||||
couchbaseScope = os.Getenv("COUCHBASE_SCOPE")
|
||||
couchbaseUser = os.Getenv("COUCHBASE_USER")
|
||||
couchbasePass = os.Getenv("COUCHBASE_PASS")
|
||||
)
|
||||
|
||||
// getCouchbaseVars validates and returns Couchbase configuration variables
|
||||
@@ -129,7 +129,7 @@ func TestCouchbaseToolEndpoints(t *testing.T) {
|
||||
|
||||
waitCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := cmd.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`))
|
||||
out, err := testutils.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`), cmd.Out)
|
||||
if err != nil {
|
||||
t.Logf("toolbox command logs: \n%s", out)
|
||||
t.Fatalf("toolbox didn't start successfully: %s", err)
|
||||
@@ -250,7 +250,7 @@ func getCouchbaseAuthToolInfo(collectionName string) (string, []map[string]any)
|
||||
toolStatement := fmt.Sprintf("SELECT name FROM %s WHERE email = $email", collectionName)
|
||||
|
||||
params := []map[string]any{
|
||||
{"name": "Alice", "email": SERVICE_ACCOUNT_EMAIL},
|
||||
{"name": "Alice", "email": tests.ServiceAccountEmail},
|
||||
{"name": "Jane", "email": "janedoe@gmail.com"},
|
||||
}
|
||||
return toolStatement, params
|
||||
|
||||
@@ -26,24 +26,24 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/googleapis/genai-toolbox/internal/testutils"
|
||||
"github.com/googleapis/genai-toolbox/tests"
|
||||
)
|
||||
|
||||
var (
|
||||
DGRAPH_SOURCE_KIND = "dgraph"
|
||||
DGRAPH_TOOL_KIND = "dgraph-dql"
|
||||
DGRAPH_API_KEY = "api-key"
|
||||
DGRAPH_URL = os.Getenv("DGRAPH_URL")
|
||||
DgraphSourceKind = "dgraph"
|
||||
DgraphApiKey = "api-key"
|
||||
DgraphUrl = os.Getenv("DGRAPH_URL")
|
||||
)
|
||||
|
||||
func getDgraphVars(t *testing.T) map[string]any {
|
||||
if DGRAPH_URL == "" {
|
||||
if DgraphUrl == "" {
|
||||
t.Fatal("'DGRAPH_URL' not set")
|
||||
}
|
||||
return map[string]any{
|
||||
"kind": DGRAPH_SOURCE_KIND,
|
||||
"dgraphUrl": DGRAPH_URL,
|
||||
"apiKey": DGRAPH_API_KEY,
|
||||
"kind": DgraphSourceKind,
|
||||
"dgraphUrl": DgraphUrl,
|
||||
"apiKey": DgraphApiKey,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -79,7 +79,7 @@ func TestDgraphToolEndpoints(t *testing.T) {
|
||||
|
||||
waitCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := cmd.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`))
|
||||
out, err := testutils.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`), cmd.Out)
|
||||
if err != nil {
|
||||
t.Logf("toolbox command logs: \n%s", out)
|
||||
t.Fatalf("toolbox didn't start successfully: %s", err)
|
||||
|
||||
@@ -28,13 +28,14 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/googleapis/genai-toolbox/internal/testutils"
|
||||
"github.com/googleapis/genai-toolbox/internal/tools"
|
||||
"github.com/googleapis/genai-toolbox/tests"
|
||||
)
|
||||
|
||||
var (
|
||||
HTTP_SOURCE_KIND = "http"
|
||||
HTTP_TOOL_KIND = "http"
|
||||
HttpSourceKind = "http"
|
||||
HttpToolKind = "http"
|
||||
)
|
||||
|
||||
func getHTTPSourceConfig(t *testing.T) map[string]any {
|
||||
@@ -44,7 +45,7 @@ func getHTTPSourceConfig(t *testing.T) map[string]any {
|
||||
}
|
||||
idToken = "Bearer " + idToken
|
||||
return map[string]any{
|
||||
"kind": HTTP_SOURCE_KIND,
|
||||
"kind": HttpSourceKind,
|
||||
"headers": map[string]string{"Authorization": idToken},
|
||||
}
|
||||
}
|
||||
@@ -245,7 +246,7 @@ func TestHttpToolEndpoints(t *testing.T) {
|
||||
|
||||
var args []string
|
||||
|
||||
toolsFile := getHTTPToolsConfig(sourceConfig, HTTP_TOOL_KIND)
|
||||
toolsFile := getHTTPToolsConfig(sourceConfig, HttpToolKind)
|
||||
cmd, cleanup, err := tests.StartCmd(ctx, toolsFile, args...)
|
||||
if err != nil {
|
||||
t.Fatalf("command initialization returned an error: %s", err)
|
||||
@@ -254,7 +255,7 @@ func TestHttpToolEndpoints(t *testing.T) {
|
||||
|
||||
waitCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := cmd.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`))
|
||||
out, err := testutils.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`), cmd.Out)
|
||||
if err != nil {
|
||||
t.Logf("toolbox command logs: \n%s", out)
|
||||
t.Fatalf("toolbox didn't start successfully: %s", err)
|
||||
|
||||
@@ -26,45 +26,46 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/googleapis/genai-toolbox/internal/testutils"
|
||||
"github.com/googleapis/genai-toolbox/tests"
|
||||
)
|
||||
|
||||
var (
|
||||
MSSQL_SOURCE_KIND = "mssql"
|
||||
MSSQL_TOOL_KIND = "mssql-sql"
|
||||
MSSQL_DATABASE = os.Getenv("MSSQL_DATABASE")
|
||||
MSSQL_HOST = os.Getenv("MSSQL_HOST")
|
||||
MSSQL_PORT = os.Getenv("MSSQL_PORT")
|
||||
MSSQL_USER = os.Getenv("MSSQL_USER")
|
||||
MSSQL_PASS = os.Getenv("MSSQL_PASS")
|
||||
MSSQLSourceKind = "mssql"
|
||||
MSSQLToolKind = "mssql-sql"
|
||||
MSSQLDatabase = os.Getenv("MSSQL_DATABASE")
|
||||
MSSQLHost = os.Getenv("MSSQL_HOST")
|
||||
MSSQLPort = os.Getenv("MSSQL_PORT")
|
||||
MSSQLUser = os.Getenv("MSSQL_USER")
|
||||
MSSQLPass = os.Getenv("MSSQL_PASS")
|
||||
)
|
||||
|
||||
func getMsSQLVars(t *testing.T) map[string]any {
|
||||
switch "" {
|
||||
case MSSQL_DATABASE:
|
||||
case MSSQLDatabase:
|
||||
t.Fatal("'MSSQL_DATABASE' not set")
|
||||
case MSSQL_HOST:
|
||||
case MSSQLHost:
|
||||
t.Fatal("'MSSQL_HOST' not set")
|
||||
case MSSQL_PORT:
|
||||
case MSSQLPort:
|
||||
t.Fatal("'MSSQL_PORT' not set")
|
||||
case MSSQL_USER:
|
||||
case MSSQLUser:
|
||||
t.Fatal("'MSSQL_USER' not set")
|
||||
case MSSQL_PASS:
|
||||
case MSSQLPass:
|
||||
t.Fatal("'MSSQL_PASS' not set")
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"kind": MSSQL_SOURCE_KIND,
|
||||
"host": MSSQL_HOST,
|
||||
"port": MSSQL_PORT,
|
||||
"database": MSSQL_DATABASE,
|
||||
"user": MSSQL_USER,
|
||||
"password": MSSQL_PASS,
|
||||
"kind": MSSQLSourceKind,
|
||||
"host": MSSQLHost,
|
||||
"port": MSSQLPort,
|
||||
"database": MSSQLDatabase,
|
||||
"user": MSSQLUser,
|
||||
"password": MSSQLPass,
|
||||
}
|
||||
}
|
||||
|
||||
// Copied over from mssql.go
|
||||
func initMssqlConnection(host, port, user, pass, dbname string) (*sql.DB, error) {
|
||||
func initMSSQLConnection(host, port, user, pass, dbname string) (*sql.DB, error) {
|
||||
// Create dsn
|
||||
query := url.Values{}
|
||||
query.Add("database", dbname)
|
||||
@@ -83,14 +84,14 @@ func initMssqlConnection(host, port, user, pass, dbname string) (*sql.DB, error)
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func TestMssqlToolEndpoints(t *testing.T) {
|
||||
func TestMSSQLToolEndpoints(t *testing.T) {
|
||||
sourceConfig := getMsSQLVars(t)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||
defer cancel()
|
||||
|
||||
var args []string
|
||||
|
||||
pool, err := initMssqlConnection(MSSQL_HOST, MSSQL_PORT, MSSQL_USER, MSSQL_PASS, MSSQL_DATABASE)
|
||||
pool, err := initMSSQLConnection(MSSQLHost, MSSQLPort, MSSQLUser, MSSQLPass, MSSQLDatabase)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to create SQL Server connection pool: %s", err)
|
||||
}
|
||||
@@ -101,20 +102,20 @@ func TestMssqlToolEndpoints(t *testing.T) {
|
||||
tableNameTemplateParam := "template_param_table_" + strings.ReplaceAll(uuid.New().String(), "-", "")
|
||||
|
||||
// set up data for param tool
|
||||
create_statement1, insert_statement1, tool_statement1, params1 := tests.GetMssqlParamToolInfo(tableNameParam)
|
||||
teardownTable1 := tests.SetupMsSQLTable(t, ctx, pool, create_statement1, insert_statement1, tableNameParam, params1)
|
||||
createStatement1, insertStatement1, toolStatement1, params1 := tests.GetMSSQLParamToolInfo(tableNameParam)
|
||||
teardownTable1 := tests.SetupMsSQLTable(t, ctx, pool, createStatement1, insertStatement1, tableNameParam, params1)
|
||||
defer teardownTable1(t)
|
||||
|
||||
// set up data for auth tool
|
||||
create_statement2, insert_statement2, tool_statement2, params2 := tests.GetMssqlAuthToolInfo(tableNameAuth)
|
||||
teardownTable2 := tests.SetupMsSQLTable(t, ctx, pool, create_statement2, insert_statement2, tableNameAuth, params2)
|
||||
createStatement2, insertStatement2, toolStatement2, params2 := tests.GetMSSQLAuthToolInfo(tableNameAuth)
|
||||
teardownTable2 := tests.SetupMsSQLTable(t, ctx, pool, createStatement2, insertStatement2, tableNameAuth, params2)
|
||||
defer teardownTable2(t)
|
||||
|
||||
// Write config into a file and pass it to command
|
||||
toolsFile := tests.GetToolsConfig(sourceConfig, MSSQL_TOOL_KIND, tool_statement1, tool_statement2)
|
||||
toolsFile = tests.AddMssqlExecuteSqlConfig(t, toolsFile)
|
||||
tmplSelectCombined, tmplSelectFilterCombined := tests.GetMssqlTmplToolStatement()
|
||||
toolsFile = tests.AddTemplateParamConfig(t, toolsFile, MSSQL_TOOL_KIND, tmplSelectCombined, tmplSelectFilterCombined, "")
|
||||
toolsFile := tests.GetToolsConfig(sourceConfig, MSSQLToolKind, toolStatement1, toolStatement2)
|
||||
toolsFile = tests.AddMSSQLExecuteSqlConfig(t, toolsFile)
|
||||
tmplSelectCombined, tmplSelectFilterCombined := tests.GetMSSQLTmplToolStatement()
|
||||
toolsFile = tests.AddTemplateParamConfig(t, toolsFile, MSSQLToolKind, tmplSelectCombined, tmplSelectFilterCombined, "")
|
||||
|
||||
cmd, cleanup, err := tests.StartCmd(ctx, toolsFile, args...)
|
||||
if err != nil {
|
||||
@@ -124,7 +125,7 @@ func TestMssqlToolEndpoints(t *testing.T) {
|
||||
|
||||
waitCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := cmd.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`))
|
||||
out, err := testutils.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`), cmd.Out)
|
||||
if err != nil {
|
||||
t.Logf("toolbox command logs: \n%s", out)
|
||||
t.Fatalf("toolbox didn't start successfully: %s", err)
|
||||
@@ -132,7 +133,7 @@ func TestMssqlToolEndpoints(t *testing.T) {
|
||||
|
||||
tests.RunToolGetTest(t)
|
||||
|
||||
select1Want, failInvocationWant, createTableStatement := tests.GetMssqlWants()
|
||||
select1Want, failInvocationWant, createTableStatement := tests.GetMSSQLWants()
|
||||
invokeParamWant, mcpInvokeParamWant := tests.GetNonSpannerInvokeParamWant()
|
||||
tests.RunToolInvokeTest(t, select1Want, invokeParamWant)
|
||||
tests.RunExecuteSqlToolInvokeTest(t, createTableStatement, select1Want)
|
||||
|
||||
@@ -25,40 +25,41 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/googleapis/genai-toolbox/internal/testutils"
|
||||
"github.com/googleapis/genai-toolbox/tests"
|
||||
)
|
||||
|
||||
var (
|
||||
MYSQL_SOURCE_KIND = "mysql"
|
||||
MYSQL_TOOL_KIND = "mysql-sql"
|
||||
MYSQL_DATABASE = os.Getenv("MYSQL_DATABASE")
|
||||
MYSQL_HOST = os.Getenv("MYSQL_HOST")
|
||||
MYSQL_PORT = os.Getenv("MYSQL_PORT")
|
||||
MYSQL_USER = os.Getenv("MYSQL_USER")
|
||||
MYSQL_PASS = os.Getenv("MYSQL_PASS")
|
||||
MySQLSourceKind = "mysql"
|
||||
MySQLToolKind = "mysql-sql"
|
||||
MySQLDatabase = os.Getenv("MYSQL_DATABASE")
|
||||
MySQLHost = os.Getenv("MYSQL_HOST")
|
||||
MySQLPort = os.Getenv("MYSQL_PORT")
|
||||
MySQLUser = os.Getenv("MYSQL_USER")
|
||||
MySQLPass = os.Getenv("MYSQL_PASS")
|
||||
)
|
||||
|
||||
func getMySQLVars(t *testing.T) map[string]any {
|
||||
switch "" {
|
||||
case MYSQL_DATABASE:
|
||||
case MySQLDatabase:
|
||||
t.Fatal("'MYSQL_DATABASE' not set")
|
||||
case MYSQL_HOST:
|
||||
case MySQLHost:
|
||||
t.Fatal("'MYSQL_HOST' not set")
|
||||
case MYSQL_PORT:
|
||||
case MySQLPort:
|
||||
t.Fatal("'MYSQL_PORT' not set")
|
||||
case MYSQL_USER:
|
||||
case MySQLUser:
|
||||
t.Fatal("'MYSQL_USER' not set")
|
||||
case MYSQL_PASS:
|
||||
case MySQLPass:
|
||||
t.Fatal("'MYSQL_PASS' not set")
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"kind": MYSQL_SOURCE_KIND,
|
||||
"host": MYSQL_HOST,
|
||||
"port": MYSQL_PORT,
|
||||
"database": MYSQL_DATABASE,
|
||||
"user": MYSQL_USER,
|
||||
"password": MYSQL_PASS,
|
||||
"kind": MySQLSourceKind,
|
||||
"host": MySQLHost,
|
||||
"port": MySQLPort,
|
||||
"database": MySQLDatabase,
|
||||
"user": MySQLUser,
|
||||
"password": MySQLPass,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,14 +75,14 @@ func initMySQLConnectionPool(host, port, user, pass, dbname string) (*sql.DB, er
|
||||
return pool, nil
|
||||
}
|
||||
|
||||
func TestMysqlToolEndpoints(t *testing.T) {
|
||||
func TestMySQLToolEndpoints(t *testing.T) {
|
||||
sourceConfig := getMySQLVars(t)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||
defer cancel()
|
||||
|
||||
var args []string
|
||||
|
||||
pool, err := initMySQLConnectionPool(MYSQL_HOST, MYSQL_PORT, MYSQL_USER, MYSQL_PASS, MYSQL_DATABASE)
|
||||
pool, err := initMySQLConnectionPool(MySQLHost, MySQLPort, MySQLUser, MySQLPass, MySQLDatabase)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to create MySQL connection pool: %s", err)
|
||||
}
|
||||
@@ -92,20 +93,20 @@ func TestMysqlToolEndpoints(t *testing.T) {
|
||||
tableNameTemplateParam := "template_param_table_" + strings.ReplaceAll(uuid.New().String(), "-", "")
|
||||
|
||||
// set up data for param tool
|
||||
create_statement1, insert_statement1, tool_statement1, params1 := tests.GetMysqlParamToolInfo(tableNameParam)
|
||||
teardownTable1 := tests.SetupMySQLTable(t, ctx, pool, create_statement1, insert_statement1, tableNameParam, params1)
|
||||
createStatement1, insertStatement1, toolStatement1, params1 := tests.GetMySQLParamToolInfo(tableNameParam)
|
||||
teardownTable1 := tests.SetupMySQLTable(t, ctx, pool, createStatement1, insertStatement1, tableNameParam, params1)
|
||||
defer teardownTable1(t)
|
||||
|
||||
// set up data for auth tool
|
||||
create_statement2, insert_statement2, tool_statement2, params2 := tests.GetMysqlAuthToolInfo(tableNameAuth)
|
||||
teardownTable2 := tests.SetupMySQLTable(t, ctx, pool, create_statement2, insert_statement2, tableNameAuth, params2)
|
||||
createStatement2, insertStatement2, toolStatement2, params2 := tests.GetMySQLAuthToolInfo(tableNameAuth)
|
||||
teardownTable2 := tests.SetupMySQLTable(t, ctx, pool, createStatement2, insertStatement2, tableNameAuth, params2)
|
||||
defer teardownTable2(t)
|
||||
|
||||
// Write config into a file and pass it to command
|
||||
toolsFile := tests.GetToolsConfig(sourceConfig, MYSQL_TOOL_KIND, tool_statement1, tool_statement2)
|
||||
toolsFile := tests.GetToolsConfig(sourceConfig, MySQLToolKind, toolStatement1, toolStatement2)
|
||||
toolsFile = tests.AddMySqlExecuteSqlConfig(t, toolsFile)
|
||||
tmplSelectCombined, tmplSelectFilterCombined := tests.GetMysqlTmplToolStatement()
|
||||
toolsFile = tests.AddTemplateParamConfig(t, toolsFile, MYSQL_TOOL_KIND, tmplSelectCombined, tmplSelectFilterCombined, "")
|
||||
tmplSelectCombined, tmplSelectFilterCombined := tests.GetMySQLTmplToolStatement()
|
||||
toolsFile = tests.AddTemplateParamConfig(t, toolsFile, MySQLToolKind, tmplSelectCombined, tmplSelectFilterCombined, "")
|
||||
|
||||
cmd, cleanup, err := tests.StartCmd(ctx, toolsFile, args...)
|
||||
if err != nil {
|
||||
@@ -115,7 +116,7 @@ func TestMysqlToolEndpoints(t *testing.T) {
|
||||
|
||||
waitCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := cmd.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`))
|
||||
out, err := testutils.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`), cmd.Out)
|
||||
if err != nil {
|
||||
t.Logf("toolbox command logs: \n%s", out)
|
||||
t.Fatalf("toolbox didn't start successfully: %s", err)
|
||||
@@ -123,7 +124,7 @@ func TestMysqlToolEndpoints(t *testing.T) {
|
||||
|
||||
tests.RunToolGetTest(t)
|
||||
|
||||
select1Want, failInvocationWant, createTableStatement := tests.GetMysqlWants()
|
||||
select1Want, failInvocationWant, createTableStatement := tests.GetMySQLWants()
|
||||
invokeParamWant, mcpInvokeParamWant := tests.GetNonSpannerInvokeParamWant()
|
||||
tests.RunToolInvokeTest(t, select1Want, invokeParamWant)
|
||||
tests.RunExecuteSqlToolInvokeTest(t, createTableStatement, select1Want)
|
||||
|
||||
@@ -26,36 +26,36 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/googleapis/genai-toolbox/internal/testutils"
|
||||
"github.com/googleapis/genai-toolbox/tests"
|
||||
)
|
||||
|
||||
var (
|
||||
NEO4J_SOURCE_KIND = "neo4j"
|
||||
NEO4J_TOOL_KIND = "neo4j-cypher"
|
||||
NEO4J_DATABASE = os.Getenv("NEO4J_DATABASE")
|
||||
NEO4J_URI = os.Getenv("NEO4J_URI")
|
||||
NEO4J_USER = os.Getenv("NEO4J_USER")
|
||||
NEO4J_PASS = os.Getenv("NEO4J_PASS")
|
||||
Neo4jSourceKind = "neo4j"
|
||||
Neo4jDatabase = os.Getenv("NEO4J_DATABASE")
|
||||
Neo4jUri = os.Getenv("NEO4J_URI")
|
||||
Neo4jUser = os.Getenv("NEO4J_USER")
|
||||
Neo4jPass = os.Getenv("NEO4J_PASS")
|
||||
)
|
||||
|
||||
func getNeo4jVars(t *testing.T) map[string]any {
|
||||
switch "" {
|
||||
case NEO4J_DATABASE:
|
||||
case Neo4jDatabase:
|
||||
t.Fatal("'NEO4J_DATABASE' not set")
|
||||
case NEO4J_URI:
|
||||
case Neo4jUri:
|
||||
t.Fatal("'NEO4J_URI' not set")
|
||||
case NEO4J_USER:
|
||||
case Neo4jUser:
|
||||
t.Fatal("'NEO4J_USER' not set")
|
||||
case NEO4J_PASS:
|
||||
case Neo4jPass:
|
||||
t.Fatal("'NEO4J_PASS' not set")
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"kind": NEO4J_SOURCE_KIND,
|
||||
"uri": NEO4J_URI,
|
||||
"database": NEO4J_DATABASE,
|
||||
"user": NEO4J_USER,
|
||||
"password": NEO4J_PASS,
|
||||
"kind": Neo4jSourceKind,
|
||||
"uri": Neo4jUri,
|
||||
"database": Neo4jDatabase,
|
||||
"user": Neo4jUser,
|
||||
"password": Neo4jPass,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -88,7 +88,7 @@ func TestNeo4jToolEndpoints(t *testing.T) {
|
||||
|
||||
waitCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := cmd.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`))
|
||||
out, err := testutils.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`), cmd.Out)
|
||||
if err != nil {
|
||||
t.Logf("toolbox command logs: \n%s", out)
|
||||
t.Fatalf("toolbox didn't start successfully: %s", err)
|
||||
|
||||
@@ -25,41 +25,42 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/googleapis/genai-toolbox/internal/testutils"
|
||||
"github.com/googleapis/genai-toolbox/tests"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
var (
|
||||
POSTGRES_SOURCE_KIND = "postgres"
|
||||
POSTGRES_TOOL_KIND = "postgres-sql"
|
||||
POSTGRES_DATABASE = os.Getenv("POSTGRES_DATABASE")
|
||||
POSTGRES_HOST = os.Getenv("POSTGRES_HOST")
|
||||
POSTGRES_PORT = os.Getenv("POSTGRES_PORT")
|
||||
POSTGRES_USER = os.Getenv("POSTGRES_USER")
|
||||
POSTGRES_PASS = os.Getenv("POSTGRES_PASS")
|
||||
PostgresSourceKind = "postgres"
|
||||
PostgresToolKind = "postgres-sql"
|
||||
PostgresDatabase = os.Getenv("POSTGRES_DATABASE")
|
||||
PostgresHost = os.Getenv("POSTGRES_HOST")
|
||||
PostgresPort = os.Getenv("POSTGRES_PORT")
|
||||
PostgresUser = os.Getenv("POSTGRES_USER")
|
||||
PostgresPass = os.Getenv("POSTGRES_PASS")
|
||||
)
|
||||
|
||||
func getPostgresVars(t *testing.T) map[string]any {
|
||||
switch "" {
|
||||
case POSTGRES_DATABASE:
|
||||
case PostgresDatabase:
|
||||
t.Fatal("'POSTGRES_DATABASE' not set")
|
||||
case POSTGRES_HOST:
|
||||
case PostgresHost:
|
||||
t.Fatal("'POSTGRES_HOST' not set")
|
||||
case POSTGRES_PORT:
|
||||
case PostgresPort:
|
||||
t.Fatal("'POSTGRES_PORT' not set")
|
||||
case POSTGRES_USER:
|
||||
case PostgresUser:
|
||||
t.Fatal("'POSTGRES_USER' not set")
|
||||
case POSTGRES_PASS:
|
||||
case PostgresPass:
|
||||
t.Fatal("'POSTGRES_PASS' not set")
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"kind": POSTGRES_SOURCE_KIND,
|
||||
"host": POSTGRES_HOST,
|
||||
"port": POSTGRES_PORT,
|
||||
"database": POSTGRES_DATABASE,
|
||||
"user": POSTGRES_USER,
|
||||
"password": POSTGRES_PASS,
|
||||
"kind": PostgresSourceKind,
|
||||
"host": PostgresHost,
|
||||
"port": PostgresPort,
|
||||
"database": PostgresDatabase,
|
||||
"user": PostgresUser,
|
||||
"password": PostgresPass,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -87,7 +88,7 @@ func TestPostgres(t *testing.T) {
|
||||
|
||||
var args []string
|
||||
|
||||
pool, err := initPostgresConnectionPool(POSTGRES_HOST, POSTGRES_PORT, POSTGRES_USER, POSTGRES_PASS, POSTGRES_DATABASE)
|
||||
pool, err := initPostgresConnectionPool(PostgresHost, PostgresPort, PostgresUser, PostgresPass, PostgresDatabase)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to create postgres connection pool: %s", err)
|
||||
}
|
||||
@@ -98,20 +99,20 @@ func TestPostgres(t *testing.T) {
|
||||
tableNameTemplateParam := "template_param_table_" + strings.ReplaceAll(uuid.New().String(), "-", "")
|
||||
|
||||
// set up data for param tool
|
||||
create_statement1, insert_statement1, tool_statement1, params1 := tests.GetPostgresSQLParamToolInfo(tableNameParam)
|
||||
teardownTable1 := tests.SetupPostgresSQLTable(t, ctx, pool, create_statement1, insert_statement1, tableNameParam, params1)
|
||||
createStatement1, insertStatement1, toolStatement1, params1 := tests.GetPostgresSQLParamToolInfo(tableNameParam)
|
||||
teardownTable1 := tests.SetupPostgresSQLTable(t, ctx, pool, createStatement1, insertStatement1, tableNameParam, params1)
|
||||
defer teardownTable1(t)
|
||||
|
||||
// set up data for auth tool
|
||||
create_statement2, insert_statement2, tool_statement2, params2 := tests.GetPostgresSQLAuthToolInfo(tableNameAuth)
|
||||
teardownTable2 := tests.SetupPostgresSQLTable(t, ctx, pool, create_statement2, insert_statement2, tableNameAuth, params2)
|
||||
createStatement2, insertStatement2, toolStatement2, params2 := tests.GetPostgresSQLAuthToolInfo(tableNameAuth)
|
||||
teardownTable2 := tests.SetupPostgresSQLTable(t, ctx, pool, createStatement2, insertStatement2, tableNameAuth, params2)
|
||||
defer teardownTable2(t)
|
||||
|
||||
// Write config into a file and pass it to command
|
||||
toolsFile := tests.GetToolsConfig(sourceConfig, POSTGRES_TOOL_KIND, tool_statement1, tool_statement2)
|
||||
toolsFile := tests.GetToolsConfig(sourceConfig, PostgresToolKind, toolStatement1, toolStatement2)
|
||||
toolsFile = tests.AddPgExecuteSqlConfig(t, toolsFile)
|
||||
tmplSelectCombined, tmplSelectFilterCombined := tests.GetPostgresSQLTmplToolStatement()
|
||||
toolsFile = tests.AddTemplateParamConfig(t, toolsFile, POSTGRES_TOOL_KIND, tmplSelectCombined, tmplSelectFilterCombined, "")
|
||||
toolsFile = tests.AddTemplateParamConfig(t, toolsFile, PostgresToolKind, tmplSelectCombined, tmplSelectFilterCombined, "")
|
||||
|
||||
cmd, cleanup, err := tests.StartCmd(ctx, toolsFile, args...)
|
||||
if err != nil {
|
||||
@@ -121,7 +122,7 @@ func TestPostgres(t *testing.T) {
|
||||
|
||||
waitCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := cmd.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`))
|
||||
out, err := testutils.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`), cmd.Out)
|
||||
if err != nil {
|
||||
t.Logf("toolbox command logs: \n%s", out)
|
||||
t.Fatalf("toolbox didn't start successfully: %s", err)
|
||||
|
||||
@@ -22,28 +22,29 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/googleapis/genai-toolbox/internal/testutils"
|
||||
"github.com/googleapis/genai-toolbox/tests"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
var (
|
||||
REDIS_SOURCE_KIND = "redis"
|
||||
REDIS_TOOL_KIND = "redis"
|
||||
REDIS_ADDRESS = os.Getenv("REDIS_ADDRESS")
|
||||
REDIS_PASS = os.Getenv("REDIS_PASS")
|
||||
RedisSourceKind = "redis"
|
||||
RedisToolKind = "redis"
|
||||
RedisAddress = os.Getenv("REDIS_ADDRESS")
|
||||
RedisPass = os.Getenv("REDIS_PASS")
|
||||
)
|
||||
|
||||
func getRedisVars(t *testing.T) map[string]any {
|
||||
switch "" {
|
||||
case REDIS_ADDRESS:
|
||||
case RedisAddress:
|
||||
t.Fatal("'REDIS_ADDRESS' not set")
|
||||
case REDIS_PASS:
|
||||
case RedisPass:
|
||||
t.Fatal("'REDIS_PASS' not set")
|
||||
}
|
||||
return map[string]any{
|
||||
"kind": REDIS_SOURCE_KIND,
|
||||
"address": []string{REDIS_ADDRESS},
|
||||
"password": REDIS_PASS,
|
||||
"kind": RedisSourceKind,
|
||||
"address": []string{RedisAddress},
|
||||
"password": RedisPass,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -70,7 +71,7 @@ func TestRedisToolEndpoints(t *testing.T) {
|
||||
|
||||
var args []string
|
||||
|
||||
client, err := initRedisClient(ctx, REDIS_ADDRESS, REDIS_PASS)
|
||||
client, err := initRedisClient(ctx, RedisAddress, RedisPass)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to create Redis connection: %s", err)
|
||||
}
|
||||
@@ -80,7 +81,7 @@ func TestRedisToolEndpoints(t *testing.T) {
|
||||
defer teardownDB(t)
|
||||
|
||||
// Write config into a file and pass it to command
|
||||
toolsFile := tests.GetRedisValkeyToolsConfig(sourceConfig, REDIS_TOOL_KIND)
|
||||
toolsFile := tests.GetRedisValkeyToolsConfig(sourceConfig, RedisToolKind)
|
||||
|
||||
cmd, cleanup, err := tests.StartCmd(ctx, toolsFile, args...)
|
||||
if err != nil {
|
||||
@@ -90,7 +91,7 @@ func TestRedisToolEndpoints(t *testing.T) {
|
||||
|
||||
waitCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := cmd.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`))
|
||||
out, err := testutils.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`), cmd.Out)
|
||||
if err != nil {
|
||||
t.Logf("toolbox command logs: \n%s", out)
|
||||
t.Fatalf("toolbox didn't start successfully: %s", err)
|
||||
@@ -109,7 +110,7 @@ func setupRedisDB(t *testing.T, ctx context.Context, client *redis.Client) func(
|
||||
{"HSET", keys[0], "id", 1, "name", "Alice"},
|
||||
{"HSET", keys[1], "id", 2, "name", "Jane"},
|
||||
{"HSET", keys[2], "id", 3, "name", "Sid"},
|
||||
{"HSET", tests.SERVICE_ACCOUNT_EMAIL, "name", "Alice"},
|
||||
{"HSET", tests.ServiceAccountEmail, "name", "Alice"},
|
||||
}
|
||||
for _, c := range commands {
|
||||
resp := client.Do(ctx, c...)
|
||||
|
||||
@@ -15,13 +15,10 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
yaml "github.com/goccy/go-yaml"
|
||||
|
||||
@@ -133,63 +130,3 @@ func (c *CmdExec) Close() {
|
||||
c.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// WaitForString waits until the server logs a single line that matches the provided regex.
|
||||
// returns the output of whatever the server sent so far.
|
||||
func (c *CmdExec) WaitForString(ctx context.Context, re *regexp.Regexp) (string, error) {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
in := bufio.NewReader(c.Out)
|
||||
|
||||
// read lines in background, sending result of each read over a channel
|
||||
// this allows us to use in.ReadString without blocking
|
||||
type result struct {
|
||||
s string
|
||||
err error
|
||||
}
|
||||
output := make(chan result)
|
||||
go func() {
|
||||
defer close(output)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// if the context is canceled, the orig thread will send back the error
|
||||
// so we can just exit the goroutine here
|
||||
return
|
||||
default:
|
||||
// otherwise read a line from the output
|
||||
s, err := in.ReadString('\n')
|
||||
if err != nil {
|
||||
output <- result{err: err}
|
||||
return
|
||||
}
|
||||
output <- result{s: s}
|
||||
// if that last string matched, exit the goroutine
|
||||
if re.MatchString(s) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// collect the output until the ctx is canceled, an error was hit,
|
||||
// or match was found (which is indicated the channel is closed)
|
||||
var sb strings.Builder
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// if ctx is done, return that error
|
||||
return sb.String(), ctx.Err()
|
||||
case o, ok := <-output:
|
||||
if !ok {
|
||||
// match was found!
|
||||
return sb.String(), nil
|
||||
}
|
||||
if o.err != nil {
|
||||
// error was found!
|
||||
return sb.String(), o.err
|
||||
}
|
||||
sb.WriteString(o.s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
"time"
|
||||
|
||||
"cloud.google.com/go/cloudsqlconn"
|
||||
"github.com/googleapis/genai-toolbox/internal/testutils"
|
||||
)
|
||||
|
||||
// RunSourceConnection test for source connection
|
||||
@@ -57,7 +58,7 @@ func RunSourceConnectionTest(t *testing.T, sourceConfig map[string]any, toolKind
|
||||
|
||||
waitCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := cmd.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`))
|
||||
out, err := testutils.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`), cmd.Out)
|
||||
if err != nil {
|
||||
t.Logf("toolbox command logs: \n%s", out)
|
||||
return fmt.Errorf("toolbox didn't start successfully: %s", err)
|
||||
|
||||
@@ -31,33 +31,34 @@ import (
|
||||
database "cloud.google.com/go/spanner/admin/database/apiv1"
|
||||
"cloud.google.com/go/spanner/admin/database/apiv1/databasepb"
|
||||
"github.com/google/uuid"
|
||||
"github.com/googleapis/genai-toolbox/internal/testutils"
|
||||
"github.com/googleapis/genai-toolbox/internal/tools"
|
||||
"github.com/googleapis/genai-toolbox/tests"
|
||||
)
|
||||
|
||||
var (
|
||||
SPANNER_SOURCE_KIND = "spanner"
|
||||
SPANNER_TOOL_KIND = "spanner-sql"
|
||||
SPANNER_PROJECT = os.Getenv("SPANNER_PROJECT")
|
||||
SPANNER_DATABASE = os.Getenv("SPANNER_DATABASE")
|
||||
SPANNER_INSTANCE = os.Getenv("SPANNER_INSTANCE")
|
||||
SpannerSourceKind = "spanner"
|
||||
SpannerToolKind = "spanner-sql"
|
||||
SpannerProject = os.Getenv("SPANNER_PROJECT")
|
||||
SpannerDatabase = os.Getenv("SPANNER_DATABASE")
|
||||
SpannerInstance = os.Getenv("SPANNER_INSTANCE")
|
||||
)
|
||||
|
||||
func getSpannerVars(t *testing.T) map[string]any {
|
||||
switch "" {
|
||||
case SPANNER_PROJECT:
|
||||
case SpannerProject:
|
||||
t.Fatal("'SPANNER_PROJECT' not set")
|
||||
case SPANNER_DATABASE:
|
||||
case SpannerDatabase:
|
||||
t.Fatal("'SPANNER_DATABASE' not set")
|
||||
case SPANNER_INSTANCE:
|
||||
case SpannerInstance:
|
||||
t.Fatal("'SPANNER_INSTANCE' not set")
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"kind": SPANNER_SOURCE_KIND,
|
||||
"project": SPANNER_PROJECT,
|
||||
"instance": SPANNER_INSTANCE,
|
||||
"database": SPANNER_DATABASE,
|
||||
"kind": SpannerSourceKind,
|
||||
"project": SpannerProject,
|
||||
"instance": SpannerInstance,
|
||||
"database": SpannerDatabase,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -96,7 +97,7 @@ func TestSpannerToolEndpoints(t *testing.T) {
|
||||
var args []string
|
||||
|
||||
// Create Spanner client
|
||||
dataClient, adminClient, err := initSpannerClients(ctx, SPANNER_PROJECT, SPANNER_INSTANCE, SPANNER_DATABASE)
|
||||
dataClient, adminClient, err := initSpannerClients(ctx, SpannerProject, SpannerInstance, SpannerDatabase)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to create Spanner client: %s", err)
|
||||
}
|
||||
@@ -107,19 +108,19 @@ func TestSpannerToolEndpoints(t *testing.T) {
|
||||
tableNameTemplateParam := "template_param_table_" + strings.ReplaceAll(uuid.New().String(), "-", "")
|
||||
|
||||
// set up data for param tool
|
||||
create_statement1, insert_statement1, tool_statement1, params1 := getSpannerParamToolInfo(tableNameParam)
|
||||
createStatement1, insertStatement1, toolStatement1, params1 := getSpannerParamToolInfo(tableNameParam)
|
||||
dbString := fmt.Sprintf(
|
||||
"projects/%s/instances/%s/databases/%s",
|
||||
SPANNER_PROJECT,
|
||||
SPANNER_INSTANCE,
|
||||
SPANNER_DATABASE,
|
||||
SpannerProject,
|
||||
SpannerInstance,
|
||||
SpannerDatabase,
|
||||
)
|
||||
teardownTable1 := setupSpannerTable(t, ctx, adminClient, dataClient, create_statement1, insert_statement1, tableNameParam, dbString, params1)
|
||||
teardownTable1 := setupSpannerTable(t, ctx, adminClient, dataClient, createStatement1, insertStatement1, tableNameParam, dbString, params1)
|
||||
defer teardownTable1(t)
|
||||
|
||||
// set up data for auth tool
|
||||
create_statement2, insert_statement2, tool_statement2, params2 := getSpannerAuthToolInfo(tableNameAuth)
|
||||
teardownTable2 := setupSpannerTable(t, ctx, adminClient, dataClient, create_statement2, insert_statement2, tableNameAuth, dbString, params2)
|
||||
createStatement2, insertStatement2, toolStatement2, params2 := getSpannerAuthToolInfo(tableNameAuth)
|
||||
teardownTable2 := setupSpannerTable(t, ctx, adminClient, dataClient, createStatement2, insertStatement2, tableNameAuth, dbString, params2)
|
||||
defer teardownTable2(t)
|
||||
|
||||
// set up data for template param tool
|
||||
@@ -128,7 +129,7 @@ func TestSpannerToolEndpoints(t *testing.T) {
|
||||
defer teardownTableTmpl(t)
|
||||
|
||||
// Write config into a file and pass it to command
|
||||
toolsFile := tests.GetToolsConfig(sourceConfig, SPANNER_TOOL_KIND, tool_statement1, tool_statement2)
|
||||
toolsFile := tests.GetToolsConfig(sourceConfig, SpannerToolKind, toolStatement1, toolStatement2)
|
||||
toolsFile = addSpannerExecuteSqlConfig(t, toolsFile)
|
||||
toolsFile = addSpannerReadOnlyConfig(t, toolsFile)
|
||||
toolsFile = addTemplateParamConfig(t, toolsFile)
|
||||
@@ -141,7 +142,7 @@ func TestSpannerToolEndpoints(t *testing.T) {
|
||||
|
||||
waitCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := cmd.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`))
|
||||
out, err := testutils.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`), cmd.Out)
|
||||
if err != nil {
|
||||
t.Logf("toolbox command logs: \n%s", out)
|
||||
t.Fatalf("toolbox didn't start successfully: %s", err)
|
||||
@@ -170,25 +171,25 @@ func TestSpannerToolEndpoints(t *testing.T) {
|
||||
|
||||
// getSpannerToolInfo returns statements and param for my-param-tool for spanner-sql kind
|
||||
func getSpannerParamToolInfo(tableName string) (string, string, string, map[string]any) {
|
||||
create_statement := fmt.Sprintf("CREATE TABLE %s (id INT64, name STRING(MAX)) PRIMARY KEY (id)", tableName)
|
||||
insert_statement := fmt.Sprintf("INSERT INTO %s (id, name) VALUES (1, @name1), (2, @name2), (3, @name3)", tableName)
|
||||
tool_statement := fmt.Sprintf("SELECT * FROM %s WHERE id = @id OR name = @name", tableName)
|
||||
createStatement := fmt.Sprintf("CREATE TABLE %s (id INT64, name STRING(MAX)) PRIMARY KEY (id)", tableName)
|
||||
insertStatement := fmt.Sprintf("INSERT INTO %s (id, name) VALUES (1, @name1), (2, @name2), (3, @name3)", tableName)
|
||||
toolStatement := fmt.Sprintf("SELECT * FROM %s WHERE id = @id OR name = @name", tableName)
|
||||
params := map[string]any{"name1": "Alice", "name2": "Jane", "name3": "Sid"}
|
||||
return create_statement, insert_statement, tool_statement, params
|
||||
return createStatement, insertStatement, toolStatement, params
|
||||
}
|
||||
|
||||
// getSpannerAuthToolInfo returns statements and param of my-auth-tool for spanner-sql kind
|
||||
func getSpannerAuthToolInfo(tableName string) (string, string, string, map[string]any) {
|
||||
create_statement := fmt.Sprintf("CREATE TABLE %s (id INT64, name STRING(MAX), email STRING(MAX)) PRIMARY KEY (id)", tableName)
|
||||
insert_statement := fmt.Sprintf("INSERT INTO %s (id, name, email) VALUES (1, @name1, @email1), (2, @name2, @email2)", tableName)
|
||||
tool_statement := fmt.Sprintf("SELECT name FROM %s WHERE email = @email", tableName)
|
||||
createStatement := fmt.Sprintf("CREATE TABLE %s (id INT64, name STRING(MAX), email STRING(MAX)) PRIMARY KEY (id)", tableName)
|
||||
insertStatement := fmt.Sprintf("INSERT INTO %s (id, name, email) VALUES (1, @name1, @email1), (2, @name2, @email2)", tableName)
|
||||
toolStatement := fmt.Sprintf("SELECT name FROM %s WHERE email = @email", tableName)
|
||||
params := map[string]any{
|
||||
"name1": "Alice",
|
||||
"email1": tests.SERVICE_ACCOUNT_EMAIL,
|
||||
"email1": tests.ServiceAccountEmail,
|
||||
"name2": "Jane",
|
||||
"email2": "janedoe@gmail.com",
|
||||
}
|
||||
return create_statement, insert_statement, tool_statement, params
|
||||
return createStatement, insertStatement, toolStatement, params
|
||||
}
|
||||
|
||||
// setupSpannerTable creates and inserts data into a table of tool
|
||||
@@ -352,7 +353,7 @@ func addTemplateParamConfig(t *testing.T, config map[string]any) map[string]any
|
||||
return config
|
||||
}
|
||||
|
||||
func runSpannerExecuteSqlToolInvokeTest(t *testing.T, select_1_want, invokeParamWant, tableNameParam, tableNameAuth string) {
|
||||
func runSpannerExecuteSqlToolInvokeTest(t *testing.T, select1Want, invokeParamWant, tableNameParam, tableNameAuth string) {
|
||||
// Get ID token
|
||||
idToken, err := tests.GetGoogleIdToken(tests.ClientId)
|
||||
if err != nil {
|
||||
@@ -373,7 +374,7 @@ func runSpannerExecuteSqlToolInvokeTest(t *testing.T, select_1_want, invokeParam
|
||||
api: "http://127.0.0.1:5000/api/tool/my-exec-sql-tool-read-only/invoke",
|
||||
requestHeader: map[string]string{},
|
||||
requestBody: bytes.NewBuffer([]byte(`{"sql":"SELECT 1"}`)),
|
||||
want: select_1_want,
|
||||
want: select1Want,
|
||||
isErr: false,
|
||||
},
|
||||
{
|
||||
@@ -417,7 +418,7 @@ func runSpannerExecuteSqlToolInvokeTest(t *testing.T, select_1_want, invokeParam
|
||||
api: "http://127.0.0.1:5000/api/tool/my-exec-sql-tool/invoke",
|
||||
requestHeader: map[string]string{},
|
||||
requestBody: bytes.NewBuffer([]byte(`{"sql":"SELECT 1"}`)),
|
||||
want: select_1_want,
|
||||
want: select1Want,
|
||||
isErr: false,
|
||||
},
|
||||
{
|
||||
@@ -455,7 +456,7 @@ func runSpannerExecuteSqlToolInvokeTest(t *testing.T, select_1_want, invokeParam
|
||||
requestHeader: map[string]string{"my-google-auth_token": idToken},
|
||||
requestBody: bytes.NewBuffer([]byte(`{"sql":"SELECT 1"}`)),
|
||||
isErr: false,
|
||||
want: select_1_want,
|
||||
want: select1Want,
|
||||
},
|
||||
{
|
||||
name: "Invoke my-auth-exec-sql-tool with invalid auth token",
|
||||
|
||||
@@ -25,19 +25,20 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/googleapis/genai-toolbox/internal/testutils"
|
||||
"github.com/googleapis/genai-toolbox/tests"
|
||||
)
|
||||
|
||||
var (
|
||||
SQLITE_SOURCE_KIND = "sqlite"
|
||||
SQLITE_TOOL_KIND = "sqlite-sql"
|
||||
SQLITE_DATABASE = os.Getenv("SQLITE_DATABASE")
|
||||
SQLiteSourceKind = "sqlite"
|
||||
SQLiteToolKind = "sqlite-sql"
|
||||
SQLiteDatabase = os.Getenv("SQLITE_DATABASE")
|
||||
)
|
||||
|
||||
func getSQLiteVars(t *testing.T) map[string]any {
|
||||
return map[string]any{
|
||||
"kind": SQLITE_SOURCE_KIND,
|
||||
"database": SQLITE_DATABASE,
|
||||
"kind": SQLiteSourceKind,
|
||||
"database": SQLiteDatabase,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,19 +82,19 @@ func setupSQLiteTestDB(t *testing.T, ctx context.Context, db *sql.DB, createStat
|
||||
}
|
||||
|
||||
func getSQLiteParamToolInfo(tableName string) (string, string, string, []any) {
|
||||
create_statement := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (id INTEGER PRIMARY KEY, name TEXT);", tableName)
|
||||
insert_statement := fmt.Sprintf("INSERT INTO %s (name) VALUES (?), (?), (?);", tableName)
|
||||
tool_statement := fmt.Sprintf("SELECT * FROM %s WHERE id = ? OR name = ?;", tableName)
|
||||
createStatement := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (id INTEGER PRIMARY KEY, name TEXT);", tableName)
|
||||
insertStatement := fmt.Sprintf("INSERT INTO %s (name) VALUES (?), (?), (?);", tableName)
|
||||
toolStatement := fmt.Sprintf("SELECT * FROM %s WHERE id = ? OR name = ?;", tableName)
|
||||
params := []any{"Alice", "Jane", "Sid"}
|
||||
return create_statement, insert_statement, tool_statement, params
|
||||
return createStatement, insertStatement, toolStatement, params
|
||||
}
|
||||
|
||||
func getSQLiteAuthToolInfo(tableName string) (string, string, string, []any) {
|
||||
create_statement := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (id INTEGER PRIMARY KEY, name TEXT NOT NULL, email TEXT)", tableName)
|
||||
insert_statement := fmt.Sprintf("INSERT INTO %s (name, email) VALUES (?, ?), (?,?) RETURNING id, name, email;", tableName)
|
||||
tool_statement := fmt.Sprintf("SELECT name FROM %s WHERE email = ?;", tableName)
|
||||
params := []any{"Alice", tests.SERVICE_ACCOUNT_EMAIL, "Jane", "janedoe@gmail.com"}
|
||||
return create_statement, insert_statement, tool_statement, params
|
||||
createStatement := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (id INTEGER PRIMARY KEY, name TEXT NOT NULL, email TEXT)", tableName)
|
||||
insertStatement := fmt.Sprintf("INSERT INTO %s (name, email) VALUES (?, ?), (?,?) RETURNING id, name, email;", tableName)
|
||||
toolStatement := fmt.Sprintf("SELECT name FROM %s WHERE email = ?;", tableName)
|
||||
params := []any{"Alice", tests.ServiceAccountEmail, "Jane", "janedoe@gmail.com"}
|
||||
return createStatement, insertStatement, toolStatement, params
|
||||
}
|
||||
|
||||
func getSQLiteTmplToolStatement() (string, string) {
|
||||
@@ -103,7 +104,7 @@ func getSQLiteTmplToolStatement() (string, string) {
|
||||
}
|
||||
|
||||
func TestSQLiteToolEndpoint(t *testing.T) {
|
||||
db, teardownDb, sqliteDb, err := initSQLiteDb(t, SQLITE_DATABASE)
|
||||
db, teardownDb, sqliteDb, err := initSQLiteDb(t, SQLiteDatabase)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -123,17 +124,17 @@ func TestSQLiteToolEndpoint(t *testing.T) {
|
||||
tableNameTemplateParam := "template_param_table_" + strings.ReplaceAll(uuid.New().String(), "-", "")
|
||||
|
||||
// set up data for param tool
|
||||
create_statement1, insert_statement1, tool_statement1, params1 := getSQLiteParamToolInfo(tableNameParam)
|
||||
setupSQLiteTestDB(t, ctx, db, create_statement1, insert_statement1, tableNameParam, params1)
|
||||
createStatement1, insertStatement1, toolStatement1, params1 := getSQLiteParamToolInfo(tableNameParam)
|
||||
setupSQLiteTestDB(t, ctx, db, createStatement1, insertStatement1, tableNameParam, params1)
|
||||
|
||||
// set up data for auth tool
|
||||
create_statement2, insert_statement2, tool_statement2, params2 := getSQLiteAuthToolInfo(tableNameAuth)
|
||||
setupSQLiteTestDB(t, ctx, db, create_statement2, insert_statement2, tableNameAuth, params2)
|
||||
createStatement2, insertStatement2, toolStatement2, params2 := getSQLiteAuthToolInfo(tableNameAuth)
|
||||
setupSQLiteTestDB(t, ctx, db, createStatement2, insertStatement2, tableNameAuth, params2)
|
||||
|
||||
// Write config into a file and pass it to command
|
||||
toolsFile := tests.GetToolsConfig(sourceConfig, SQLITE_TOOL_KIND, tool_statement1, tool_statement2)
|
||||
toolsFile := tests.GetToolsConfig(sourceConfig, SQLiteToolKind, toolStatement1, toolStatement2)
|
||||
tmplSelectCombined, tmplSelectFilterCombined := getSQLiteTmplToolStatement()
|
||||
toolsFile = tests.AddTemplateParamConfig(t, toolsFile, SQLITE_TOOL_KIND, tmplSelectCombined, tmplSelectFilterCombined, "")
|
||||
toolsFile = tests.AddTemplateParamConfig(t, toolsFile, SQLiteToolKind, tmplSelectCombined, tmplSelectFilterCombined, "")
|
||||
|
||||
cmd, cleanup, err := tests.StartCmd(ctx, toolsFile, args...)
|
||||
if err != nil {
|
||||
@@ -143,7 +144,7 @@ func TestSQLiteToolEndpoint(t *testing.T) {
|
||||
|
||||
waitCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := cmd.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`))
|
||||
out, err := testutils.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`), cmd.Out)
|
||||
if err != nil {
|
||||
t.Logf("toolbox command logs: \n%s", out)
|
||||
t.Fatalf("toolbox didn't start successfully: %s", err)
|
||||
|
||||
@@ -109,15 +109,15 @@ func RunToolInvokeTest(t *testing.T, select1Want, invokeParamWant string) {
|
||||
isErr: false,
|
||||
},
|
||||
{
|
||||
name: "Invoke my-tool without parameters",
|
||||
api: "http://127.0.0.1:5000/api/tool/my-tool/invoke",
|
||||
name: "Invoke my-param-tool without parameters",
|
||||
api: "http://127.0.0.1:5000/api/tool/my-param-tool/invoke",
|
||||
requestHeader: map[string]string{},
|
||||
requestBody: bytes.NewBuffer([]byte(`{}`)),
|
||||
isErr: true,
|
||||
},
|
||||
{
|
||||
name: "Invoke my-tool with insufficient parameters",
|
||||
api: "http://127.0.0.1:5000/api/tool/my-tool/invoke",
|
||||
name: "Invoke my-param-tool with insufficient parameters",
|
||||
api: "http://127.0.0.1:5000/api/tool/my-param-tool/invoke",
|
||||
requestHeader: map[string]string{},
|
||||
requestBody: bytes.NewBuffer([]byte(`{"id": 1}`)),
|
||||
isErr: true,
|
||||
@@ -428,7 +428,7 @@ func RunToolInvokeWithTemplateParameters(t *testing.T, tableName string, config
|
||||
}
|
||||
}
|
||||
|
||||
func RunExecuteSqlToolInvokeTest(t *testing.T, createTableStatement string, select_1_want string) {
|
||||
func RunExecuteSqlToolInvokeTest(t *testing.T, createTableStatement string, select1Want string) {
|
||||
// Get ID token
|
||||
idToken, err := GetGoogleIdToken(ClientId)
|
||||
if err != nil {
|
||||
@@ -449,7 +449,7 @@ func RunExecuteSqlToolInvokeTest(t *testing.T, createTableStatement string, sele
|
||||
api: "http://127.0.0.1:5000/api/tool/my-exec-sql-tool/invoke",
|
||||
requestHeader: map[string]string{},
|
||||
requestBody: bytes.NewBuffer([]byte(`{"sql":"SELECT 1"}`)),
|
||||
want: select_1_want,
|
||||
want: select1Want,
|
||||
isErr: false,
|
||||
},
|
||||
{
|
||||
@@ -489,7 +489,7 @@ func RunExecuteSqlToolInvokeTest(t *testing.T, createTableStatement string, sele
|
||||
requestHeader: map[string]string{"my-google-auth_token": idToken},
|
||||
requestBody: bytes.NewBuffer([]byte(`{"sql":"SELECT 1"}`)),
|
||||
isErr: false,
|
||||
want: select_1_want,
|
||||
want: select1Want,
|
||||
},
|
||||
{
|
||||
name: "Invoke my-auth-exec-sql-tool with invalid auth token",
|
||||
@@ -597,7 +597,7 @@ func RunInitialize(t *testing.T, protocolVersion string) string {
|
||||
}
|
||||
|
||||
// RunMCPToolCallMethod runs the tool/call for mcp endpoint
|
||||
func RunMCPToolCallMethod(t *testing.T, invokeParamWant, fail_invocation_want string) {
|
||||
func RunMCPToolCallMethod(t *testing.T, invokeParamWant, failInvocationWant string) {
|
||||
sessionId := RunInitialize(t, "2024-11-05")
|
||||
header := map[string]string{}
|
||||
if sessionId != "" {
|
||||
@@ -715,7 +715,7 @@ func RunMCPToolCallMethod(t *testing.T, invokeParamWant, fail_invocation_want st
|
||||
"arguments": map[string]any{"id": 1},
|
||||
},
|
||||
},
|
||||
want: fail_invocation_want,
|
||||
want: failInvocationWant,
|
||||
},
|
||||
}
|
||||
for _, tc := range invokeTcs {
|
||||
|
||||
@@ -22,24 +22,25 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/googleapis/genai-toolbox/internal/testutils"
|
||||
"github.com/googleapis/genai-toolbox/tests"
|
||||
"github.com/valkey-io/valkey-go"
|
||||
)
|
||||
|
||||
var (
|
||||
VALKEY_SOURCE_KIND = "valkey"
|
||||
VALKEY_TOOL_KIND = "valkey"
|
||||
VALKEY_ADDRESS = os.Getenv("VALKEY_ADDRESS")
|
||||
ValkeySourceKind = "valkey"
|
||||
ValkeyToolKind = "valkey"
|
||||
ValkeyAddress = os.Getenv("VALKEY_ADDRESS")
|
||||
)
|
||||
|
||||
func getValkeyVars(t *testing.T) map[string]any {
|
||||
switch "" {
|
||||
case VALKEY_ADDRESS:
|
||||
case ValkeyAddress:
|
||||
t.Fatal("'VALKEY_ADDRESS' not set")
|
||||
}
|
||||
return map[string]any{
|
||||
"kind": VALKEY_SOURCE_KIND,
|
||||
"address": []string{VALKEY_ADDRESS},
|
||||
"kind": ValkeySourceKind,
|
||||
"address": []string{ValkeyAddress},
|
||||
"disableCache": true,
|
||||
}
|
||||
}
|
||||
@@ -73,7 +74,7 @@ func TestValkeyToolEndpoints(t *testing.T) {
|
||||
|
||||
var args []string
|
||||
|
||||
client, err := initValkeyClient(ctx, []string{VALKEY_ADDRESS})
|
||||
client, err := initValkeyClient(ctx, []string{ValkeyAddress})
|
||||
if err != nil {
|
||||
t.Fatalf("unable to create Valkey connection: %s", err)
|
||||
}
|
||||
@@ -83,7 +84,7 @@ func TestValkeyToolEndpoints(t *testing.T) {
|
||||
defer teardownDB(t)
|
||||
|
||||
// Write config into a file and pass it to command
|
||||
toolsFile := tests.GetRedisValkeyToolsConfig(sourceConfig, VALKEY_TOOL_KIND)
|
||||
toolsFile := tests.GetRedisValkeyToolsConfig(sourceConfig, ValkeyToolKind)
|
||||
|
||||
cmd, cleanup, err := tests.StartCmd(ctx, toolsFile, args...)
|
||||
if err != nil {
|
||||
@@ -93,7 +94,7 @@ func TestValkeyToolEndpoints(t *testing.T) {
|
||||
|
||||
waitCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := cmd.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`))
|
||||
out, err := testutils.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`), cmd.Out)
|
||||
if err != nil {
|
||||
t.Logf("toolbox command logs: \n%s", out)
|
||||
t.Fatalf("toolbox didn't start successfully: %s", err)
|
||||
@@ -112,7 +113,7 @@ func setupValkeyDB(t *testing.T, ctx context.Context, client valkey.Client) func
|
||||
{"HSET", keys[0], "name", "Alice", "id", "1"},
|
||||
{"HSET", keys[1], "name", "Jane", "id", "2"},
|
||||
{"HSET", keys[2], "name", "Sid", "id", "3"},
|
||||
{"HSET", tests.SERVICE_ACCOUNT_EMAIL, "name", "Alice"},
|
||||
{"HSET", tests.ServiceAccountEmail, "name", "Alice"},
|
||||
}
|
||||
builtCmds := make(valkey.Commands, len(commands))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user