Compare commits

..

3 Commits

Author SHA1 Message Date
Disha Prakash
2ffa026c9b add documentation 2026-01-13 11:38:13 +00:00
dishaprakash
344a093a60 Merge branch 'main' into multi_prebuilt 2026-01-13 10:04:08 +00:00
Disha Prakash
d833d1992d feat: Support combining multiple prebuilt configurations 2026-01-12 21:00:26 +00:00
7 changed files with 70 additions and 202 deletions

View File

@@ -313,15 +313,15 @@ func Execute() {
type Command struct {
*cobra.Command
cfg server.ServerConfig
logger log.Logger
tools_file string
tools_files []string
tools_folder string
prebuiltConfig string
inStream io.Reader
outStream io.Writer
errStream io.Writer
cfg server.ServerConfig
logger log.Logger
tools_file string
tools_files []string
tools_folder string
prebuiltConfigs []string
inStream io.Reader
outStream io.Writer
errStream io.Writer
}
// NewCommand returns a Command object representing an invocation of the CLI.
@@ -374,10 +374,10 @@ func NewCommand(opts ...Option) *Command {
flags.StringVar(&cmd.cfg.TelemetryServiceName, "telemetry-service-name", "toolbox", "Sets the value of the service.name resource attribute for telemetry data.")
// Fetch prebuilt tools sources to customize the help description
prebuiltHelp := fmt.Sprintf(
"Use a prebuilt tool configuration by source type. Allowed: '%s'.",
"Use a prebuilt tool configuration by source type. Allowed: '%s'. Can be specified multiple times.",
strings.Join(prebuiltconfigs.GetPrebuiltSources(), "', '"),
)
flags.StringVar(&cmd.prebuiltConfig, "prebuilt", "", prebuiltHelp)
flags.StringSliceVar(&cmd.prebuiltConfigs, "prebuilt", []string{}, prebuiltHelp)
flags.BoolVar(&cmd.cfg.Stdio, "stdio", false, "Listens via MCP STDIO instead of acting as a remote HTTP server.")
flags.BoolVar(&cmd.cfg.DisableReload, "disable-reload", false, "Disables dynamic reloading of tools file.")
flags.BoolVar(&cmd.cfg.UI, "ui", false, "Launches the Toolbox UI web server.")
@@ -864,24 +864,32 @@ func run(cmd *Command) error {
var allToolsFiles []ToolsFile
// Load Prebuilt Configuration
if cmd.prebuiltConfig != "" {
buf, err := prebuiltconfigs.Get(cmd.prebuiltConfig)
if err != nil {
cmd.logger.ErrorContext(ctx, err.Error())
return err
}
logMsg := fmt.Sprint("Using prebuilt tool configuration for ", cmd.prebuiltConfig)
cmd.logger.InfoContext(ctx, logMsg)
// Append prebuilt.source to Version string for the User Agent
cmd.cfg.Version += "+prebuilt." + cmd.prebuiltConfig
parsed, err := parseToolsFile(ctx, buf)
if err != nil {
errMsg := fmt.Errorf("unable to parse prebuilt tool configuration: %w", err)
cmd.logger.ErrorContext(ctx, errMsg.Error())
return errMsg
if len(cmd.prebuiltConfigs) > 0 {
slices.Sort(cmd.prebuiltConfigs)
sourcesList := strings.Join(cmd.prebuiltConfigs, ", ")
logMsg := fmt.Sprintf("Using prebuilt tool configurations for: %s", sourcesList)
cmd.logger.InfoContext(ctx, logMsg)
for _, configName := range cmd.prebuiltConfigs {
buf, err := prebuiltconfigs.Get(configName)
if err != nil {
cmd.logger.ErrorContext(ctx, err.Error())
return err
}
// Update version string
cmd.cfg.Version += "+prebuilt." + configName
// Parse into ToolsFile struct
parsed, err := parseToolsFile(ctx, buf)
if err != nil {
errMsg := fmt.Errorf("unable to parse prebuilt tool configuration for '%s': %w", configName, err)
cmd.logger.ErrorContext(ctx, errMsg.Error())
return errMsg
}
allToolsFiles = append(allToolsFiles, parsed)
}
allToolsFiles = append(allToolsFiles, parsed)
}
// Determine if Custom Files should be loaded
@@ -889,7 +897,7 @@ func run(cmd *Command) error {
isCustomConfigured := cmd.tools_file != "" || len(cmd.tools_files) > 0 || cmd.tools_folder != ""
// Determine if default 'tools.yaml' should be used (No prebuilt AND No custom flags)
useDefaultToolsFile := cmd.prebuiltConfig == "" && !isCustomConfigured
useDefaultToolsFile := len(cmd.prebuiltConfigs) == 0 && !isCustomConfigured
if useDefaultToolsFile {
cmd.tools_file = "tools.yaml"

View File

@@ -420,17 +420,27 @@ func TestPrebuiltFlag(t *testing.T) {
tcs := []struct {
desc string
args []string
want string
want []string
}{
{
desc: "default value",
args: []string{},
want: "",
want: []string{},
},
{
desc: "custom pre built flag",
args: []string{"--tools-file", "alloydb"},
want: "alloydb",
desc: "single prebuilt flag",
args: []string{"--prebuilt", "alloydb"},
want: []string{"alloydb"},
},
{
desc: "multiple prebuilt flags",
args: []string{"--prebuilt", "alloydb", "--prebuilt", "bigquery"},
want: []string{"alloydb", "bigquery"},
},
{
desc: "comma separated prebuilt flags",
args: []string{"--prebuilt", "alloydb,bigquery"},
want: []string{"alloydb", "bigquery"},
},
}
for _, tc := range tcs {
@@ -439,8 +449,8 @@ func TestPrebuiltFlag(t *testing.T) {
if err != nil {
t.Fatalf("unexpected error invoking command: %s", err)
}
if c.tools_file != tc.want {
t.Fatalf("got %v, want %v", c.cfg, tc.want)
if diff := cmp.Diff(c.prebuiltConfigs, tc.want); diff != "" {
t.Fatalf("got %v, want %v, diff %s", c.prebuiltConfigs, tc.want, diff)
}
})
}
@@ -2063,6 +2073,12 @@ authSources:
return nil
},
},
{
desc: "sqlite called twice error",
args: []string{"--prebuilt", "sqlite", "--prebuilt", "sqlite"},
wantErr: true,
errString: "resource conflicts detected",
},
{
desc: "tool conflict error",
args: []string{"--prebuilt", "sqlite", "--tools-file", toolConflictFile},

View File

@@ -16,7 +16,7 @@ description: >
| | `--log-level` | Specify the minimum level logged. Allowed: 'DEBUG', 'INFO', 'WARN', 'ERROR'. | `info` |
| | `--logging-format` | Specify logging format to use. Allowed: 'standard' or 'JSON'. | `standard` |
| `-p` | `--port` | Port the server will listen on. | `5000` |
| | `--prebuilt` | Use a prebuilt tool configuration by source type. See [Prebuilt Tools Reference](prebuilt-tools.md) for allowed values. | |
| | `--prebuilt` | Use one or more prebuilt tool configuration by source type. See [Prebuilt Tools Reference](prebuilt-tools.md) for allowed values. | |
| | `--stdio` | Listens via MCP STDIO instead of acting as a remote HTTP server. | |
| | `--telemetry-gcp` | Enable exporting directly to Google Cloud Monitoring. | |
| | `--telemetry-otlp` | Enable exporting using OpenTelemetry Protocol (OTLP) to the specified endpoint (e.g. 'http://127.0.0.1:4318') | |
@@ -50,6 +50,11 @@ description: >
# Server with prebuilt + custom tools configurations
./toolbox --tools-file tools.yaml --prebuilt alloydb-postgres
# Server with multiple prebuilt tools configurations
./toolbox --prebuilt alloydb-postgres,alloydb-postgres-admin
# OR
./toolbox --prebuilt alloydb-postgres --prebuilt alloydb-postgres-admin
```
### Tool Configuration Sources
@@ -70,7 +75,7 @@ The CLI supports multiple mutually exclusive ways to specify tool configurations
**Prebuilt Configurations:**
- `--prebuilt`: Use predefined configurations for specific database types (e.g.,
- `--prebuilt`: Use one or more predefined configurations for specific database types (e.g.,
'bigquery', 'postgres', 'spanner'). See [Prebuilt Tools
Reference](prebuilt-tools.md) for allowed values.

View File

@@ -16,6 +16,9 @@ details on how to connect your AI tools (IDEs) to databases via Toolbox and MCP.
{{< notice tip >}}
You can now use `--prebuilt` along `--tools-file`, `--tools-files`, or
`--tools-folder` to combine prebuilt configs with custom tools.
You can also combine multiple prebuilt configs.
See [Usage Examples](../reference/cli.md#examples).
{{< /notice >}}

View File

@@ -16,7 +16,6 @@ package server
import (
"context"
"fmt"
"regexp"
"strings"
yaml "github.com/goccy/go-yaml"
@@ -140,10 +139,6 @@ func (c *SourceConfigs) UnmarshalYAML(ctx context.Context, unmarshal func(interf
}
for name, u := range raw {
err := NameValidation(name)
if err != nil {
return err
}
// Unmarshal to a general type that ensure it capture all fields
var v map[string]any
if err := u.Unmarshal(&v); err != nil {
@@ -188,10 +183,6 @@ func (c *AuthServiceConfigs) UnmarshalYAML(ctx context.Context, unmarshal func(i
}
for name, u := range raw {
err := NameValidation(name)
if err != nil {
return err
}
var v map[string]any
if err := u.Unmarshal(&v); err != nil {
return fmt.Errorf("unable to unmarshal %q: %w", name, err)
@@ -235,10 +226,6 @@ func (c *EmbeddingModelConfigs) UnmarshalYAML(ctx context.Context, unmarshal fun
}
for name, u := range raw {
err := NameValidation(name)
if err != nil {
return err
}
// Unmarshal to a general type that ensure it capture all fields
var v map[string]any
if err := u.Unmarshal(&v); err != nil {
@@ -283,10 +270,6 @@ func (c *ToolConfigs) UnmarshalYAML(ctx context.Context, unmarshal func(interfac
}
for name, u := range raw {
err := NameValidation(name)
if err != nil {
return err
}
var v map[string]any
if err := u.Unmarshal(&v); err != nil {
return fmt.Errorf("unable to unmarshal %q: %w", name, err)
@@ -340,10 +323,6 @@ func (c *ToolsetConfigs) UnmarshalYAML(ctx context.Context, unmarshal func(inter
}
for name, toolList := range raw {
err := NameValidation(name)
if err != nil {
return err
}
(*c)[name] = tools.ToolsetConfig{Name: name, ToolNames: toolList}
}
return nil
@@ -363,10 +342,6 @@ func (c *PromptConfigs) UnmarshalYAML(ctx context.Context, unmarshal func(interf
}
for name, u := range raw {
err := NameValidation(name)
if err != nil {
return err
}
var v map[string]any
if err := u.Unmarshal(&v); err != nil {
return fmt.Errorf("unable to unmarshal prompt %q: %w", name, err)
@@ -414,31 +389,7 @@ func (c *PromptsetConfigs) UnmarshalYAML(ctx context.Context, unmarshal func(int
}
for name, promptList := range raw {
err := NameValidation(name)
if err != nil {
return err
}
(*c)[name] = prompts.PromptsetConfig{Name: name, PromptNames: promptList}
}
return nil
}
// Tools naming validation is added in the MCP v2025-11-25, but we'll be
// implementing it across Toolbox
// Tool names SHOULD be between 1 and 128 characters in length (inclusive).
// Tool names SHOULD be considered case-sensitive.
// The following SHOULD be the only allowed characters: uppercase and lowercase ASCII letters (A-Z, a-z), digits (0-9), underscore (_), hyphen (-), and dot (.)
// Tool names SHOULD NOT contain spaces, commas, or other special characters.
// Tool names SHOULD be unique within a server.
func NameValidation(name string) error {
strLen := len(name)
if strLen < 1 || strLen > 128 {
return fmt.Errorf("resource name SHOULD be between 1 and 128 characters in length (inclusive)")
}
validChars := regexp.MustCompile("^[a-zA-Z0-9_.-]+$")
isValid := validChars.MatchString(name)
if !isValid {
return fmt.Errorf("invalid character for resource name; only uppercase and lowercase ASCII letters (A-Z, a-z), digits (0-9), underscore (_), hyphen (-), and dot (.) is allowed")
}
return nil
}

View File

@@ -200,62 +200,3 @@ func TestUpdateServer(t *testing.T) {
t.Errorf("error updating server, promptset (-want +got):\n%s", diff)
}
}
func TestNameValidation(t *testing.T) {
testCases := []struct {
desc string
resourceName string
errStr string
}{
{
desc: "names with 0 length",
resourceName: "",
errStr: "resource name SHOULD be between 1 and 128 characters in length (inclusive)",
},
{
desc: "names with allowed length",
resourceName: "foo",
},
{
desc: "names with 128 length",
resourceName: strings.Repeat("a", 128),
},
{
desc: "names with more than 128 length",
resourceName: strings.Repeat("a", 129),
errStr: "resource name SHOULD be between 1 and 128 characters in length (inclusive)",
},
{
desc: "names with space",
resourceName: "foo bar",
errStr: "invalid character for resource name; only uppercase and lowercase ASCII letters (A-Z, a-z), digits (0-9), underscore (_), hyphen (-), and dot (.) is allowed",
},
{
desc: "names with commas",
resourceName: "foo,bar",
errStr: "invalid character for resource name; only uppercase and lowercase ASCII letters (A-Z, a-z), digits (0-9), underscore (_), hyphen (-), and dot (.) is allowed",
},
{
desc: "names with other special character",
resourceName: "foo!",
errStr: "invalid character for resource name; only uppercase and lowercase ASCII letters (A-Z, a-z), digits (0-9), underscore (_), hyphen (-), and dot (.) is allowed",
},
{
desc: "names with allowed special character",
resourceName: "foo_.-bar6",
},
}
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
err := server.NameValidation(tc.resourceName)
if err != nil {
if tc.errStr != err.Error() {
t.Fatalf("unexpected error: %s", err)
}
}
if err == nil && tc.errStr != "" {
t.Fatalf("expect error: %s", tc.errStr)
}
})
}
}

View File

@@ -85,66 +85,13 @@ func initDataplexConnection(ctx context.Context) (*dataplex.CatalogClient, error
return client, nil
}
// cleanupOldAspectTypes Deletes AspectTypes older than the specified duration.
func cleanupOldAspectTypes(t *testing.T, ctx context.Context, client *dataplex.CatalogClient, oldThreshold time.Duration) {
parent := fmt.Sprintf("projects/%s/locations/us", DataplexProject)
olderThanTime := time.Now().Add(-oldThreshold)
listReq := &dataplexpb.ListAspectTypesRequest{
Parent: parent,
PageSize: 100, // Fetch up to 100 items
OrderBy: "create_time asc", // Order by creation time
}
const maxDeletes = 8 // Explicitly limit the number of deletions
it := client.ListAspectTypes(ctx, listReq)
var aspectTypesToDelete []string
for len(aspectTypesToDelete) < maxDeletes {
aspectType, err := it.Next()
if err == iterator.Done {
break
}
if err != nil {
t.Logf("Warning: Failed to list aspect types during cleanup: %v", err)
return
}
// Perform time-based filtering in memory
if aspectType.CreateTime != nil {
createTime := aspectType.CreateTime.AsTime()
if createTime.Before(olderThanTime) {
aspectTypesToDelete = append(aspectTypesToDelete, aspectType.GetName())
}
} else {
t.Logf("Warning: AspectType %s has no CreateTime", aspectType.GetName())
}
}
if len(aspectTypesToDelete) == 0 {
t.Logf("cleanupOldAspectTypes: No aspect types found older than %s to delete.", oldThreshold.String())
return
}
for _, aspectTypeName := range aspectTypesToDelete {
deleteReq := &dataplexpb.DeleteAspectTypeRequest{Name: aspectTypeName}
op, err := client.DeleteAspectType(ctx, deleteReq)
if err != nil {
t.Logf("Warning: Failed to delete aspect type %s: %v", aspectTypeName, err)
continue // Skip to the next item if initiation fails
}
if err := op.Wait(ctx); err != nil {
t.Logf("Warning: Failed to delete aspect type %s, operation error: %v", aspectTypeName, err)
} else {
t.Logf("cleanupOldAspectTypes: Successfully deleted %s", aspectTypeName)
}
}
}
func TestDataplexToolEndpoints(t *testing.T) {
sourceConfig := getDataplexVars(t)
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
defer cancel()
var args []string
bigqueryClient, err := initBigQueryConnection(ctx, DataplexProject)
if err != nil {
t.Fatalf("unable to create Cloud SQL connection pool: %s", err)
@@ -155,9 +102,6 @@ func TestDataplexToolEndpoints(t *testing.T) {
t.Fatalf("unable to create Dataplex connection: %s", err)
}
// Cleanup older aspecttypes
cleanupOldAspectTypes(t, ctx, dataplexClient, 1*time.Hour)
// create resources with UUID
datasetName := fmt.Sprintf("temp_toolbox_test_%s", strings.ReplaceAll(uuid.New().String(), "-", ""))
tableName := fmt.Sprintf("param_table_%s", strings.ReplaceAll(uuid.New().String(), "-", ""))