Compare commits

...

2 Commits

Author SHA1 Message Date
Yuan Teoh
1e3cefebe6 chore: update yaml tag in source and auth services 2025-09-09 11:17:00 -07:00
Yuan Teoh
66d5a10790 chore: update unmarshal function for ToolsFile 2025-09-05 17:41:01 -07:00
56 changed files with 433 additions and 382 deletions

View File

@@ -265,7 +265,6 @@ func NewCommand(opts ...Option) *Command {
type ToolsFile struct {
Sources server.SourceConfigs `yaml:"sources"`
AuthSources server.AuthServiceConfigs `yaml:"authSources"` // Deprecated: Kept for compatibility.
AuthServices server.AuthServiceConfigs `yaml:"authServices"`
Tools server.ToolConfigs `yaml:"tools"`
Toolsets server.ToolsetConfigs `yaml:"toolsets"`
@@ -374,8 +373,13 @@ func parseToolsFile(ctx context.Context, raw []byte) (ToolsFile, error) {
}
raw = []byte(output)
raw, err = convertToolsFile(ctx, raw)
if err != nil {
return toolsFile, fmt.Errorf("error converting tools file: %s", err)
}
// Parse contents
err = yaml.UnmarshalContext(ctx, raw, &toolsFile, yaml.Strict())
toolsFile.Sources, toolsFile.AuthServices, toolsFile.Tools, toolsFile.Toolsets, err = server.UnmarshalResourceConfig(ctx, raw)
if err != nil {
return toolsFile, err
}
@@ -405,15 +409,6 @@ func mergeToolsFiles(files ...ToolsFile) (ToolsFile, error) {
}
}
// Check for conflicts and merge authSources (deprecated, but still support)
for name, authSource := range file.AuthSources {
if _, exists := merged.AuthSources[name]; exists {
conflicts = append(conflicts, fmt.Sprintf("authSource '%s' (file #%d)", name, fileIndex+1))
} else {
merged.AuthSources[name] = authSource
}
}
// Check for conflicts and merge authServices
for name, authService := range file.AuthServices {
if _, exists := merged.AuthServices[name]; exists {
@@ -869,11 +864,6 @@ func run(cmd *Command) error {
}
cmd.cfg.SourceConfigs, cmd.cfg.AuthServiceConfigs, cmd.cfg.ToolConfigs, cmd.cfg.ToolsetConfigs = toolsFile.Sources, toolsFile.AuthServices, toolsFile.Tools, toolsFile.Toolsets
authSourceConfigs := toolsFile.AuthSources
if authSourceConfigs != nil {
cmd.logger.WarnContext(ctx, "`authSources` is deprecated, use `authServices` instead")
cmd.cfg.AuthServiceConfigs = authSourceConfigs
}
instrumentation, err := telemetry.CreateTelemetryInstrumentation(versionString)
if err != nil {

View File

@@ -723,6 +723,71 @@ func TestParseToolFile(t *testing.T) {
},
},
},
{
description: "basic example",
in: `
kind: sources
name: my-pg-instance
type: cloud-sql-postgres
project: my-project
region: my-region
instance: my-instance
database: my_db
user: my_user
password: my_pass
---
kind: tools
name: example_tool
type: postgres-sql
source: my-pg-instance
description: some description
statement: |
SELECT * FROM SQL_STATEMENT;
parameters:
- name: country
type: string
description: some description
---
kind: toolsets
name: example_toolset
tools:
- example_tool
`,
wantToolsFile: ToolsFile{
Sources: server.SourceConfigs{
"my-pg-instance": cloudsqlpgsrc.Config{
Name: "my-pg-instance",
Type: cloudsqlpgsrc.SourceType,
Project: "my-project",
Region: "my-region",
Instance: "my-instance",
IPType: "public",
Database: "my_db",
User: "my_user",
Password: "my_pass",
},
},
Tools: server.ToolConfigs{
"example_tool": postgressql.Config{
Name: "example_tool",
Type: "postgres-sql",
Source: "my-pg-instance",
Description: "some description",
Statement: "SELECT * FROM SQL_STATEMENT;\n",
Parameters: []tools.Parameter{
tools.NewStringParameter("country", "some description"),
},
AuthRequired: []string{},
},
},
Toolsets: server.ToolsetConfigs{
"example_toolset": tools.ToolsetConfig{
Name: "example_toolset",
ToolNames: []string{"example_tool"},
},
},
},
},
}
for _, tc := range tcs {
t.Run(tc.description, func(t *testing.T) {
@@ -920,7 +985,7 @@ func TestParseToolFileWithAuth(t *testing.T) {
Password: "my_pass",
},
},
AuthSources: server.AuthServiceConfigs{
AuthServices: server.AuthServiceConfigs{
"my-google-service": google.Config{
Name: "my-google-service",
Type: google.AuthServiceType,

View File

@@ -31,7 +31,7 @@ var _ auth.AuthServiceConfig = Config{}
// Auth service configuration
type Config struct {
Name string `yaml:"name" validate:"required"`
Type string `yaml:"kind" validate:"required"`
Type string `yaml:"type" validate:"required"`
ClientID string `yaml:"clientId" validate:"required"`
}

View File

@@ -14,6 +14,7 @@
package server
import (
"bytes"
"context"
"fmt"
"strings"
@@ -111,162 +112,151 @@ func (s *StringLevel) Type() string {
return "stringLevel"
}
func UnmarshalResourceConfig(ctx context.Context, raw []byte) (SourceConfigs, AuthServiceConfigs, ToolConfigs, ToolsetConfigs, error) {
// prepare configs map
sourceConfigs := make(SourceConfigs)
authServiceConfigs := make(AuthServiceConfigs)
toolConfigs := make(ToolConfigs)
toolsetConfigs := make(ToolsetConfigs)
decoder := yaml.NewDecoder(bytes.NewReader(raw))
// for loop to unmarshal documents with the `---` separator
for {
var resource map[string]any
if err := decoder.DecodeContext(ctx, &resource); err != nil {
if err.Error() == "EOF" {
break
}
return nil, nil, nil, nil, fmt.Errorf("unable to parse kind: %s", err)
}
var kind, name string
var ok bool
if kind, ok = resource["kind"].(string); !ok {
return nil, nil, nil, nil, fmt.Errorf("missing 'kind' field or it is not a string")
}
if name, ok = resource["name"].(string); !ok {
return nil, nil, nil, nil, fmt.Errorf("missing 'name' field or it is not a string")
}
// remove 'kind' from map for strict unmarshaling
delete(resource, "kind")
switch kind {
case "sources":
c, err := UnmarshalYAMLSourceConfig(ctx, name, resource)
if err != nil {
return nil, nil, nil, nil, fmt.Errorf("error unmarshaling %s: %s", kind, err)
}
sourceConfigs[name] = c
case "authServices":
c, err := UnmarshalYAMLAuthServiceConfig(ctx, name, resource)
if err != nil {
return nil, nil, nil, nil, fmt.Errorf("error unmarshaling %s: %s", kind, err)
}
authServiceConfigs[name] = c
case "tools":
c, err := UnmarshalYAMLToolConfig(ctx, name, resource)
if err != nil {
return nil, nil, nil, nil, fmt.Errorf("error unmarshaling %s: %s", kind, err)
}
toolConfigs[name] = c
case "toolsets":
c, err := UnmarshalYAMLToolsetConfig(ctx, name, resource)
if err != nil {
return nil, nil, nil, nil, fmt.Errorf("error unmarshaling %s: %s", kind, err)
}
toolsetConfigs[name] = c
default:
return nil, nil, nil, nil, fmt.Errorf("invalid kind %s", kind)
}
}
return sourceConfigs, authServiceConfigs, toolConfigs, toolsetConfigs, nil
}
// SourceConfigs is a type used to allow unmarshal of the data source config map
type SourceConfigs map[string]sources.SourceConfig
// validate interface
var _ yaml.InterfaceUnmarshalerContext = &SourceConfigs{}
func (c *SourceConfigs) UnmarshalYAML(ctx context.Context, unmarshal func(interface{}) error) error {
*c = make(SourceConfigs)
// Parse the 'type' fields for each source
var raw map[string]util.DelayedUnmarshaler
if err := unmarshal(&raw); err != nil {
return err
func UnmarshalYAMLSourceConfig(ctx context.Context, name string, r map[string]any) (sources.SourceConfig, error) {
typeStr, ok := r["type"].(string)
if !ok {
return nil, fmt.Errorf("missing 'name' field or it is not a string")
}
for name, u := range raw {
// Unmarshal to a general type that ensure it capture all fields
var v map[string]any
if err := u.Unmarshal(&v); err != nil {
return fmt.Errorf("unable to unmarshal %q: %w", name, err)
}
sourceType, ok := v["kind"]
if !ok {
return fmt.Errorf("missing 'kind' field for source %q", name)
}
typeStr, ok := sourceType.(string)
if !ok {
return fmt.Errorf("invalid 'kind' field for source %q (must be a string)", name)
}
yamlDecoder, err := util.NewStrictDecoder(v)
if err != nil {
return fmt.Errorf("error creating YAML decoder for source %q: %w", name, err)
}
sourceConfig, err := sources.DecodeConfig(ctx, typeStr, name, yamlDecoder)
if err != nil {
return err
}
(*c)[name] = sourceConfig
dec, err := util.NewStrictDecoder(r)
if err != nil {
return nil, fmt.Errorf("error creating decoder: %s", err)
}
return nil
sourceConfig, err := sources.DecodeConfig(ctx, typeStr, name, dec)
if err != nil {
return nil, err
}
return sourceConfig, nil
}
// AuthServiceConfigs is a type used to allow unmarshal of the data authService config map
type AuthServiceConfigs map[string]auth.AuthServiceConfig
// validate interface
var _ yaml.InterfaceUnmarshalerContext = &AuthServiceConfigs{}
func (c *AuthServiceConfigs) UnmarshalYAML(ctx context.Context, unmarshal func(interface{}) error) error {
*c = make(AuthServiceConfigs)
// Parse the 'type' fields for each authService
var raw map[string]util.DelayedUnmarshaler
if err := unmarshal(&raw); err != nil {
return err
func UnmarshalYAMLAuthServiceConfig(ctx context.Context, name string, r map[string]any) (auth.AuthServiceConfig, error) {
typeStr, ok := r["type"].(string)
if !ok {
return nil, fmt.Errorf("missing 'name' field or it is not a string")
}
for name, u := range raw {
var v map[string]any
if err := u.Unmarshal(&v); err != nil {
return fmt.Errorf("unable to unmarshal %q: %w", name, err)
}
asType, ok := v["kind"]
if !ok {
return fmt.Errorf("missing 'kind' field for %q", name)
}
dec, err := util.NewStrictDecoder(v)
if err != nil {
return fmt.Errorf("error creating decoder: %w", err)
}
switch asType {
case google.AuthServiceType:
actual := google.Config{Name: name}
if err := dec.DecodeContext(ctx, &actual); err != nil {
return fmt.Errorf("unable to parse as %q: %w", asType, err)
}
(*c)[name] = actual
default:
return fmt.Errorf("%q is not a valid kind of auth source", asType)
}
if typeStr != google.AuthServiceType {
return nil, fmt.Errorf("%s is not a valid type of auth source", typeStr)
}
return nil
dec, err := util.NewStrictDecoder(r)
if err != nil {
return nil, fmt.Errorf("error creating decoder: %s", err)
}
actual := google.Config{Name: name}
if err := dec.DecodeContext(ctx, &actual); err != nil {
return nil, fmt.Errorf("unable to parse as %s: %w", name, err)
}
return actual, nil
}
// ToolConfigs is a type used to allow unmarshal of the tool configs
type ToolConfigs map[string]tools.ToolConfig
// validate interface
var _ yaml.InterfaceUnmarshalerContext = &ToolConfigs{}
func (c *ToolConfigs) UnmarshalYAML(ctx context.Context, unmarshal func(interface{}) error) error {
*c = make(ToolConfigs)
// Parse the 'type' fields for each source
var raw map[string]util.DelayedUnmarshaler
if err := unmarshal(&raw); err != nil {
return err
func UnmarshalYAMLToolConfig(ctx context.Context, name string, r map[string]any) (tools.ToolConfig, error) {
typeStr, ok := r["type"].(string)
if !ok {
return nil, fmt.Errorf("missing 'name' field or it is not a string")
}
for name, u := range raw {
var v map[string]any
if err := u.Unmarshal(&v); err != nil {
return fmt.Errorf("unable to unmarshal %q: %w", name, err)
}
// `authRequired` and `useClientOAuth` cannot be specified together
if v["authRequired"] != nil && v["useClientOAuth"] == true {
return fmt.Errorf("`authRequired` and `useClientOAuth` are mutually exclusive. Choose only one authentication method")
}
// Make `authRequired` an empty list instead of nil for Tool manifest
if v["authRequired"] == nil {
v["authRequired"] = []string{}
}
typeVal, ok := v["kind"]
if !ok {
return fmt.Errorf("missing 'kind' field for tool %q", name)
}
typeStr, ok := typeVal.(string)
if !ok {
return fmt.Errorf("invalid 'kind' field for tool %q (must be a string)", name)
}
yamlDecoder, err := util.NewStrictDecoder(v)
if err != nil {
return fmt.Errorf("error creating YAML decoder for tool %q: %w", name, err)
}
toolCfg, err := tools.DecodeConfig(ctx, typeStr, name, yamlDecoder)
if err != nil {
return err
}
(*c)[name] = toolCfg
// `authRequired` and `useClientOAuth` cannot be specified together
if r["authRequired"] != nil && r["useClientOAuth"] == true {
return nil, fmt.Errorf("`authRequired` and `useClientOAuth` are mutually exclusive. Choose only one authentication method")
}
return nil
// Make `authRequired` an empty list instead of nil for Tool manifest
if r["authRequired"] == nil {
r["authRequired"] = []string{}
}
dec, err := util.NewStrictDecoder(r)
if err != nil {
return nil, fmt.Errorf("error creating decoder: %s", err)
}
toolCfg, err := tools.DecodeConfig(ctx, typeStr, name, dec)
if err != nil {
return nil, err
}
return toolCfg, nil
}
// ToolConfigs is a type used to allow unmarshal of the toolset configs
type ToolsetConfigs map[string]tools.ToolsetConfig
// validate interface
var _ yaml.InterfaceUnmarshalerContext = &ToolsetConfigs{}
func (c *ToolsetConfigs) UnmarshalYAML(ctx context.Context, unmarshal func(interface{}) error) error {
*c = make(ToolsetConfigs)
func UnmarshalYAMLToolsetConfig(ctx context.Context, name string, r map[string]any) (tools.ToolsetConfig, error) {
var toolsetConfig tools.ToolsetConfig
justTools := map[string]any{"tools": r["tools"]}
dec, err := util.NewStrictDecoder(justTools)
if err != nil {
return toolsetConfig, fmt.Errorf("error creating decoder: %s", err)
}
var raw map[string][]string
if err := unmarshal(&raw); err != nil {
return err
if err := dec.DecodeContext(ctx, &raw); err != nil {
return toolsetConfig, fmt.Errorf("unable to unmarshal tools: %s", err)
}
for name, toolList := range raw {
(*c)[name] = tools.ToolsetConfig{Name: name, ToolNames: toolList}
}
return nil
return tools.ToolsetConfig{Name: name, ToolNames: raw["tools"]}, nil
}

View File

@@ -49,7 +49,7 @@ func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (sources
type Config struct {
Name string `yaml:"name" validate:"required"`
Type string `yaml:"kind" validate:"required"`
Type string `yaml:"type" validate:"required"`
Project string `yaml:"project" validate:"required"`
Region string `yaml:"region" validate:"required"`
Cluster string `yaml:"cluster" validate:"required"`

View File

@@ -15,14 +15,13 @@
package alloydbpg_test
import (
"context"
"testing"
yaml "github.com/goccy/go-yaml"
"github.com/google/go-cmp/cmp"
"github.com/googleapis/genai-toolbox/internal/server"
"github.com/googleapis/genai-toolbox/internal/sources"
"github.com/googleapis/genai-toolbox/internal/sources/alloydbpg"
"github.com/googleapis/genai-toolbox/internal/testutils"
)
func TestParseFromYamlAlloyDBPg(t *testing.T) {
@@ -34,17 +33,17 @@ func TestParseFromYamlAlloyDBPg(t *testing.T) {
{
desc: "basic example",
in: `
sources:
my-pg-instance:
kind: alloydb-postgres
project: my-project
region: my-region
cluster: my-cluster
instance: my-instance
database: my_db
user: my_user
password: my_pass
`,
type: sources
name: my-pg-instance
type: alloydb-postgres
project: my-project
region: my-region
cluster: my-cluster
instance: my-instance
database: my_db
user: my_user
password: my_pass
`,
want: map[string]sources.SourceConfig{
"my-pg-instance": alloydbpg.Config{
Name: "my-pg-instance",
@@ -63,18 +62,18 @@ func TestParseFromYamlAlloyDBPg(t *testing.T) {
{
desc: "public ipType",
in: `
sources:
my-pg-instance:
kind: alloydb-postgres
project: my-project
region: my-region
cluster: my-cluster
instance: my-instance
ipType: Public
database: my_db
user: my_user
password: my_pass
`,
type: sources
name: my-pg-instance
type: alloydb-postgres
project: my-project
region: my-region
cluster: my-cluster
instance: my-instance
ipType: Public
database: my_db
user: my_user
password: my_pass
`,
want: map[string]sources.SourceConfig{
"my-pg-instance": alloydbpg.Config{
Name: "my-pg-instance",
@@ -93,18 +92,18 @@ func TestParseFromYamlAlloyDBPg(t *testing.T) {
{
desc: "private ipType",
in: `
sources:
my-pg-instance:
kind: alloydb-postgres
project: my-project
region: my-region
cluster: my-cluster
instance: my-instance
ipType: private
database: my_db
user: my_user
password: my_pass
`,
type: sources
name: my-pg-instance
type: alloydb-postgres
project: my-project
region: my-region
cluster: my-cluster
instance: my-instance
ipType: private
database: my_db
user: my_user
password: my_pass
`,
want: map[string]sources.SourceConfig{
"my-pg-instance": alloydbpg.Config{
Name: "my-pg-instance",
@@ -123,16 +122,12 @@ func TestParseFromYamlAlloyDBPg(t *testing.T) {
}
for _, tc := range tcs {
t.Run(tc.desc, func(t *testing.T) {
got := struct {
Sources server.SourceConfigs `yaml:"sources"`
}{}
// Parse contents
err := yaml.Unmarshal(testutils.FormatYaml(tc.in), &got)
sources, _, _, _, err := server.UnmarshalResourceConfig(context.Background(), []byte(tc.in))
if err != nil {
t.Fatalf("unable to unmarshal: %s", err)
}
if !cmp.Equal(tc.want, got.Sources) {
t.Fatalf("incorrect parse: want %v, got %v", tc.want, got.Sources)
if !cmp.Equal(tc.want, sources) {
t.Fatalf("incorrect parse: want %v, got %v", tc.want, sources)
}
})
}
@@ -147,60 +142,71 @@ func TestFailParseFromYaml(t *testing.T) {
{
desc: "invalid ipType",
in: `
sources:
my-pg-instance:
kind: alloydb-postgres
project: my-project
region: my-region
cluster: my-cluster
instance: my-instance
ipType: fail
database: my_db
user: my_user
password: my_pass
`,
err: "unable to parse source \"my-pg-instance\" as \"alloydb-postgres\": ipType invalid: must be one of \"public\", or \"private\"",
type: sources
name: my-pg-instance
type: alloydb-postgres
project: my-project
region: my-region
cluster: my-cluster
instance: my-instance
ipType: fail
database: my_db
user: my_user
password: my_pass
`,
err: "error unmarshaling sources: unable to parse source \"my-pg-instance\" as \"alloydb-postgres\": ipType invalid: must be one of \"public\", or \"private\"",
},
{
desc: "extra field",
in: `
sources:
my-pg-instance:
kind: alloydb-postgres
project: my-project
region: my-region
cluster: my-cluster
instance: my-instance
database: my_db
user: my_user
password: my_pass
foo: bar
`,
err: "unable to parse source \"my-pg-instance\" as \"alloydb-postgres\": [3:1] unknown field \"foo\"\n 1 | cluster: my-cluster\n 2 | database: my_db\n> 3 | foo: bar\n ^\n 4 | instance: my-instance\n 5 | kind: alloydb-postgres\n 6 | password: my_pass\n 7 | ",
type: sources
name: my-pg-instance
type: alloydb-postgres
project: my-project
region: my-region
cluster: my-cluster
instance: my-instance
database: my_db
user: my_user
password: my_pass
foo: bar
`,
err: "error unmarshaling sources: unable to parse source \"my-pg-instance\" as \"alloydb-postgres\": [3:1] unknown field \"foo\"\n 1 | cluster: my-cluster\n 2 | database: my_db\n> 3 | foo: bar\n ^\n 4 | instance: my-instance\n 5 | name: my-pg-instance\n 6 | password: my_pass\n 7 | ",
},
{
desc: "missing required field",
in: `
sources:
my-pg-instance:
kind: alloydb-postgres
region: my-region
cluster: my-cluster
instance: my-instance
database: my_db
user: my_user
password: my_pass
`,
err: "unable to parse source \"my-pg-instance\" as \"alloydb-postgres\": Key: 'Config.Project' Error:Field validation for 'Project' failed on the 'required' tag",
type: sources
name: my-pg-instance
type: alloydb-postgres
region: my-region
cluster: my-cluster
instance: my-instance
database: my_db
user: my_user
password: my_pass
`,
err: "error unmarshaling sources: unable to parse source \"my-pg-instance\" as \"alloydb-postgres\": Key: 'Config.Project' Error:Field validation for 'Project' failed on the 'required' tag",
},
{
desc: "old tools file format",
in: `
sources:
my-pg-instance:
type: alloydb-postgres
region: my-region
cluster: my-cluster
instance: my-instance
database: my_db
user: my_user
password: my_pass
`,
err: "missing 'type' field or it is not a string",
},
}
for _, tc := range tcs {
t.Run(tc.desc, func(t *testing.T) {
got := struct {
Sources server.SourceConfigs `yaml:"sources"`
}{}
// Parse contents
err := yaml.Unmarshal(testutils.FormatYaml(tc.in), &got)
_, _, _, _, err := server.UnmarshalResourceConfig(context.Background(), []byte(tc.in))
if err == nil {
t.Fatalf("expect parsing to fail")
}

View File

@@ -54,7 +54,7 @@ func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (sources
type Config struct {
// BigQuery configs
Name string `yaml:"name" validate:"required"`
Type string `yaml:"kind" validate:"required"`
Type string `yaml:"type" validate:"required"`
Project string `yaml:"project" validate:"required"`
Location string `yaml:"location"`
UseClientOAuth bool `yaml:"useClientOAuth"`

View File

@@ -35,7 +35,7 @@ func TestParseFromYamlBigQuery(t *testing.T) {
in: `
sources:
my-instance:
kind: bigquery
type: bigquery
project: my-project
location: us
`,
@@ -54,7 +54,7 @@ func TestParseFromYamlBigQuery(t *testing.T) {
in: `
sources:
my-instance:
kind: bigquery
type: bigquery
project: my-project
location: us
useClientOAuth: true
@@ -99,19 +99,19 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
my-instance:
kind: bigquery
type: bigquery
project: my-project
location: us
foo: bar
`,
err: "unable to parse source \"my-instance\" as \"bigquery\": [1:1] unknown field \"foo\"\n> 1 | foo: bar\n ^\n 2 | kind: bigquery\n 3 | location: us\n 4 | project: my-project",
err: "unable to parse source \"my-instance\" as \"bigquery\": [1:1] unknown field \"foo\"\n> 1 | foo: bar\n ^\n 2 | type: bigquery\n 3 | location: us\n 4 | project: my-project",
},
{
desc: "missing required field",
in: `
sources:
my-instance:
kind: bigquery
type: bigquery
location: us
`,
err: "unable to parse source \"my-instance\" as \"bigquery\": Key: 'Config.Project' Error:Field validation for 'Project' failed on the 'required' tag",

View File

@@ -47,7 +47,7 @@ func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (sources
type Config struct {
Name string `yaml:"name" validate:"required"`
Type string `yaml:"kind" validate:"required"`
Type string `yaml:"type" validate:"required"`
Project string `yaml:"project" validate:"required"`
Instance string `yaml:"instance" validate:"required"`
}

View File

@@ -36,7 +36,7 @@ func TestParseFromYamlBigtableDb(t *testing.T) {
in: `
sources:
my-bigtable-instance:
kind: bigtable
type: bigtable
project: my-project
instance: my-instance
`,
@@ -79,19 +79,19 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
my-bigtable-instance:
kind: bigtable
type: bigtable
project: my-project
instance: my-instance
foo: bar
`,
err: "unable to parse source \"my-bigtable-instance\" as \"bigtable\": [1:1] unknown field \"foo\"\n> 1 | foo: bar\n ^\n 2 | instance: my-instance\n 3 | kind: bigtable\n 4 | project: my-project",
err: "unable to parse source \"my-bigtable-instance\" as \"bigtable\": [1:1] unknown field \"foo\"\n> 1 | foo: bar\n ^\n 2 | instance: my-instance\n 3 | type: bigtable\n 4 | project: my-project",
},
{
desc: "missing required field",
in: `
sources:
my-bigtable-instance:
kind: bigtable
type: bigtable
project: my-project
`,
err: "unable to parse source \"my-bigtable-instance\" as \"bigtable\": Key: 'Config.Instance' Error:Field validation for 'Instance' failed on the 'required' tag",

View File

@@ -48,7 +48,7 @@ func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (sources
type Config struct {
Name string `yaml:"name" validate:"required"`
Type string `yaml:"kind" validate:"required"`
Type string `yaml:"type" validate:"required"`
Host string `yaml:"host" validate:"required"`
Port string `yaml:"port" validate:"required"`
Database string `yaml:"database" validate:"required"`

View File

@@ -42,7 +42,7 @@ func TestNewConfig(t *testing.T) {
name: "all fields specified",
yaml: `
name: test-clickhouse
kind: clickhouse
type: clickhouse
host: localhost
port: "8443"
user: default
@@ -67,7 +67,7 @@ func TestNewConfig(t *testing.T) {
name: "minimal configuration with defaults",
yaml: `
name: minimal-clickhouse
kind: clickhouse
type: clickhouse
host: 127.0.0.1
port: "8123"
user: testuser
@@ -89,7 +89,7 @@ func TestNewConfig(t *testing.T) {
name: "http protocol",
yaml: `
name: http-clickhouse
kind: clickhouse
type: clickhouse
host: clickhouse.example.com
port: "8123"
user: analytics
@@ -114,7 +114,7 @@ func TestNewConfig(t *testing.T) {
name: "https with secure connection",
yaml: `
name: secure-clickhouse
kind: clickhouse
type: clickhouse
host: secure.clickhouse.io
port: "8443"
user: secureuser
@@ -167,7 +167,7 @@ func TestNewConfigInvalidYAML(t *testing.T) {
name: "invalid yaml syntax",
yaml: `
name: test-clickhouse
kind: clickhouse
type: clickhouse
host: [invalid
`,
expectError: true,
@@ -176,7 +176,7 @@ func TestNewConfigInvalidYAML(t *testing.T) {
name: "missing required fields",
yaml: `
name: test-clickhouse
kind: clickhouse
type: clickhouse
`,
expectError: false,
},

View File

@@ -50,7 +50,7 @@ func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (sources
type Config struct {
// Cloud SQL MSSQL configs
Name string `yaml:"name" validate:"required"`
Type string `yaml:"kind" validate:"required"`
Type string `yaml:"type" validate:"required"`
Project string `yaml:"project" validate:"required"`
Region string `yaml:"region" validate:"required"`
Instance string `yaml:"instance" validate:"required"`

View File

@@ -35,7 +35,7 @@ func TestParseFromYamlCloudSQLMssql(t *testing.T) {
in: `
sources:
my-instance:
kind: cloud-sql-mssql
type: cloud-sql-mssql
project: my-project
region: my-region
instance: my-instance
@@ -89,7 +89,7 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
my-instance:
kind: cloud-sql-mssql
type: cloud-sql-mssql
project: my-project
region: my-region
instance: my-instance
@@ -106,7 +106,7 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
my-instance:
kind: cloud-sql-mssql
type: cloud-sql-mssql
project: my-project
region: my-region
instance: my-instance
@@ -116,14 +116,14 @@ func TestFailParseFromYaml(t *testing.T) {
password: my_pass
foo: bar
`,
err: "unable to parse source \"my-instance\" as \"cloud-sql-mssql\": [2:1] unknown field \"foo\"\n 1 | database: my_db\n> 2 | foo: bar\n ^\n 3 | instance: my-instance\n 4 | ipAddress: localhost\n 5 | kind: cloud-sql-mssql\n 6 | ",
err: "unable to parse source \"my-instance\" as \"cloud-sql-mssql\": [2:1] unknown field \"foo\"\n 1 | database: my_db\n> 2 | foo: bar\n ^\n 3 | instance: my-instance\n 4 | ipAddress: localhost\n 5 | type: cloud-sql-mssql\n 6 | ",
},
{
desc: "missing required field",
in: `
sources:
my-instance:
kind: cloud-sql-mssql
type: cloud-sql-mssql
region: my-region
instance: my-instance
database: my_db

View File

@@ -48,7 +48,7 @@ func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (sources
type Config struct {
Name string `yaml:"name" validate:"required"`
Type string `yaml:"kind" validate:"required"`
Type string `yaml:"type" validate:"required"`
Project string `yaml:"project" validate:"required"`
Region string `yaml:"region" validate:"required"`
Instance string `yaml:"instance" validate:"required"`

View File

@@ -35,7 +35,7 @@ func TestParseFromYamlCloudSQLMySQL(t *testing.T) {
in: `
sources:
my-mysql-instance:
kind: cloud-sql-mysql
type: cloud-sql-mysql
project: my-project
region: my-region
instance: my-instance
@@ -62,7 +62,7 @@ func TestParseFromYamlCloudSQLMySQL(t *testing.T) {
in: `
sources:
my-mysql-instance:
kind: cloud-sql-mysql
type: cloud-sql-mysql
project: my-project
region: my-region
instance: my-instance
@@ -90,7 +90,7 @@ func TestParseFromYamlCloudSQLMySQL(t *testing.T) {
in: `
sources:
my-mysql-instance:
kind: cloud-sql-mysql
type: cloud-sql-mysql
project: my-project
region: my-region
instance: my-instance
@@ -143,7 +143,7 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
my-mysql-instance:
kind: cloud-sql-mysql
type: cloud-sql-mysql
project: my-project
region: my-region
instance: my-instance
@@ -159,7 +159,7 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
my-mysql-instance:
kind: cloud-sql-mysql
type: cloud-sql-mysql
project: my-project
region: my-region
instance: my-instance
@@ -168,14 +168,14 @@ func TestFailParseFromYaml(t *testing.T) {
password: my_pass
foo: bar
`,
err: "unable to parse source \"my-mysql-instance\" as \"cloud-sql-mysql\": [2:1] unknown field \"foo\"\n 1 | database: my_db\n> 2 | foo: bar\n ^\n 3 | instance: my-instance\n 4 | kind: cloud-sql-mysql\n 5 | password: my_pass\n 6 | ",
err: "unable to parse source \"my-mysql-instance\" as \"cloud-sql-mysql\": [2:1] unknown field \"foo\"\n 1 | database: my_db\n> 2 | foo: bar\n ^\n 3 | instance: my-instance\n 4 | type: cloud-sql-mysql\n 5 | password: my_pass\n 6 | ",
},
{
desc: "missing required field",
in: `
sources:
my-mysql-instance:
kind: cloud-sql-mysql
type: cloud-sql-mysql
region: my-region
instance: my-instance
database: my_db

View File

@@ -48,7 +48,7 @@ func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (sources
type Config struct {
Name string `yaml:"name" validate:"required"`
Type string `yaml:"kind" validate:"required"`
Type string `yaml:"type" validate:"required"`
Project string `yaml:"project" validate:"required"`
Region string `yaml:"region" validate:"required"`
Instance string `yaml:"instance" validate:"required"`

View File

@@ -35,7 +35,7 @@ func TestParseFromYamlCloudSQLPg(t *testing.T) {
in: `
sources:
my-pg-instance:
kind: cloud-sql-postgres
type: cloud-sql-postgres
project: my-project
region: my-region
instance: my-instance
@@ -62,7 +62,7 @@ func TestParseFromYamlCloudSQLPg(t *testing.T) {
in: `
sources:
my-pg-instance:
kind: cloud-sql-postgres
type: cloud-sql-postgres
project: my-project
region: my-region
instance: my-instance
@@ -90,7 +90,7 @@ func TestParseFromYamlCloudSQLPg(t *testing.T) {
in: `
sources:
my-pg-instance:
kind: cloud-sql-postgres
type: cloud-sql-postgres
project: my-project
region: my-region
instance: my-instance
@@ -143,7 +143,7 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
my-pg-instance:
kind: cloud-sql-postgres
type: cloud-sql-postgres
project: my-project
region: my-region
instance: my-instance
@@ -159,7 +159,7 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
my-pg-instance:
kind: cloud-sql-postgres
type: cloud-sql-postgres
project: my-project
region: my-region
instance: my-instance
@@ -168,14 +168,14 @@ func TestFailParseFromYaml(t *testing.T) {
password: my_pass
foo: bar
`,
err: "unable to parse source \"my-pg-instance\" as \"cloud-sql-postgres\": [2:1] unknown field \"foo\"\n 1 | database: my_db\n> 2 | foo: bar\n ^\n 3 | instance: my-instance\n 4 | kind: cloud-sql-postgres\n 5 | password: my_pass\n 6 | ",
err: "unable to parse source \"my-pg-instance\" as \"cloud-sql-postgres\": [2:1] unknown field \"foo\"\n 1 | database: my_db\n> 2 | foo: bar\n ^\n 3 | instance: my-instance\n 4 | type: cloud-sql-postgres\n 5 | password: my_pass\n 6 | ",
},
{
desc: "missing required field",
in: `
sources:
my-pg-instance:
kind: cloud-sql-postgres
type: cloud-sql-postgres
region: my-region
instance: my-instance
database: my_db

View File

@@ -48,7 +48,7 @@ func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (sources
type Config struct {
Name string `yaml:"name" validate:"required"`
Type string `yaml:"kind" validate:"required"`
Type string `yaml:"type" validate:"required"`
ConnectionString string `yaml:"connectionString" validate:"required"`
Bucket string `yaml:"bucket" validate:"required"`
Scope string `yaml:"scope" validate:"required"`

View File

@@ -35,7 +35,7 @@ func TestParseFromYamlCouchbase(t *testing.T) {
in: `
sources:
my-couchbase-instance:
kind: couchbase
type: couchbase
connectionString: localhost
username: Administrator
password: password
@@ -59,7 +59,7 @@ func TestParseFromYamlCouchbase(t *testing.T) {
in: `
sources:
my-couchbase-instance:
kind: couchbase
type: couchbase
connectionString: couchbases://localhost
bucket: travel-sample
scope: inventory
@@ -117,7 +117,7 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
my-couchbase-instance:
kind: couchbase
type: couchbase
connectionString: localhost
username: Administrator
password: password
@@ -125,14 +125,14 @@ func TestFailParseFromYaml(t *testing.T) {
scope: inventory
foo: bar
`,
err: "unable to parse source \"my-couchbase-instance\" as \"couchbase\": [3:1] unknown field \"foo\"\n 1 | bucket: travel-sample\n 2 | connectionString: localhost\n> 3 | foo: bar\n ^\n 4 | kind: couchbase\n 5 | password: password\n 6 | scope: inventory\n 7 | ",
err: "unable to parse source \"my-couchbase-instance\" as \"couchbase\": [3:1] unknown field \"foo\"\n 1 | bucket: travel-sample\n 2 | connectionString: localhost\n> 3 | foo: bar\n ^\n 4 | type: couchbase\n 5 | password: password\n 6 | scope: inventory\n 7 | ",
},
{
desc: "missing required field",
in: `
sources:
my-couchbase-instance:
kind: couchbase
type: couchbase
username: Administrator
password: password
bucket: travel-sample

View File

@@ -49,7 +49,7 @@ func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (sources
type Config struct {
// Dataplex configs
Name string `yaml:"name" validate:"required"`
Type string `yaml:"kind" validate:"required"`
Type string `yaml:"type" validate:"required"`
Project string `yaml:"project" validate:"required"`
}

View File

@@ -35,7 +35,7 @@ func TestParseFromYamlDataplex(t *testing.T) {
in: `
sources:
my-instance:
kind: dataplex
type: dataplex
project: my-project
`,
want: server.SourceConfigs{
@@ -76,18 +76,18 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
my-instance:
kind: dataplex
type: dataplex
project: my-project
foo: bar
`,
err: "unable to parse source \"my-instance\" as \"dataplex\": [1:1] unknown field \"foo\"\n> 1 | foo: bar\n ^\n 2 | kind: dataplex\n 3 | project: my-project",
err: "unable to parse source \"my-instance\" as \"dataplex\": [1:1] unknown field \"foo\"\n> 1 | foo: bar\n ^\n 2 | type: dataplex\n 3 | project: my-project",
},
{
desc: "missing required field",
in: `
sources:
my-instance:
kind: dataplex
type: dataplex
`,
err: "unable to parse source \"my-instance\" as \"dataplex\": Key: 'Config.Project' Error:Field validation for 'Project' failed on the 'required' tag",
},

View File

@@ -66,7 +66,7 @@ type DgraphClient struct {
type Config struct {
Name string `yaml:"name" validate:"required"`
Type string `yaml:"kind" validate:"required"`
Type string `yaml:"type" validate:"required"`
DgraphUrl string `yaml:"dgraphUrl" validate:"required"`
User string `yaml:"user"`
Password string `yaml:"password"`

View File

@@ -35,7 +35,7 @@ func TestParseFromYamlDgraph(t *testing.T) {
in: `
sources:
my-dgraph-instance:
kind: dgraph
type: dgraph
dgraphUrl: https://localhost:8080
apiKey: abc123
password: pass@123
@@ -59,7 +59,7 @@ func TestParseFromYamlDgraph(t *testing.T) {
in: `
sources:
my-dgraph-instance:
kind: dgraph
type: dgraph
dgraphUrl: https://localhost:8080
`,
want: server.SourceConfigs{
@@ -102,18 +102,18 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
my-dgraph-instance:
kind: dgraph
type: dgraph
dgraphUrl: https://localhost:8080
foo: bar
`,
err: "unable to parse source \"my-dgraph-instance\" as \"dgraph\": [2:1] unknown field \"foo\"\n 1 | dgraphUrl: https://localhost:8080\n> 2 | foo: bar\n ^\n 3 | kind: dgraph",
err: "unable to parse source \"my-dgraph-instance\" as \"dgraph\": [2:1] unknown field \"foo\"\n 1 | dgraphUrl: https://localhost:8080\n> 2 | foo: bar\n ^\n 3 | type: dgraph",
},
{
desc: "missing required field",
in: `
sources:
my-dgraph-instance:
kind: dgraph
type: dgraph
`,
err: "unable to parse source \"my-dgraph-instance\" as \"dgraph\": Key: 'Config.DgraphUrl' Error:Field validation for 'DgraphUrl' failed on the 'required' tag",
},

View File

@@ -45,7 +45,7 @@ func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (sources
type Config struct {
Name string `yaml:"name" validate:"required"`
Type string `yaml:"kind" validate:"required"`
Type string `yaml:"type" validate:"required"`
Host string `yaml:"host" validate:"required"`
Port string `yaml:"port" validate:"required"`
User string `yaml:"user" validate:"required"`

View File

@@ -35,7 +35,7 @@ func TestParseFromYamlFirebird(t *testing.T) {
in: `
sources:
my-fdb-instance:
kind: firebird
type: firebird
host: my-host
port: my-port
database: my_db
@@ -84,7 +84,7 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
my-fdb-instance:
kind: firebird
type: firebird
host: my-host
port: my-port
database: my_db
@@ -92,14 +92,14 @@ func TestFailParseFromYaml(t *testing.T) {
password: my_pass
foo: bar
`,
err: "unable to parse source \"my-fdb-instance\" as \"firebird\": [2:1] unknown field \"foo\"\n 1 | database: my_db\n> 2 | foo: bar\n ^\n 3 | host: my-host\n 4 | kind: firebird\n 5 | password: my_pass\n 6 | ",
err: "unable to parse source \"my-fdb-instance\" as \"firebird\": [2:1] unknown field \"foo\"\n 1 | database: my_db\n> 2 | foo: bar\n ^\n 3 | host: my-host\n 4 | type: firebird\n 5 | password: my_pass\n 6 | ",
},
{
desc: "missing required field",
in: `
sources:
my-fdb-instance:
kind: firebird
type: firebird
host: my-host
port: my-port
database: my_db

View File

@@ -49,7 +49,7 @@ func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (sources
type Config struct {
// Firestore configs
Name string `yaml:"name" validate:"required"`
Type string `yaml:"kind" validate:"required"`
Type string `yaml:"type" validate:"required"`
Project string `yaml:"project" validate:"required"`
Database string `yaml:"database"` // Optional, defaults to "(default)"
}

View File

@@ -35,7 +35,7 @@ func TestParseFromYamlFirestore(t *testing.T) {
in: `
sources:
my-firestore:
kind: firestore
type: firestore
project: my-project
`,
want: server.SourceConfigs{
@@ -52,7 +52,7 @@ func TestParseFromYamlFirestore(t *testing.T) {
in: `
sources:
my-firestore:
kind: firestore
type: firestore
project: my-project
database: my-database
`,
@@ -94,18 +94,18 @@ func TestFailParseFromYamlFirestore(t *testing.T) {
in: `
sources:
my-firestore:
kind: firestore
type: firestore
project: my-project
foo: bar
`,
err: "unable to parse source \"my-firestore\" as \"firestore\": [1:1] unknown field \"foo\"\n> 1 | foo: bar\n ^\n 2 | kind: firestore\n 3 | project: my-project",
err: "unable to parse source \"my-firestore\" as \"firestore\": [1:1] unknown field \"foo\"\n> 1 | foo: bar\n ^\n 2 | type: firestore\n 3 | project: my-project",
},
{
desc: "missing required field",
in: `
sources:
my-firestore:
kind: firestore
type: firestore
database: my-database
`,
err: "unable to parse source \"my-firestore\" as \"firestore\": Key: 'Config.Project' Error:Field validation for 'Project' failed on the 'required' tag",

View File

@@ -48,7 +48,7 @@ func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (sources
type Config struct {
Name string `yaml:"name" validate:"required"`
Type string `yaml:"kind" validate:"required"`
Type string `yaml:"type" validate:"required"`
BaseURL string `yaml:"baseUrl"`
Timeout string `yaml:"timeout"`
DefaultHeaders map[string]string `yaml:"headers"`

View File

@@ -36,7 +36,7 @@ func TestParseFromYamlHttp(t *testing.T) {
in: `
sources:
my-http-instance:
kind: http
type: http
baseUrl: http://test_server/
`,
want: map[string]sources.SourceConfig{
@@ -54,7 +54,7 @@ func TestParseFromYamlHttp(t *testing.T) {
in: `
sources:
my-http-instance:
kind: http
type: http
baseUrl: http://test_server/
timeout: 10s
headers:
@@ -106,7 +106,7 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
my-http-instance:
kind: http
type: http
baseUrl: http://test_server/
timeout: 10s
headers:
@@ -115,7 +115,7 @@ func TestFailParseFromYaml(t *testing.T) {
api-key: test_api_key
project: test-project
`,
err: "unable to parse source \"my-http-instance\" as \"http\": [5:1] unknown field \"project\"\n 2 | headers:\n 3 | Authorization: test_header\n 4 | kind: http\n> 5 | project: test-project\n ^\n 6 | queryParams:\n 7 | api-key: test_api_key\n 8 | timeout: 10s",
err: "unable to parse source \"my-http-instance\" as \"http\": [5:1] unknown field \"project\"\n 2 | headers:\n 3 | Authorization: test_header\n 4 | type: http\n> 5 | project: test-project\n ^\n 6 | queryParams:\n 7 | api-key: test_api_key\n 8 | timeout: 10s",
},
{
desc: "missing required field",
@@ -124,7 +124,7 @@ func TestFailParseFromYaml(t *testing.T) {
my-http-instance:
baseUrl: http://test_server/
`,
err: "missing 'kind' field for source \"my-http-instance\"",
err: "missing 'type' field for source \"my-http-instance\"",
},
}
for _, tc := range tcs {

View File

@@ -56,7 +56,7 @@ func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (sources
type Config struct {
Name string `yaml:"name" validate:"required"`
Type string `yaml:"kind" validate:"required"`
Type string `yaml:"type" validate:"required"`
BaseURL string `yaml:"base_url" validate:"required"`
ClientId string `yaml:"client_id" validate:"required"`
ClientSecret string `yaml:"client_secret" validate:"required"`

View File

@@ -36,7 +36,7 @@ func TestParseFromYamlLooker(t *testing.T) {
in: `
sources:
my-looker-instance:
kind: looker
type: looker
base_url: http://example.looker.com/
client_id: jasdl;k;tjl
client_secret: sdakl;jgflkasdfkfg
@@ -86,20 +86,20 @@ func TestFailParseFromYamlLooker(t *testing.T) {
in: `
sources:
my-looker-instance:
kind: looker
type: looker
base_url: http://example.looker.com/
client_id: jasdl;k;tjl
client_secret: sdakl;jgflkasdfkfg
project: test-project
`,
err: "unable to parse source \"my-looker-instance\" as \"looker\": [5:1] unknown field \"project\"\n 2 | client_id: jasdl;k;tjl\n 3 | client_secret: sdakl;jgflkasdfkfg\n 4 | kind: looker\n> 5 | project: test-project\n ^\n",
err: "unable to parse source \"my-looker-instance\" as \"looker\": [5:1] unknown field \"project\"\n 2 | client_id: jasdl;k;tjl\n 3 | client_secret: sdakl;jgflkasdfkfg\n 4 | type: looker\n> 5 | project: test-project\n ^\n",
},
{
desc: "missing required field",
in: `
sources:
my-looker-instance:
kind: looker
type: looker
base_url: http://example.looker.com/
client_id: jasdl;k;tjl
`,

View File

@@ -47,7 +47,7 @@ func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (sources
type Config struct {
Name string `yaml:"name" validate:"required"`
Type string `yaml:"kind" validate:"required"`
Type string `yaml:"type" validate:"required"`
Uri string `yaml:"uri" validate:"required"` // MongoDB Atlas connection URI
}

View File

@@ -35,7 +35,7 @@ func TestParseFromYamlMongoDB(t *testing.T) {
in: `
sources:
mongo-db:
kind: "mongodb"
type: "mongodb"
uri: "mongodb+srv://username:password@host/dbname"
`,
want: server.SourceConfigs{
@@ -76,18 +76,18 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
mongo-db:
kind: mongodb
type: mongodb
uri: "mongodb+srv://username:password@host/dbname"
foo: bar
`,
err: "unable to parse source \"mongo-db\" as \"mongodb\": [1:1] unknown field \"foo\"\n> 1 | foo: bar\n ^\n 2 | kind: mongodb\n 3 | uri: mongodb+srv://username:password@host/dbname",
err: "unable to parse source \"mongo-db\" as \"mongodb\": [1:1] unknown field \"foo\"\n> 1 | foo: bar\n ^\n 2 | type: mongodb\n 3 | uri: mongodb+srv://username:password@host/dbname",
},
{
desc: "missing required field",
in: `
sources:
mongo-db:
kind: mongodb
type: mongodb
`,
err: "unable to parse source \"mongo-db\" as \"mongodb\": Key: 'Config.Uri' Error:Field validation for 'Uri' failed on the 'required' tag",
},

View File

@@ -48,7 +48,7 @@ func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (sources
type Config struct {
// Cloud SQL MSSQL configs
Name string `yaml:"name" validate:"required"`
Type string `yaml:"kind" validate:"required"`
Type string `yaml:"type" validate:"required"`
Host string `yaml:"host" validate:"required"`
Port string `yaml:"port" validate:"required"`
User string `yaml:"user" validate:"required"`

View File

@@ -35,7 +35,7 @@ func TestParseFromYamlMssql(t *testing.T) {
in: `
sources:
my-mssql-instance:
kind: mssql
type: mssql
host: 0.0.0.0
port: my-port
database: my_db
@@ -59,7 +59,7 @@ func TestParseFromYamlMssql(t *testing.T) {
in: `
sources:
my-mssql-instance:
kind: mssql
type: mssql
host: 0.0.0.0
port: my-port
database: my_db
@@ -109,7 +109,7 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
my-mssql-instance:
kind: mssql
type: mssql
host: 0.0.0.0
port: my-port
database: my_db
@@ -117,14 +117,14 @@ func TestFailParseFromYaml(t *testing.T) {
password: my_pass
foo: bar
`,
err: "unable to parse source \"my-mssql-instance\" as \"mssql\": [2:1] unknown field \"foo\"\n 1 | database: my_db\n> 2 | foo: bar\n ^\n 3 | host: 0.0.0.0\n 4 | kind: mssql\n 5 | password: my_pass\n 6 | ",
err: "unable to parse source \"my-mssql-instance\" as \"mssql\": [2:1] unknown field \"foo\"\n 1 | database: my_db\n> 2 | foo: bar\n ^\n 3 | host: 0.0.0.0\n 4 | type: mssql\n 5 | password: my_pass\n 6 | ",
},
{
desc: "missing required field",
in: `
sources:
my-mssql-instance:
kind: mssql
type: mssql
host: 0.0.0.0
port: my-port
database: my_db

View File

@@ -48,7 +48,7 @@ func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (sources
type Config struct {
Name string `yaml:"name" validate:"required"`
Type string `yaml:"kind" validate:"required"`
Type string `yaml:"type" validate:"required"`
Host string `yaml:"host" validate:"required"`
Port string `yaml:"port" validate:"required"`
User string `yaml:"user" validate:"required"`

View File

@@ -40,7 +40,7 @@ func TestParseFromYamlCloudSQLMySQL(t *testing.T) {
in: `
sources:
my-mysql-instance:
kind: mysql
type: mysql
host: 0.0.0.0
port: my-port
database: my_db
@@ -64,7 +64,7 @@ func TestParseFromYamlCloudSQLMySQL(t *testing.T) {
in: `
sources:
my-mysql-instance:
kind: mysql
type: mysql
host: 0.0.0.0
port: my-port
database: my_db
@@ -90,7 +90,7 @@ func TestParseFromYamlCloudSQLMySQL(t *testing.T) {
in: `
sources:
my-mysql-instance:
kind: mysql
type: mysql
host: 0.0.0.0
port: my-port
database: my_db
@@ -147,7 +147,7 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
my-mysql-instance:
kind: mysql
type: mysql
host: 0.0.0.0
port: my-port
database: my_db
@@ -162,7 +162,7 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
my-mysql-instance:
kind: mysql
type: mysql
port: my-port
database: my_db
user: my_user
@@ -175,7 +175,7 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
my-mysql-instance:
kind: mysql
type: mysql
host: 0.0.0.0
port: 3306
database: my_db

View File

@@ -45,7 +45,7 @@ func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (sources
type Config struct {
Name string `yaml:"name" validate:"required"`
Type string `yaml:"kind" validate:"required"`
Type string `yaml:"type" validate:"required"`
Uri string `yaml:"uri" validate:"required"`
User string `yaml:"user" validate:"required"`
Password string `yaml:"password" validate:"required"`

View File

@@ -35,7 +35,7 @@ func TestParseFromYamlNeo4j(t *testing.T) {
in: `
sources:
my-neo4j-instance:
kind: neo4j
type: neo4j
uri: neo4j+s://my-host:7687
database: my_db
user: my_user
@@ -82,21 +82,21 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
my-neo4j-instance:
kind: neo4j
type: neo4j
uri: neo4j+s://my-host:7687
database: my_db
user: my_user
password: my_pass
foo: bar
`,
err: "unable to parse source \"my-neo4j-instance\" as \"neo4j\": [2:1] unknown field \"foo\"\n 1 | database: my_db\n> 2 | foo: bar\n ^\n 3 | kind: neo4j\n 4 | password: my_pass\n 5 | uri: neo4j+s://my-host:7687\n 6 | ",
err: "unable to parse source \"my-neo4j-instance\" as \"neo4j\": [2:1] unknown field \"foo\"\n 1 | database: my_db\n> 2 | foo: bar\n ^\n 3 | type: neo4j\n 4 | password: my_pass\n 5 | uri: neo4j+s://my-host:7687\n 6 | ",
},
{
desc: "missing required field",
in: `
sources:
my-neo4j-instance:
kind: neo4j
type: neo4j
uri: neo4j+s://my-host:7687
database: my_db
user: my_user

View File

@@ -47,7 +47,7 @@ func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (sources
type Config struct {
Name string `yaml:"name" validate:"required"`
Type string `yaml:"kind" validate:"required"`
Type string `yaml:"type" validate:"required"`
Host string `yaml:"host" validate:"required"`
Port string `yaml:"port" validate:"required"`
User string `yaml:"user" validate:"required"`

View File

@@ -36,7 +36,7 @@ func TestParseFromYamlOceanBase(t *testing.T) {
in: `
sources:
my-oceanbase-instance:
kind: oceanbase
type: oceanbase
host: 0.0.0.0
port: 2881
database: ob_db
@@ -60,7 +60,7 @@ func TestParseFromYamlOceanBase(t *testing.T) {
in: `
sources:
my-oceanbase-instance:
kind: oceanbase
type: oceanbase
host: 0.0.0.0
port: 2881
database: ob_db
@@ -111,7 +111,7 @@ func TestFailParseFromYamlOceanBase(t *testing.T) {
in: `
sources:
my-oceanbase-instance:
kind: oceanbase
type: oceanbase
host: 0.0.0.0
port: 2881
database: ob_db
@@ -119,14 +119,14 @@ func TestFailParseFromYamlOceanBase(t *testing.T) {
password: ob_pass
foo: bar
`,
err: "unable to parse source \"my-oceanbase-instance\" as \"oceanbase\": [2:1] unknown field \"foo\"\n 1 | database: ob_db\n> 2 | foo: bar\n ^\n 3 | host: 0.0.0.0\n 4 | kind: oceanbase\n 5 | password: ob_pass\n 6 | ",
err: "unable to parse source \"my-oceanbase-instance\" as \"oceanbase\": [2:1] unknown field \"foo\"\n 1 | database: ob_db\n> 2 | foo: bar\n ^\n 3 | host: 0.0.0.0\n 4 | type: oceanbase\n 5 | password: ob_pass\n 6 | ",
},
{
desc: "missing required field",
in: `
sources:
my-oceanbase-instance:
kind: oceanbase
type: oceanbase
port: 2881
database: ob_db
user: ob_user

View File

@@ -47,7 +47,7 @@ func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (sources
type Config struct {
Name string `yaml:"name" validate:"required"`
Type string `yaml:"kind" validate:"required"`
Type string `yaml:"type" validate:"required"`
Host string `yaml:"host" validate:"required"`
Port string `yaml:"port" validate:"required"`
User string `yaml:"user" validate:"required"`

View File

@@ -37,7 +37,7 @@ func TestParseFromYamlPostgres(t *testing.T) {
in: `
sources:
my-pg-instance:
kind: postgres
type: postgres
host: my-host
port: my-port
database: my_db
@@ -61,7 +61,7 @@ func TestParseFromYamlPostgres(t *testing.T) {
in: `
sources:
my-pg-instance:
kind: postgres
type: postgres
host: my-host
port: my-port
database: my_db
@@ -117,7 +117,7 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
my-pg-instance:
kind: postgres
type: postgres
host: my-host
port: my-port
database: my_db
@@ -125,14 +125,14 @@ func TestFailParseFromYaml(t *testing.T) {
password: my_pass
foo: bar
`,
err: "unable to parse source \"my-pg-instance\" as \"postgres\": [2:1] unknown field \"foo\"\n 1 | database: my_db\n> 2 | foo: bar\n ^\n 3 | host: my-host\n 4 | kind: postgres\n 5 | password: my_pass\n 6 | ",
err: "unable to parse source \"my-pg-instance\" as \"postgres\": [2:1] unknown field \"foo\"\n 1 | database: my_db\n> 2 | foo: bar\n ^\n 3 | host: my-host\n 4 | type: postgres\n 5 | password: my_pass\n 6 | ",
},
{
desc: "missing required field",
in: `
sources:
my-pg-instance:
kind: postgres
type: postgres
host: my-host
port: my-port
database: my_db

View File

@@ -45,7 +45,7 @@ func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (sources
type Config struct {
Name string `yaml:"name" validate:"required"`
Type string `yaml:"kind" validate:"required"`
Type string `yaml:"type" validate:"required"`
Address []string `yaml:"address" validate:"required"`
Username string `yaml:"username"`
Password string `yaml:"password"`

View File

@@ -36,7 +36,7 @@ func TestParseFromYamlRedis(t *testing.T) {
in: `
sources:
my-redis-instance:
kind: redis
type: redis
address:
- 127.0.0.1
`,
@@ -55,7 +55,7 @@ func TestParseFromYamlRedis(t *testing.T) {
in: `
sources:
my-redis-instance:
kind: redis
type: redis
address:
- 127.0.0.1
password: my-pass
@@ -105,7 +105,7 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
my-redis-instance:
kind: redis
type: redis
project: my-project
address:
- 127.0.0.1
@@ -119,7 +119,7 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
my-redis-instance:
kind: redis
type: redis
project: my-project
address:
- 127.0.0.1
@@ -133,7 +133,7 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
my-redis-instance:
kind: redis
type: redis
`,
err: "unable to parse source \"my-redis-instance\" as \"redis\": Key: 'Config.Address' Error:Field validation for 'Address' failed on the 'required' tag",
},

View File

@@ -46,7 +46,7 @@ func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (sources
type Config struct {
Name string `yaml:"name" validate:"required"`
Type string `yaml:"kind" validate:"required"`
Type string `yaml:"type" validate:"required"`
Project string `yaml:"project" validate:"required"`
Instance string `yaml:"instance" validate:"required"`
Dialect sources.Dialect `yaml:"dialect" validate:"required"`

View File

@@ -36,7 +36,7 @@ func TestParseFromYamlSpannerDb(t *testing.T) {
in: `
sources:
my-spanner-instance:
kind: spanner
type: spanner
project: my-project
instance: my-instance
database: my_db
@@ -57,7 +57,7 @@ func TestParseFromYamlSpannerDb(t *testing.T) {
in: `
sources:
my-spanner-instance:
kind: spanner
type: spanner
project: my-project
instance: my-instance
dialect: Googlesql
@@ -79,7 +79,7 @@ func TestParseFromYamlSpannerDb(t *testing.T) {
in: `
sources:
my-spanner-instance:
kind: spanner
type: spanner
project: my-project
instance: my-instance
dialect: postgresql
@@ -126,7 +126,7 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
my-spanner-instance:
kind: spanner
type: spanner
project: my-project
instance: my-instance
dialect: fail
@@ -139,20 +139,20 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
my-spanner-instance:
kind: spanner
type: spanner
project: my-project
instance: my-instance
database: my_db
foo: bar
`,
err: "unable to parse source \"my-spanner-instance\" as \"spanner\": [2:1] unknown field \"foo\"\n 1 | database: my_db\n> 2 | foo: bar\n ^\n 3 | instance: my-instance\n 4 | kind: spanner\n 5 | project: my-project",
err: "unable to parse source \"my-spanner-instance\" as \"spanner\": [2:1] unknown field \"foo\"\n 1 | database: my_db\n> 2 | foo: bar\n ^\n 3 | instance: my-instance\n 4 | type: spanner\n 5 | project: my-project",
},
{
desc: "missing required field",
in: `
sources:
my-spanner-instance:
kind: spanner
type: spanner
project: my-project
instance: my-instance
`,

View File

@@ -46,7 +46,7 @@ func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (sources
type Config struct {
Name string `yaml:"name" validate:"required"`
Type string `yaml:"kind" validate:"required"`
Type string `yaml:"type" validate:"required"`
Database string `yaml:"database" validate:"required"` // Path to SQLite database file
}

View File

@@ -36,7 +36,7 @@ func TestParseFromYamlSQLite(t *testing.T) {
in: `
sources:
my-sqlite-db:
kind: sqlite
type: sqlite
database: /path/to/database.db
`,
want: map[string]sources.SourceConfig{

View File

@@ -54,7 +54,7 @@ func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (sources
type Config struct {
Name string `yaml:"name" validate:"required"`
Type string `yaml:"kind" validate:"required"`
Type string `yaml:"type" validate:"required"`
Host string `yaml:"host" validate:"required"`
Port string `yaml:"port" validate:"required"`
User string `yaml:"user" validate:"required"`

View File

@@ -35,7 +35,7 @@ func TestParseFromYamlTiDB(t *testing.T) {
in: `
sources:
my-tidb-instance:
kind: tidb
type: tidb
host: 0.0.0.0
port: my-port
database: my_db
@@ -60,7 +60,7 @@ func TestParseFromYamlTiDB(t *testing.T) {
in: `
sources:
my-tidb-cloud:
kind: tidb
type: tidb
host: gateway01.us-west-2.prod.aws.tidbcloud.com
port: 4000
database: test_db
@@ -86,7 +86,7 @@ func TestParseFromYamlTiDB(t *testing.T) {
in: `
sources:
my-tidb-cloud:
kind: tidb
type: tidb
host: gateway01.us-west-2.prod.aws.tidbcloud.com
port: 4000
database: test_db
@@ -136,7 +136,7 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
my-tidb-instance:
kind: tidb
type: tidb
host: 0.0.0.0
port: my-port
database: my_db
@@ -145,14 +145,14 @@ func TestFailParseFromYaml(t *testing.T) {
ssl: false
foo: bar
`,
err: "unable to parse source \"my-tidb-instance\" as \"tidb\": [2:1] unknown field \"foo\"\n 1 | database: my_db\n> 2 | foo: bar\n ^\n 3 | host: 0.0.0.0\n 4 | kind: tidb\n 5 | password: my_pass\n 6 | ",
err: "unable to parse source \"my-tidb-instance\" as \"tidb\": [2:1] unknown field \"foo\"\n 1 | database: my_db\n> 2 | foo: bar\n ^\n 3 | host: 0.0.0.0\n 4 | type: tidb\n 5 | password: my_pass\n 6 | ",
},
{
desc: "missing required field",
in: `
sources:
my-tidb-instance:
kind: tidb
type: tidb
port: my-port
database: my_db
user: my_user

View File

@@ -48,7 +48,7 @@ func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (sources
type Config struct {
Name string `yaml:"name" validate:"required"`
Type string `yaml:"kind" validate:"required"`
Type string `yaml:"type" validate:"required"`
Host string `yaml:"host" validate:"required"`
Port string `yaml:"port" validate:"required"`
User string `yaml:"user"`

View File

@@ -140,7 +140,7 @@ func TestParseFromYamlTrino(t *testing.T) {
in: `
sources:
my-trino-instance:
kind: trino
type: trino
host: localhost
port: "8080"
user: testuser
@@ -164,7 +164,7 @@ func TestParseFromYamlTrino(t *testing.T) {
in: `
sources:
my-trino-instance:
kind: trino
type: trino
host: localhost
port: "8443"
user: testuser
@@ -198,7 +198,7 @@ func TestParseFromYamlTrino(t *testing.T) {
in: `
sources:
my-trino-anonymous:
kind: trino
type: trino
host: localhost
port: "8080"
catalog: hive

View File

@@ -45,7 +45,7 @@ func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (sources
type Config struct {
Name string `yaml:"name" validate:"required"`
Type string `yaml:"kind" validate:"required"`
Type string `yaml:"type" validate:"required"`
Address []string `yaml:"address" validate:"required"`
Username string `yaml:"username"`
Password string `yaml:"password"`

View File

@@ -37,7 +37,7 @@ func TestParseFromYamlValkey(t *testing.T) {
in: `
sources:
my-valkey-instance:
kind: valkey
type: valkey
address:
- 127.0.0.1
`,
@@ -59,7 +59,7 @@ func TestParseFromYamlValkey(t *testing.T) {
in: `
sources:
my-valkey-instance:
kind: valkey
type: valkey
address:
- 127.0.0.1
database: 1
@@ -111,7 +111,7 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
my-valkey-instance:
kind: valkey
type: valkey
project: my-project
address:
- 127.0.0.1
@@ -125,7 +125,7 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
my-valkey-instance:
kind: valkey
type: valkey
address:
- 127.0.0.1
project: proj
@@ -138,7 +138,7 @@ func TestFailParseFromYaml(t *testing.T) {
in: `
sources:
my-valkey-instance:
kind: valkey
type: valkey
`,
err: "unable to parse source \"my-valkey-instance\" as \"valkey\": Key: 'Config.Address' Error:Field validation for 'Address' failed on the 'required' tag",
},