diff --git a/internal/tools/bigquery/bigquerycommon/conversion_test.go b/internal/tools/bigquery/bigquerycommon/conversion_test.go new file mode 100644 index 0000000000..c735d0ebe1 --- /dev/null +++ b/internal/tools/bigquery/bigquerycommon/conversion_test.go @@ -0,0 +1,123 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquerycommon + +import ( + "math/big" + "reflect" + "testing" +) + +func TestNormalizeValue(t *testing.T) { + tests := []struct { + name string + input any + expected any + }{ + { + name: "big.Rat 1/3 (NUMERIC scale 9)", + input: new(big.Rat).SetFrac64(1, 3), // 0.33333333333... + expected: "0.33333333333333333333333333333333333333", // FloatString(38) + }, + { + name: "big.Rat 19/2 (9.5)", + input: new(big.Rat).SetFrac64(19, 2), + expected: "9.5", + }, + { + name: "big.Rat 12341/10 (1234.1)", + input: new(big.Rat).SetFrac64(12341, 10), + expected: "1234.1", + }, + { + name: "big.Rat 10/1 (10)", + input: new(big.Rat).SetFrac64(10, 1), + expected: "10", + }, + { + name: "string", + input: "hello", + expected: "hello", + }, + { + name: "int", + input: 123, + expected: 123, + }, + { + name: "nested slice of big.Rat", + input: []any{ + new(big.Rat).SetFrac64(19, 2), + new(big.Rat).SetFrac64(1, 4), + }, + expected: []any{"9.5", "0.25"}, + }, + { + name: "nested map of big.Rat", + input: map[string]any{ + "val1": new(big.Rat).SetFrac64(19, 2), + "val2": new(big.Rat).SetFrac64(1, 2), + }, + expected: map[string]any{ + "val1": "9.5", + "val2": "0.5", + }, + }, + { + name: "complex nested structure", + input: map[string]any{ + "list": []any{ + map[string]any{ + "rat": new(big.Rat).SetFrac64(3, 2), + }, + }, + }, + expected: map[string]any{ + "list": []any{ + map[string]any{ + "rat": "1.5", + }, + }, + }, + }, + { + name: "slice of *big.Rat", + input: []*big.Rat{ + new(big.Rat).SetFrac64(19, 2), + new(big.Rat).SetFrac64(1, 4), + }, + expected: []any{"9.5", "0.25"}, + }, + { + name: "slice of strings", + input: []string{"a", "b"}, + expected: []any{"a", "b"}, + }, + { + name: "byte slice (BYTES)", + input: []byte("hello"), + expected: []byte("hello"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := NormalizeValue(tt.input) + if !reflect.DeepEqual(got, tt.expected) { + t.Errorf("NormalizeValue() = %v, want %v", got, tt.expected) + } + }) + } +} diff --git a/internal/tools/bigquery/bigquerycommon/util.go b/internal/tools/bigquery/bigquerycommon/util.go index 5486ac36ed..d9b6fd0283 100644 --- a/internal/tools/bigquery/bigquerycommon/util.go +++ b/internal/tools/bigquery/bigquerycommon/util.go @@ -17,6 +17,8 @@ package bigquerycommon import ( "context" "fmt" + "math/big" + "reflect" "sort" "strings" @@ -118,3 +120,54 @@ func InitializeDatasetParameters( return projectParam, datasetParam } + +// NormalizeValue converts BigQuery specific types to standard JSON-compatible types. +// Specifically, it handles *big.Rat (used for NUMERIC/BIGNUMERIC) by converting +// them to decimal strings with up to 38 digits of precision, trimming trailing zeros. +// It recursively handles slices (arrays) and maps (structs) using reflection. +func NormalizeValue(v any) any { + if v == nil { + return nil + } + + // Handle *big.Rat specifically. + if rat, ok := v.(*big.Rat); ok { + // Convert big.Rat to a decimal string. + // Use a precision of 38 digits (enough for BIGNUMERIC and NUMERIC) + // and trim trailing zeros to match BigQuery's behavior. + s := rat.FloatString(38) + if strings.Contains(s, ".") { + s = strings.TrimRight(s, "0") + s = strings.TrimRight(s, ".") + } + return s + } + + // Use reflection for slices and maps to handle various underlying types. + rv := reflect.ValueOf(v) + switch rv.Kind() { + case reflect.Slice, reflect.Array: + // Preserve []byte as is, so json.Marshal encodes it as Base64 string (BigQuery BYTES behavior). + if rv.Type().Elem().Kind() == reflect.Uint8 { + return v + } + newSlice := make([]any, rv.Len()) + for i := 0; i < rv.Len(); i++ { + newSlice[i] = NormalizeValue(rv.Index(i).Interface()) + } + return newSlice + case reflect.Map: + // Ensure keys are strings to produce a JSON-compatible map. + if rv.Type().Key().Kind() != reflect.String { + return v + } + newMap := make(map[string]any, rv.Len()) + iter := rv.MapRange() + for iter.Next() { + newMap[iter.Key().String()] = NormalizeValue(iter.Value().Interface()) + } + return newMap + } + + return v +} diff --git a/internal/tools/bigquery/bigqueryexecutesql/bigqueryexecutesql.go b/internal/tools/bigquery/bigqueryexecutesql/bigqueryexecutesql.go index fe5dcb5029..b3e4d04f16 100644 --- a/internal/tools/bigquery/bigqueryexecutesql/bigqueryexecutesql.go +++ b/internal/tools/bigquery/bigqueryexecutesql/bigqueryexecutesql.go @@ -337,7 +337,7 @@ func (t Tool) Invoke(ctx context.Context, resourceMgr tools.SourceProvider, para schema := it.Schema row := orderedmap.Row{} for i, field := range schema { - row.Add(field.Name, val[i]) + row.Add(field.Name, bqutil.NormalizeValue(val[i])) } out = append(out, row) } diff --git a/internal/tools/bigquery/bigquerysql/bigquerysql.go b/internal/tools/bigquery/bigquerysql/bigquerysql.go index ad4784cb63..5e941deb7a 100644 --- a/internal/tools/bigquery/bigquerysql/bigquerysql.go +++ b/internal/tools/bigquery/bigquerysql/bigquerysql.go @@ -274,7 +274,7 @@ func (t Tool) Invoke(ctx context.Context, resourceMgr tools.SourceProvider, para } vMap := make(map[string]any) for key, value := range row { - vMap[key] = value + vMap[key] = bqutil.NormalizeValue(value) } out = append(out, vMap) }