mirror of
https://github.com/googleapis/genai-toolbox.git
synced 2026-01-09 07:28:05 -05:00
fix: format BigQuery numeric output as decimal strings (#2084)
## Description
> Should include a concise description of the changes (bug or feature),
it's
> impact, along with a summary of the solution
This change updates both `bigquery-sql` and `bigquery-execute-sql` tools
to format `NUMERIC` and `BIGNUMERIC` values as decimal strings (e.g.,
"9.5") instead of rational fractions (e.g., "19/2"). This ensures the
tools' output matches the BigQuery REST API JSON format.
Key changes:
- Added `NormalizeValue` function in
`internal/tools/bigquery/bigquerycommon` to handle `*big.Rat` conversion
with 38-digit precision and trailing zero trimming.
- Updated `bigquery-sql` and `bigquery-execute-sql` to use
`NormalizeValue`.
- Added comprehensive tests in
`internal/tools/bigquery/bigquerycommon/conversion_test.go`.
With these changes the formatting for NUMERIC and BIGNUMERIC is fixed.
**Before:**
```
[
{
"id": 3,
"numeric_value": "1"
},
{
"id": 2,
"numeric_value": "333333333/1000000000"
},
{
"id": 4,
"numeric_value": "12341/10"
},
{
"id": 1,
"numeric_value": "19/2"
}
]
```
**After:**
```
[
{
"id": 3,
"numeric_value": "1"
},
{
"id": 2,
"numeric_value": "0.333333333"
},
{
"id": 4,
"numeric_value": "1234.1"
},
{
"id": 1,
"numeric_value": "9.5"
}
]
```
## PR Checklist
> Thank you for opening a Pull Request! Before submitting your PR, there
are a
> few things you can do to make sure it goes smoothly:
- [x] Make sure you reviewed
[CONTRIBUTING.md](https://github.com/googleapis/genai-toolbox/blob/main/CONTRIBUTING.md)
- [ ] Make sure to open an issue as a
[bug/issue](https://github.com/googleapis/genai-toolbox/issues/new/choose)
before writing your code! That way we can discuss the change, evaluate
designs, and agree on the general idea
- [x] Ensure the tests and linter pass
- [x] Code coverage does not decrease (if any source code was changed)
- [ ] Appropriate docs were updated (if necessary)
- [ ] Make sure to add `!` if this involve a breaking change
🛠️ Fixes #1194
---------
Co-authored-by: google-labs-jules[bot] <161369871+google-labs-jules[bot]@users.noreply.github.com>
Co-authored-by: Averi Kitsch <akitsch@google.com>
This commit is contained in:
123
internal/tools/bigquery/bigquerycommon/conversion_test.go
Normal file
123
internal/tools/bigquery/bigquerycommon/conversion_test.go
Normal file
@@ -0,0 +1,123 @@
|
||||
// Copyright 2025 Google LLC
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package bigquerycommon
|
||||
|
||||
import (
|
||||
"math/big"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNormalizeValue(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input any
|
||||
expected any
|
||||
}{
|
||||
{
|
||||
name: "big.Rat 1/3 (NUMERIC scale 9)",
|
||||
input: new(big.Rat).SetFrac64(1, 3), // 0.33333333333...
|
||||
expected: "0.33333333333333333333333333333333333333", // FloatString(38)
|
||||
},
|
||||
{
|
||||
name: "big.Rat 19/2 (9.5)",
|
||||
input: new(big.Rat).SetFrac64(19, 2),
|
||||
expected: "9.5",
|
||||
},
|
||||
{
|
||||
name: "big.Rat 12341/10 (1234.1)",
|
||||
input: new(big.Rat).SetFrac64(12341, 10),
|
||||
expected: "1234.1",
|
||||
},
|
||||
{
|
||||
name: "big.Rat 10/1 (10)",
|
||||
input: new(big.Rat).SetFrac64(10, 1),
|
||||
expected: "10",
|
||||
},
|
||||
{
|
||||
name: "string",
|
||||
input: "hello",
|
||||
expected: "hello",
|
||||
},
|
||||
{
|
||||
name: "int",
|
||||
input: 123,
|
||||
expected: 123,
|
||||
},
|
||||
{
|
||||
name: "nested slice of big.Rat",
|
||||
input: []any{
|
||||
new(big.Rat).SetFrac64(19, 2),
|
||||
new(big.Rat).SetFrac64(1, 4),
|
||||
},
|
||||
expected: []any{"9.5", "0.25"},
|
||||
},
|
||||
{
|
||||
name: "nested map of big.Rat",
|
||||
input: map[string]any{
|
||||
"val1": new(big.Rat).SetFrac64(19, 2),
|
||||
"val2": new(big.Rat).SetFrac64(1, 2),
|
||||
},
|
||||
expected: map[string]any{
|
||||
"val1": "9.5",
|
||||
"val2": "0.5",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "complex nested structure",
|
||||
input: map[string]any{
|
||||
"list": []any{
|
||||
map[string]any{
|
||||
"rat": new(big.Rat).SetFrac64(3, 2),
|
||||
},
|
||||
},
|
||||
},
|
||||
expected: map[string]any{
|
||||
"list": []any{
|
||||
map[string]any{
|
||||
"rat": "1.5",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "slice of *big.Rat",
|
||||
input: []*big.Rat{
|
||||
new(big.Rat).SetFrac64(19, 2),
|
||||
new(big.Rat).SetFrac64(1, 4),
|
||||
},
|
||||
expected: []any{"9.5", "0.25"},
|
||||
},
|
||||
{
|
||||
name: "slice of strings",
|
||||
input: []string{"a", "b"},
|
||||
expected: []any{"a", "b"},
|
||||
},
|
||||
{
|
||||
name: "byte slice (BYTES)",
|
||||
input: []byte("hello"),
|
||||
expected: []byte("hello"),
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := NormalizeValue(tt.input)
|
||||
if !reflect.DeepEqual(got, tt.expected) {
|
||||
t.Errorf("NormalizeValue() = %v, want %v", got, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,8 @@ package bigquerycommon
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
@@ -118,3 +120,54 @@ func InitializeDatasetParameters(
|
||||
|
||||
return projectParam, datasetParam
|
||||
}
|
||||
|
||||
// NormalizeValue converts BigQuery specific types to standard JSON-compatible types.
|
||||
// Specifically, it handles *big.Rat (used for NUMERIC/BIGNUMERIC) by converting
|
||||
// them to decimal strings with up to 38 digits of precision, trimming trailing zeros.
|
||||
// It recursively handles slices (arrays) and maps (structs) using reflection.
|
||||
func NormalizeValue(v any) any {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Handle *big.Rat specifically.
|
||||
if rat, ok := v.(*big.Rat); ok {
|
||||
// Convert big.Rat to a decimal string.
|
||||
// Use a precision of 38 digits (enough for BIGNUMERIC and NUMERIC)
|
||||
// and trim trailing zeros to match BigQuery's behavior.
|
||||
s := rat.FloatString(38)
|
||||
if strings.Contains(s, ".") {
|
||||
s = strings.TrimRight(s, "0")
|
||||
s = strings.TrimRight(s, ".")
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Use reflection for slices and maps to handle various underlying types.
|
||||
rv := reflect.ValueOf(v)
|
||||
switch rv.Kind() {
|
||||
case reflect.Slice, reflect.Array:
|
||||
// Preserve []byte as is, so json.Marshal encodes it as Base64 string (BigQuery BYTES behavior).
|
||||
if rv.Type().Elem().Kind() == reflect.Uint8 {
|
||||
return v
|
||||
}
|
||||
newSlice := make([]any, rv.Len())
|
||||
for i := 0; i < rv.Len(); i++ {
|
||||
newSlice[i] = NormalizeValue(rv.Index(i).Interface())
|
||||
}
|
||||
return newSlice
|
||||
case reflect.Map:
|
||||
// Ensure keys are strings to produce a JSON-compatible map.
|
||||
if rv.Type().Key().Kind() != reflect.String {
|
||||
return v
|
||||
}
|
||||
newMap := make(map[string]any, rv.Len())
|
||||
iter := rv.MapRange()
|
||||
for iter.Next() {
|
||||
newMap[iter.Key().String()] = NormalizeValue(iter.Value().Interface())
|
||||
}
|
||||
return newMap
|
||||
}
|
||||
|
||||
return v
|
||||
}
|
||||
|
||||
@@ -337,7 +337,7 @@ func (t Tool) Invoke(ctx context.Context, resourceMgr tools.SourceProvider, para
|
||||
schema := it.Schema
|
||||
row := orderedmap.Row{}
|
||||
for i, field := range schema {
|
||||
row.Add(field.Name, val[i])
|
||||
row.Add(field.Name, bqutil.NormalizeValue(val[i]))
|
||||
}
|
||||
out = append(out, row)
|
||||
}
|
||||
|
||||
@@ -274,7 +274,7 @@ func (t Tool) Invoke(ctx context.Context, resourceMgr tools.SourceProvider, para
|
||||
}
|
||||
vMap := make(map[string]any)
|
||||
for key, value := range row {
|
||||
vMap[key] = value
|
||||
vMap[key] = bqutil.NormalizeValue(value)
|
||||
}
|
||||
out = append(out, vMap)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user