From 7e8d751d3f2b5025dd2cbc5f05a9490b0147a2bc Mon Sep 17 00:00:00 2001 From: Yuan <45984206+Yuan325@users.noreply.github.com> Date: Thu, 12 Dec 2024 13:56:04 -0800 Subject: [PATCH] ci(cloudsql-pg): add end to end integration test (#113) End to end integration test for cloudsql postgres. Include checks for one tool's get (manifest) and post (invoke) endpoint. Integration tests are excluded from regular unit tests. --- .github/sync-repo-settings.yaml | 2 +- .github/workflows/tests.yaml | 4 +- integration.cloudbuild.yaml | 31 ++-- internal/server/server.go | 10 +- tests/cloud_sql_pg_integration_test.go | 184 +++++++++++++++++++++- tests/common_test.go | 204 +++++++++++++++++++++++++ 6 files changed, 412 insertions(+), 23 deletions(-) create mode 100644 tests/common_test.go diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index 58170706da..4c9c23d31c 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -30,7 +30,7 @@ branchProtectionRules: - "conventionalcommits.org" - "header-check" # - Add required status checks like presubmit tests - - "integration tests (ubuntu-latest)" + - "unit tests (ubuntu-latest)" requiredApprovingReviewCount: 1 requiresCodeOwnerReviews: true requiresStrictStatusChecks: true diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 8ae96a05ce..bda4ac5987 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -28,7 +28,7 @@ jobs: integration: # run job on proper workflow event triggers (skip job for pull_request event from forks and only run pull_request_target for "tests: run" label) if: "${{ (github.event.action != 'labeled' && github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name) || github.event.label.name == 'tests: run' }}" - name: integration tests + name: unit tests runs-on: ${{ matrix.os }} strategy: matrix: @@ -76,4 +76,4 @@ jobs: run: go build -v ./... - name: Run tests - run: go test -race -v ./... + run: go test -race -tags="!integration" -v ./... diff --git a/integration.cloudbuild.yaml b/integration.cloudbuild.yaml index afe366070a..f3e2ecbc82 100644 --- a/integration.cloudbuild.yaml +++ b/integration.cloudbuild.yaml @@ -15,23 +15,30 @@ steps: - name: golang:1.22 entrypoint: /bin/bash + env: + - "CLOUD_SQL_POSTGRES_PROJECT=$PROJECT_ID" + - "CLOUD_SQL_POSTGRES_INSTANCE=$_CLOUD_SQL_POSTGRES_INSTANCE" + - "CLOUD_SQL_POSTGRES_DATABASE=$_DATABASE_NAME" + - "CLOUD_SQL_POSTGRES_REGION=$_CLOUD_SQL_POSTGRES_REGION" + secretEnv: ["CLOUD_SQL_POSTGRES_USER", "CLOUD_SQL_POSTGRES_PASS"] args: - -c - | - go test -race -v ./tests/... - env: - - "PROJECT_ID=$PROJECT_ID" - - "INSTANCE_ID=$_INSTANCE_ID" - - "DATABASE_ID=$_DATABASE_ID" - - "REGION=$_REGION" - secretEnv: ["cloud_sql_pg_user", "cloud_sql_pg_pass"] + go test -race -tags="integration" -v ./... availableSecrets: secretManager: - - versionName: projects/$PROJECT_ID/secrets/cloud_sql_pg_user/versions/1 - env: "cloud_sql_pg_user" - - versionName: projects/$PROJECT_ID/secrets/cloud_sql_pg_pass/versions/1 - env: "cloud_sql_pg_pass" + - versionName: projects/$PROJECT_ID/secrets/cloud_sql_pg_user/versions/latest + env: CLOUD_SQL_POSTGRES_USER + - versionName: projects/$PROJECT_ID/secrets/cloud_sql_pg_pass/versions/latest + env: CLOUD_SQL_POSTGRES_PASS options: - logging: CLOUD_LOGGING_ONLY \ No newline at end of file + logging: CLOUD_LOGGING_ONLY + dynamicSubstitutions: true + +substitutions: + _DATABASE_NAME: test_database + _CLOUD_SQL_POSTGRES_REGION: "us-central1" + _CLOUD_SQL_POSTGRES_INSTANCE: "cloud-sql-pg-testing" + diff --git a/internal/server/server.go b/internal/server/server.go index c400e53824..8a1adf747e 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -87,9 +87,9 @@ func NewServer(cfg ServerConfig, log logLib.Logger) (*Server, error) { } sourcesMap[name] = s } - log.Info(fmt.Sprintf("Initalized %d sources.", len(sourcesMap))) + log.Info(fmt.Sprintf("Initialized %d sources.", len(sourcesMap))) - // initalize and validate the tools + // initialize and validate the tools toolsMap := make(map[string]tools.Tool) for name, tc := range cfg.ToolConfigs { t, err := tc.Initialize(sourcesMap) @@ -98,7 +98,7 @@ func NewServer(cfg ServerConfig, log logLib.Logger) (*Server, error) { } toolsMap[name] = t } - log.Info(fmt.Sprintf("Initalized %d tools.", len(toolsMap))) + log.Info(fmt.Sprintf("Initialized %d tools.", len(toolsMap))) // create a default toolset that contains all tools allToolNames := make([]string, 0, len(toolsMap)) @@ -109,7 +109,7 @@ func NewServer(cfg ServerConfig, log logLib.Logger) (*Server, error) { cfg.ToolsetConfigs = make(ToolsetConfigs) } cfg.ToolsetConfigs[""] = tools.ToolsetConfig{Name: "", ToolNames: allToolNames} - // initalize and validate the toolsets + // initialize and validate the toolsets toolsetsMap := make(map[string]tools.Toolset) for name, tc := range cfg.ToolsetConfigs { t, err := tc.Initialize(cfg.Version, toolsMap) @@ -118,7 +118,7 @@ func NewServer(cfg ServerConfig, log logLib.Logger) (*Server, error) { } toolsetsMap[name] = t } - log.Info(fmt.Sprintf("Initalized %d toolsets.", len(toolsetsMap))) + log.Info(fmt.Sprintf("Initialized %d toolsets.", len(toolsetsMap))) s := &Server{ conf: cfg, diff --git a/tests/cloud_sql_pg_integration_test.go b/tests/cloud_sql_pg_integration_test.go index e22c7204b5..84ddb4ac92 100644 --- a/tests/cloud_sql_pg_integration_test.go +++ b/tests/cloud_sql_pg_integration_test.go @@ -1,7 +1,185 @@ +//go:build integration + +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package tests -import "testing" +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "os" + "reflect" + "regexp" + "testing" + "time" +) -func TestHelloWorld(t *testing.T) { - t.Log("Hello World Test - Placeholder for Cloud SQL integration tests.") +var ( + CLOUD_SQL_POSTGRES_PROJECT = os.Getenv("CLOUD_SQL_POSTGRES_PROJECT") + CLOUD_SQL_POSTGRES_REGION = os.Getenv("CLOUD_SQL_POSTGRES_REGION") + CLOUD_SQL_POSTGRES_INSTANCE = os.Getenv("CLOUD_SQL_POSTGRES_INSTANCE") + CLOUD_SQL_POSTGRES_DATABASE = os.Getenv("CLOUD_SQL_POSTGRES_DATABASE") + CLOUD_SQL_POSTGRES_USER = os.Getenv("CLOUD_SQL_POSTGRES_USER") + CLOUD_SQL_POSTGRES_PASS = os.Getenv("CLOUD_SQL_POSTGRES_PASS") +) + +func requireCloudSQLPgVars(t *testing.T) { + switch "" { + case CLOUD_SQL_POSTGRES_PROJECT: + t.Fatal("'CLOUD_SQL_POSTGRES_PROJECT' not set") + case CLOUD_SQL_POSTGRES_REGION: + t.Fatal("'CLOUD_SQL_POSTGRES_REGION' not set") + case CLOUD_SQL_POSTGRES_INSTANCE: + t.Fatal("'CLOUD_SQL_POSTGRES_INSTANCE' not set") + case CLOUD_SQL_POSTGRES_DATABASE: + t.Fatal("'CLOUD_SQL_POSTGRES_DATABASE' not set") + case CLOUD_SQL_POSTGRES_USER: + t.Fatal("'CLOUD_SQL_POSTGRES_USER' not set") + case CLOUD_SQL_POSTGRES_PASS: + t.Fatal("'CLOUD_SQL_POSTGRES_PASS' not set") + } +} + +func TestCloudSQLPostgres(t *testing.T) { + requireCloudSQLPgVars(t) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + var args []string + + // Write config into a file and pass it to command + toolsFile := map[string]any{ + "sources": map[string]any{ + "my-pg-instance": map[string]any{ + "kind": "cloud-sql-postgres", + "project": CLOUD_SQL_POSTGRES_PROJECT, + "instance": CLOUD_SQL_POSTGRES_INSTANCE, + "region": CLOUD_SQL_POSTGRES_REGION, + "database": CLOUD_SQL_POSTGRES_DATABASE, + "user": CLOUD_SQL_POSTGRES_USER, + "password": CLOUD_SQL_POSTGRES_PASS, + }, + }, + "tools": map[string]any{ + "my-simple-tool": map[string]any{ + "kind": "postgres-sql", + "source": "my-pg-instance", + "description": "Simple tool to test end to end functionality.", + "statement": "SELECT 1;", + }, + }, + } + cmd, cleanup, err := StartCmd(ctx, toolsFile, args...) + if err != nil { + t.Fatalf("command initialization returned an error: %s", err) + } + defer cleanup() + + waitCtx, cancel := context.WithTimeout(ctx, 10*time.Minute) + defer cancel() + out, err := cmd.WaitForString(waitCtx, regexp.MustCompile(`INFO "Server ready to serve"`)) + if err != nil { + t.Logf("toolbox command logs: \n%s", out) + t.Fatalf("toolbox didn't start successfully: %s", err) + } + + // Test tool get endpoint + tcs := []struct { + name string + api string + want map[string]any + }{ + { + name: "get my-simple-tool", + api: "http://127.0.0.1:5000/api/tool/my-simple-tool/", + want: map[string]any{ + "my-simple-tool": map[string]any{ + "description": "Simple tool to test end to end functionality.", + "parameters": []any{}, + }, + }, + }, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + resp, err := http.Get(tc.api) + if err != nil { + t.Fatalf("error when sending a request: %s", err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatalf("response status code is not 200") + } + + var body map[string]interface{} + err = json.NewDecoder(resp.Body).Decode(&body) + if err != nil { + t.Fatalf("error parsing response body") + } + + got, ok := body["tools"] + if !ok { + t.Fatalf("unable to find tools in response body") + } + if !reflect.DeepEqual(got, tc.want) { + t.Fatalf("got %q, want %q", got, tc.want) + } + }) + } + + // Test tool invoke endpoint + invokeTcs := []struct { + name string + api string + requestBody io.Reader + want string + }{ + { + name: "invoke my-simple-tool", + api: "http://127.0.0.1:5000/api/tool/my-simple-tool/invoke", + requestBody: bytes.NewBuffer([]byte(`{}`)), + want: "Stub tool call for \"my-simple-tool\"! Parameters parsed: [] \n Output: [%!s(int32=1)]", + }, + } + for _, tc := range invokeTcs { + t.Run(tc.name, func(t *testing.T) { + resp, err := http.Post(tc.api, "application/json", tc.requestBody) + if err != nil { + t.Fatalf("error when sending a request: %s", err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatalf("response status code is not 200") + } + + var body map[string]interface{} + err = json.NewDecoder(resp.Body).Decode(&body) + if err != nil { + t.Fatalf("error parsing response body") + } + got, ok := body["result"].(string) + if !ok { + t.Fatalf("unable to find result in response body") + } + + if got != tc.want { + t.Fatalf("unexpected value: got %q, want %q", got, tc.want) + } + }) + } } diff --git a/tests/common_test.go b/tests/common_test.go new file mode 100644 index 0000000000..0d47216500 --- /dev/null +++ b/tests/common_test.go @@ -0,0 +1,204 @@ +//go:build integration + +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package tests contains end to end tests meant to verify the Toolbox Server +// works as expected when executed as a binary. + +package tests + +import ( + "bufio" + "context" + "fmt" + "io" + "os" + "regexp" + "strings" + + "gopkg.in/yaml.v3" + + "github.com/googleapis/genai-toolbox/cmd" +) + +// tmpFileWithCleanup creates a temporary file with the content and returns the path and +// a function to clean it up, or any errors encountered instead +func tmpFileWithCleanup(content []byte) (string, func(), error) { + // create a random file in the temp dir + f, err := os.CreateTemp("", "*") // * indicates random string + if err != nil { + return "", nil, err + } + cleanup := func() { os.Remove(f.Name()) } + + if _, err := f.Write(content); err != nil { + cleanup() + return "", nil, err + } + if err := f.Close(); err != nil { + cleanup() + return "", nil, err + } + return f.Name(), cleanup, err +} + +// CmdExec represents an invocation of a toolbox command. +type CmdExec struct { + Out io.ReadCloser + + cmd *cmd.Command + cancel context.CancelFunc + closers []io.Closer + done chan bool // closed once the cmd is completed + err error +} + +// StartCmd returns a CmdExec representing a running instance of a toolbox command. +func StartCmd(ctx context.Context, toolsFile map[string]any, args ...string) (*CmdExec, func(), error) { + b, err := yaml.Marshal(toolsFile) + if err != nil { + return nil, nil, fmt.Errorf("unable to marshal tools file: %s", err) + } + path, cleanup, err := tmpFileWithCleanup(b) + if err != nil { + return nil, nil, fmt.Errorf("unable to write tools file: %s", err) + } + args = append(args, "--tools_file", path) + + ctx, cancel := context.WithCancel(ctx) + // Open a pipe for tracking the output from the cmd + pr, pw, err := os.Pipe() + if err != nil { + cancel() + return nil, nil, fmt.Errorf("unable to open stdout pipe: %w", err) + } + + if err != nil { + cancel() + return nil, nil, fmt.Errorf("unable to initiate logger: %w", err) + } + c := cmd.NewCommand(cmd.WithStreams(pw, pw)) + c.SetArgs(args) + + t := &CmdExec{ + Out: pr, + cmd: c, + cancel: cancel, + closers: []io.Closer{pr, pw}, + done: make(chan bool), + } + + // Start the command in the background + go func() { + defer close(t.done) + defer cancel() + t.err = c.ExecuteContext(ctx) + }() + return t, cleanup, nil + +} + +// Stop sends the TERM signal to the cmd and returns. +func (c *CmdExec) Stop() { + c.cancel() +} + +// Waits until the execution is completed and returns any error from the result. +func (c *CmdExec) Wait(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-c.done: + return c.err + } +} + +// Done returns true if the command has exited. +func (c *CmdExec) Done() bool { + select { + case <-c.done: + return true + default: + } + return false +} + +// Close releases any resources associated with the instance. +func (c *CmdExec) Close() { + c.cancel() + for _, c := range c.closers { + c.Close() + } +} + +// WaitForString waits until the server logs a single line that matches the provided regex. +// returns the output of whatever the server sent so far. +func (c *CmdExec) WaitForString(ctx context.Context, re *regexp.Regexp) (string, error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + in := bufio.NewReader(c.Out) + + // read lines in background, sending result of each read over a channel + // this allows us to use in.ReadString without blocking + type result struct { + s string + err error + } + output := make(chan result) + go func() { + defer close(output) + for { + select { + case <-ctx.Done(): + // if the context is canceled, the orig thread will send back the error + // so we can just exit the goroutine here + return + default: + // otherwise read a line from the output + s, err := in.ReadString('\n') + if err != nil { + output <- result{err: err} + return + } + output <- result{s: s} + // if that last string matched, exit the goroutine + if re.MatchString(s) { + return + } + } + } + }() + + // collect the output until the ctx is canceled, an error was hit, + // or match was found (which is indicated the channel is closed) + var sb strings.Builder + for { + select { + case <-ctx.Done(): + // if ctx is done, return that error + return sb.String(), ctx.Err() + case o, ok := <-output: + if !ok { + // match was found! + return sb.String(), nil + } + if o.err != nil { + // error was found! + return sb.String(), o.err + } + sb.WriteString(o.s) + } + } +}