github/genai-toolbox
1
0
Fork 0
mirror of https://github.com/googleapis/genai-toolbox.git synced 2026-01-10 07:58:12 -05:00
Code Issues Packages Projects Releases 14 Wiki Activity

feat: Support env replacement for tool.yaml (#462)

Browse Source 
Environment variable replacement is needed so that users don't have to
hardcode their secrets in configuring `tools.yaml`.
Both formats `$ENV_NAME` and `${ENV_NAME}` are standard ways to declare
an environment variable.
However, some database statement placeholders that are already using the
`$ENV_NAME` format.
Therefore, we only support env var declaration using `${ENV_NAME}` to
disambiguate it from other usages.

Fixes issue: https://github.com/googleapis/genai-toolbox/issues/431
This commit is contained in:
Wenxin Du
2025-04-23 07:33:02 -04:00
committed by GitHub
parent fc14cbfd07
commit eadb678a7b
21 changed files with 460 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
cmd/root.go
View File

@@ -21,6 +21,7 @@ import (
"io"
"os"
"os/signal"
"regexp"
"strings"
"syscall"
"time"
@@ -129,9 +130,31 @@ type ToolsFile struct {
Toolsets server.ToolsetConfigs `yaml:"toolsets"`
}
// parseEnv replaces environment variables ${ENV_NAME} with their values.
func parseEnv(input string) string {
re := regexp.MustCompile(`\$\{(\w+)\}`)
return re.ReplaceAllStringFunc(input, func(match string) string {
parts := re.FindStringSubmatch(match)
if len(parts) < 2 {
// technically shouldn't happen
return match
}
// extract the variable name
variableName := parts[1]
if value, found := os.LookupEnv(variableName); found {
return value
}
return match
})
}
// parseToolsFile parses the provided yaml into appropriate configs.
func parseToolsFile(ctx context.Context, raw []byte) (ToolsFile, error) {
var toolsFile ToolsFile
// Replace environment variables if found
raw = []byte(parseEnv(string(raw)))
// Parse contents
err := yaml.UnmarshalContext(ctx, raw, &toolsFile, yaml.Strict())
if err != nil {

166
cmd/root_test.go
View File

@@ -26,8 +26,10 @@ import (
"github.com/googleapis/genai-toolbox/internal/auth/google"
"github.com/googleapis/genai-toolbox/internal/server"
cloudsqlpgsrc "github.com/googleapis/genai-toolbox/internal/sources/cloudsqlpg"
httpsrc "github.com/googleapis/genai-toolbox/internal/sources/http"
"github.com/googleapis/genai-toolbox/internal/testutils"
"github.com/googleapis/genai-toolbox/internal/tools"
"github.com/googleapis/genai-toolbox/internal/tools/http"
"github.com/googleapis/genai-toolbox/internal/tools/postgressql"
"github.com/spf13/cobra"
)
@@ -588,3 +590,167 @@ func TestParseToolFileWithAuth(t *testing.T) {
}
}
func TestEnvVarReplacement(t *testing.T) {
ctx, err := testutils.ContextWithNewLogger()
os.Setenv("TestHeader", "ACTUAL_HEADER")
os.Setenv("API_KEY", "ACTUAL_API_KEY")
os.Setenv("clientId", "ACTUAL_CLIENT_ID")
os.Setenv("clientId2", "ACTUAL_CLIENT_ID_2")
os.Setenv("toolset_name", "ACTUAL_TOOLSET_NAME")
os.Setenv("cat_string", "cat")
os.Setenv("food_string", "food")
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
tcs := []struct {
description string
in string
wantToolsFile ToolsFile
}{
{
description: "file with env var example",
in: `
sources:
my-http-instance:
kind: http
baseUrl: http://test_server/
timeout: 10s
headers:
Authorization: ${TestHeader}
queryParams:
api-key: ${API_KEY}
authServices:
my-google-service:
kind: google
clientId: ${clientId}
other-google-service:
kind: google
clientId: ${clientId2}
tools:
example_tool:
kind: http
source: my-instance
method: GET
path: "search?name=alice&pet=${cat_string}"
description: some description
authRequired:
- my-google-auth-service
- other-auth-service
queryParams:
- name: country
type: string
description: some description
authServices:
- name: my-google-auth-service
field: user_id
- name: other-auth-service
field: user_id
requestBody: |
{
"age": {{.age}},
"city": "{{.city}}",
"food": "${food_string}",
"other": "$OTHER"
}
bodyParams:
- name: age
type: integer
description: age num
- name: city
type: string
description: city string
headers:
Authorization: API_KEY
Content-Type: application/json
headerParams:
- name: Language
type: string
description: language string
toolsets:
${toolset_name}:
- example_tool
`,
wantToolsFile: ToolsFile{
Sources: server.SourceConfigs{
"my-http-instance": httpsrc.Config{
Name: "my-http-instance",
Kind: httpsrc.SourceKind,
BaseURL: "http://test_server/",
Timeout: "10s",
DefaultHeaders: map[string]string{"Authorization": "ACTUAL_HEADER"},
QueryParams: map[string]string{"api-key": "ACTUAL_API_KEY"},
},
},
AuthServices: server.AuthServiceConfigs{
"my-google-service": google.Config{
Name: "my-google-service",
Kind: google.AuthServiceKind,
ClientID: "ACTUAL_CLIENT_ID",
},
"other-google-service": google.Config{
Name: "other-google-service",
Kind: google.AuthServiceKind,
ClientID: "ACTUAL_CLIENT_ID_2",
},
},
Tools: server.ToolConfigs{
"example_tool": http.Config{
Name: "example_tool",
Kind: http.ToolKind,
Source: "my-instance",
Method: "GET",
Path: "search?name=alice&pet=cat",
Description: "some description",
AuthRequired: []string{"my-google-auth-service", "other-auth-service"},
QueryParams: []tools.Parameter{
tools.NewStringParameterWithAuth("country", "some description",
[]tools.ParamAuthService{{Name: "my-google-auth-service", Field: "user_id"},
{Name: "other-auth-service", Field: "user_id"}}),
},
RequestBody: `{
"age": {{.age}},
"city": "{{.city}}",
"food": "food",
"other": "$OTHER"
}
`,
BodyParams: []tools.Parameter{tools.NewIntParameter("age", "age num"), tools.NewStringParameter("city", "city string")},
Headers: map[string]string{"Authorization": "API_KEY", "Content-Type": "application/json"},
HeaderParams: []tools.Parameter{tools.NewStringParameter("Language", "language string")},
},
},
Toolsets: server.ToolsetConfigs{
"ACTUAL_TOOLSET_NAME": tools.ToolsetConfig{
Name: "ACTUAL_TOOLSET_NAME",
ToolNames: []string{"example_tool"},
},
},
},
},
}
for _, tc := range tcs {
t.Run(tc.description, func(t *testing.T) {
toolsFile, err := parseToolsFile(ctx, testutils.FormatYaml(tc.in))
if err != nil {
t.Fatalf("failed to parse input: %v", err)
}
if diff := cmp.Diff(tc.wantToolsFile.Sources, toolsFile.Sources); diff != "" {
t.Fatalf("incorrect sources parse: diff %v", diff)
}
if diff := cmp.Diff(tc.wantToolsFile.AuthServices, toolsFile.AuthServices); diff != "" {
t.Fatalf("incorrect authServices parse: diff %v", diff)
}
if diff := cmp.Diff(tc.wantToolsFile.Tools, toolsFile.Tools); diff != "" {
t.Fatalf("incorrect tools parse: diff %v", diff)
}
if diff := cmp.Diff(tc.wantToolsFile.Toolsets, toolsFile.Toolsets); diff != "" {
t.Fatalf("incorrect tools parse: diff %v", diff)
}
})
}
}

19
docs/en/getting-started/configure.md
View File

@@ -6,12 +6,22 @@ description: How to configure Toolbox's tools.yaml file.
---
The primary way to configure Toolbox is through the `tools.yaml` file. If you
have multiple files, you can tell toolbox which to load with the `--tools_file
have multiple files, you can tell toolbox which to load with the `--tools-file
tools.yaml` flag.
You can find more detailed reference documentation to all resource types in the
[Resources](../resources/).
### Using Environment Variables
To avoid hardcoding certain secret fields like passwords, usernames, API keys
etc., you could use environment variables instead with the format `${ENV_NAME}`.
```yaml
user: ${USER_NAME}
password: ${PASSWORD}
```
### Sources
The `sources` section of your `tools.yaml` defines what data sources your
@@ -25,8 +35,8 @@ sources:
host: 127.0.0.1
port: 5432
database: toolbox_db
user: toolbox_user
password: my-password
user: ${USER_NAME}
password: ${PASSWORD}
```
For more details on configuring different types of sources, see the
@@ -54,7 +64,6 @@ tools:
For more details on configuring different types of tools, see the
[Tools](../resources/tools/).
### Toolsets
The `toolsets` section of your `tools.yaml` allows you to define groups of tools
@@ -79,4 +88,4 @@ all_tools = client.load_toolset()
# This will only load the tools listed in 'my_second_toolset'
my_second_toolset = client.load_toolset("my_second_toolset")
```
```

104
docs/en/getting-started/local_quickstart.md
View File

@@ -12,7 +12,7 @@ description: >
## Before you begin
This guide assumes you have already done the following:
This guide assumes you have already done the following:
1. Installed [Python 3.9+][install-python] (including [pip][install-pip] and
your preferred virtual environment tool for managing dependencies e.g. [venv][install-venv])
@@ -26,7 +26,7 @@ This guide assumes you have already done the following:
## Step 1: Set up your database
In this section, we will create a database, insert some data that needs to be
access by our agent, and create a database user for Toolbox to connect with.
access by our agent, and create a database user for Toolbox to connect with.
1. Connect to postgres using the `psql` command:
@@ -38,9 +38,9 @@ access by our agent, and create a database user for Toolbox to connect with.
1. Create a new database and a new user:
{{< notice tip >}}
For a real application, it's best to follow the principle of least permission
and only grant the privileges your application needs.
{{< notice tip >}}
For a real application, it's best to follow the principle of least permission
and only grant the privileges your application needs.
{{< /notice >}}
```sql
@@ -52,8 +52,6 @@ access by our agent, and create a database user for Toolbox to connect with.
ALTER DATABASE toolbox_db OWNER TO toolbox_user;
```
1. End the database session:
```bash
@@ -103,7 +101,6 @@ access by our agent, and create a database user for Toolbox to connect with.
\q
```
## Step 2: Install and configure Toolbox
In this section, we will download Toolbox, configure our tools in a
@@ -111,10 +108,10 @@ In this section, we will download Toolbox, configure our tools in a
1. Download the latest version of Toolbox as a binary:
{{< notice tip >}}
Select the
[correct binary](https://github.com/googleapis/genai-toolbox/releases)
corresponding to your OS and CPU architecture.
{{< notice tip >}}
Select the
[correct binary](https://github.com/googleapis/genai-toolbox/releases)
corresponding to your OS and CPU architecture.
{{< /notice >}}
<!-- {x-release-please-start-version} -->
```bash
@@ -133,6 +130,11 @@ In this section, we will download Toolbox, configure our tools in a
such as `user`, `password`, or `database` that you may have customized in the
previous step.
{{< notice tip >}}
In practice, use environment variable replacement with the format ${ENV_NAME}
instead of hardcoding your secrets into the configuration file.
{{< /notice >}}
```yaml
sources:
my-pg-source:
@@ -140,8 +142,8 @@ In this section, we will download Toolbox, configure our tools in a
host: 127.0.0.1
port: 5432
database: toolbox_db
user: toolbox_user
password: my-password
user: ${USER_NAME}
password: ${PASSWORD}
tools:
search-hotels-by-name:
kind: postgres-sql
@@ -207,6 +209,7 @@ In this section, we will download Toolbox, configure our tools in a
- update-hotel
- cancel-hotel
```
For more info on tools, check out the `Resources` section of the docs.
1. Run the Toolbox server, pointing to the `tools.yaml` file created earlier:
@@ -220,14 +223,13 @@ In this section, we will download Toolbox, configure our tools in a
In this section, we will write and run an agent that will load the Tools
from Toolbox.
{{< notice tip>}} If you prefer to experiment within a Google Colab environment,
you can connect to a
[local runtime](https://research.google.com/colaboratory/local-runtimes.html).
{{< notice tip>}} If you prefer to experiment within a Google Colab environment,
you can connect to a
[local runtime](https://research.google.com/colaboratory/local-runtimes.html).
{{< /notice >}}
1. In a new terminal, install the SDK package.
{{< tabpane persist=header >}}
{{< tab header="Core" lang="bash" >}}
@@ -248,7 +250,7 @@ pip install toolbox-llamaindex
{{< /tabpane >}}
1. Install other required dependencies:
{{< tabpane persist=header >}}
{{< tab header="Core" lang="bash" >}}
@@ -261,18 +263,25 @@ pip install google-adk langchain
{{< tab header="Langchain" lang="bash" >}}
# TODO(developer): replace with correct package if needed
pip install langgraph langchain-google-vertexai
# pip install langchain-google-genai
# pip install langchain-anthropic
{{< /tab >}}
{{< tab header="LlamaIndex" lang="bash" >}}
# TODO(developer): replace with correct package if needed
pip install llama-index-llms-google-genai
# pip install llama-index-llms-anthropic
{{< /tab >}}
{{< /tabpane >}}
1. Create a new file named `hotel_agent.py` and copy the following
code to create an agent:
{{< tabpane persist=header >}}
@@ -308,9 +317,8 @@ queries = [
"My check in dates for my booking would be from April 10, 2024 to April 19, 2024.",
]
async def run_application():
toolbox_client = ToolboxClient("http://127.0.0.1:5000")
toolbox_client = ToolboxClient("<http://127.0.0.1:5000>")
# The toolbox_tools list contains Python callables (functions/methods) designed for LLM tool-use
# integration. While this example uses Google's genai client, these callables can be adapted for
@@ -385,7 +393,6 @@ async def run_application():
history.append(final_model_response_content)
print(response2.text)
asyncio.run(run_application())
{{< /tab >}}
@@ -398,17 +405,19 @@ from google.adk.artifacts.in_memory_artifact_service import InMemoryArtifactServ
from google.genai import types
import os
# TODO(developer): replace this with your Google API key
os.environ['GOOGLE_API_KEY'] = 'your-api-key'
toolbox_tools = ToolboxTool("http://127.0.0.1:5000")
toolbox_tools = ToolboxTool("<http://127.0.0.1:5000>")
prompt = """
You're a helpful hotel assistant. You handle hotel searching, booking and
cancellations. When the user searches for a hotel, mention it's name, id,
location and price tier. Always mention hotel ids while performing any
searches. This is very important for any operations. For any bookings or
cancellations, please provide the appropriate confirmation. Be sure to
cancellations. When the user searches for a hotel, mention it's name, id,
location and price tier. Always mention hotel ids while performing any
searches. This is very important for any operations. For any bookings or
cancellations, please provide the appropriate confirmation. Be sure to
update checkin or checkout dates if mentioned by the user.
Don't ask for confirmations from the user.
"""
@@ -458,20 +467,25 @@ for query in queries:
{{< tab header="LangChain" lang="python" >}}
from langgraph.prebuilt import create_react_agent
# TODO(developer): replace this with another import if needed
from langchain_google_vertexai import ChatVertexAI
# from langchain_google_genai import ChatGoogleGenerativeAI
# from langchain_anthropic import ChatAnthropic
from langgraph.checkpoint.memory import MemorySaver
from toolbox_langchain import ToolboxClient
prompt = """
You're a helpful hotel assistant. You handle hotel searching, booking and
cancellations. When the user searches for a hotel, mention it's name, id,
location and price tier. Always mention hotel ids while performing any
searches. This is very important for any operations. For any bookings or
cancellations, please provide the appropriate confirmation. Be sure to
cancellations. When the user searches for a hotel, mention it's name, id,
location and price tier. Always mention hotel ids while performing any
searches. This is very important for any operations. For any bookings or
cancellations, please provide the appropriate confirmation. Be sure to
update checkin or checkout dates if mentioned by the user.
Don't ask for confirmations from the user.
"""
@@ -488,7 +502,7 @@ async def run_application():
model = ChatVertexAI(model_name="gemini-1.5-pro")
# model = ChatGoogleGenerativeAI(model="gemini-1.5-pro")
# model = ChatAnthropic(model="claude-3-5-sonnet-20240620")
# Load the tools from the Toolbox server
client = ToolboxClient("http://127.0.0.1:5000")
tools = await client.aload_toolset()
@@ -511,18 +525,20 @@ from llama_index.core.agent.workflow import AgentWorkflow
from llama_index.core.workflow import Context
# TODO(developer): replace this with another import if needed
# TODO(developer): replace this with another import if needed
from llama_index.llms.google_genai import GoogleGenAI
# from llama_index.llms.anthropic import Anthropic
from toolbox_llamaindex import ToolboxClient
prompt = """
You're a helpful hotel assistant. You handle hotel searching, booking and
cancellations. When the user searches for a hotel, mention it's name, id,
location and price tier. Always mention hotel ids while performing any
searches. This is very important for any operations. For any bookings or
cancellations, please provide the appropriate confirmation. Be sure to
cancellations. When the user searches for a hotel, mention it's name, id,
location and price tier. Always mention hotel ids while performing any
searches. This is very important for any operations. For any bookings or
cancellations, please provide the appropriate confirmation. Be sure to
update checkin or checkout dates if mentioned by the user.
Don't ask for confirmations from the user.
"""
@@ -548,7 +564,7 @@ async def run_application():
# model="claude-3-7-sonnet-latest",
# api_key=os.getenv("ANTHROPIC_API_KEY")
# )
# Load the tools from the Toolbox server
client = ToolboxClient("http://127.0.0.1:5000")
tools = await client.aload_toolset()
@@ -567,10 +583,10 @@ async def run_application():
asyncio.run(run_application())
{{< /tab >}}
{{< /tabpane >}}
{{< tabpane text=true persist=header >}}
{{% tab header="Core" lang="en" %}}
To learn more about tool calling with Google GenAI, check out the
To learn more about tool calling with Google GenAI, check out the
[Google GenAI Documentation](https://github.com/googleapis/python-genai?tab=readme-ov-file#manually-declare-and-invoke-a-function-for-function-calling).
{{% /tab %}}
{{% tab header="ADK" lang="en" %}}
@@ -580,11 +596,13 @@ To learn more about Agent Development Kit, check out the [ADK documentation.](ht
To learn more about Agents in LangChain, check out the [LangGraph Agent documentation.](https://langchain-ai.github.io/langgraph/reference/prebuilt/#langgraph.prebuilt.chat_agent_executor.create_react_agent)
{{% /tab %}}
{{% tab header="LlamaIndex" lang="en" %}}
To learn more about Agents in LlamaIndex, check out the
To learn more about Agents in LlamaIndex, check out the
[LlamaIndex AgentWorkflow documentation.](https://docs.llamaindex.ai/en/stable/examples/agent/agent_workflow_basic/)
{{% /tab %}}
{{< /tabpane >}}
1. Run your agent, and observe the results:
```sh
python hotel_agent.py
```

25
docs/en/getting-started/mcp_quickstart/_index.md
View File

@@ -7,6 +7,7 @@ description: >
---
## Overview
[Model Context Protocol](https://modelcontextprotocol.io) is an open protocol
that standardizes how applications provide context to LLMs. Check out this page
on how to [connect to Toolbox via MCP](../../how-to/connect_via_mcp.md).
@@ -14,7 +15,7 @@ on how to [connect to Toolbox via MCP](../../how-to/connect_via_mcp.md).
## Step 1: Set up your database
In this section, we will create a database, insert some data that needs to be
access by our agent, and create a database user for Toolbox to connect with.
access by our agent, and create a database user for Toolbox to connect with.
1. Connect to postgres using the `psql` command:
@@ -26,9 +27,9 @@ access by our agent, and create a database user for Toolbox to connect with.
1. Create a new database and a new user:
{{< notice tip >}}
For a real application, it's best to follow the principle of least permission
and only grant the privileges your application needs.
{{< notice tip >}}
For a real application, it's best to follow the principle of least permission
and only grant the privileges your application needs.
{{< /notice >}}
```sql
@@ -40,8 +41,6 @@ access by our agent, and create a database user for Toolbox to connect with.
ALTER DATABASE toolbox_db OWNER TO toolbox_user;
```
1. End the database session:
```bash
@@ -98,10 +97,10 @@ In this section, we will download Toolbox, configure our tools in a
1. Download the latest version of Toolbox as a binary:
{{< notice tip >}}
Select the
[correct binary](https://github.com/googleapis/genai-toolbox/releases)
corresponding to your OS and CPU architecture.
{{< notice tip >}}
Select the
[correct binary](https://github.com/googleapis/genai-toolbox/releases)
corresponding to your OS and CPU architecture.
{{< /notice >}}
<!-- {x-release-please-start-version} -->
```bash
@@ -120,6 +119,11 @@ In this section, we will download Toolbox, configure our tools in a
such as `user`, `password`, or `database` that you may have customized in the
previous step.
{{< notice tip >}}
In practice, use environment variable replacement with the format ${ENV_NAME}
instead of hardcoding your secrets into the configuration file.
{{< /notice >}}
```yaml
sources:
my-pg-source:
@@ -194,6 +198,7 @@ In this section, we will download Toolbox, configure our tools in a
- update-hotel
- cancel-hotel
```
For more info on tools, check out the [Tools](../../resources/tools/_index.md) section.
1. Run the Toolbox server, pointing to the `tools.yaml` file created earlier:

41
docs/en/resources/authServices/_index.md
View File

@@ -7,7 +7,7 @@ description: >
---
AuthServices represent services that handle authentication and authorization. It
can primarily be used by [Tools](../tools) in two different ways:
can primarily be used by [Tools](../tools) in two different ways:
- [**Authorized Invocation**][auth-invoke] is when a tool
to be validate by the auth service before the call can be invoked. Toolbox
@@ -35,29 +35,35 @@ If you are accessing Toolbox with multiple applications, each
authServices:
my_auth_app_1:
kind: google
clientId: YOUR_CLIENT_ID_1
clientId: ${YOUR_CLIENT_ID_1}
my_auth_app_2:
kind: google
clientId: YOUR_CLIENT_ID_2
clientId: ${YOUR_CLIENT_ID_2}
```
{{< notice tip >}}
Use environment variable replacement with the format ${ENV_NAME}
instead of hardcoding your secrets into the configuration file.
{{< /notice >}}
After you've configured an `authService` you'll, need to reference it in the
configuration for each tool that should use it:
- **Authorized Invocations** for authorizing a tool call, [use the
`requiredAuth` field in a tool config][auth-invoke]
- **Authenticated Parameters** for using the value from a ODIC claim, [use the
`authServices` field in a parameter config][auth-params]
## Specifying ID Tokens from Clients
After [configuring](#example) your `authServices` section, use a Toolbox SDK to
add your ID tokens to the header of a Tool invocation request. When specifying a
token you will provide a function (that returns an id). This function is called
when the tool is invoked. This allows you to cache and refresh the ID token as
needed.
needed.
### Specifying tokens during load
{{< tabpane persist=header >}}
{{< tab header="LangChain" lang="Python" >}}
async def get_auth_token():
@@ -65,10 +71,12 @@ async def get_auth_token():
# This example just returns a placeholder. Replace with your actual token retrieval.
return "YOUR_ID_TOKEN" # Placeholder
# for a single tool use:
# for a single tool use
authorized_tool = toolbox.load_tool("my-tool-name", auth_tokens={"my_auth": get_auth_token})
# for a toolset use:
# for a toolset use
authorized_tools = toolbox.load_toolset("my-toolset-name", auth_tokens={"my_auth": get_auth_token})
{{< /tab >}}
{{< tab header="Llamaindex" lang="Python" >}}
@@ -77,37 +85,46 @@ async def get_auth_token():
# This example just returns a placeholder. Replace with your actual token retrieval.
return "YOUR_ID_TOKEN" # Placeholder
# for a single tool use:
# for a single tool use
authorized_tool = toolbox.load_tool("my-tool-name", auth_tokens={"my_auth": get_auth_token})
# for a toolset use:
# for a toolset use
authorized_tools = toolbox.load_toolset("my-toolset-name", auth_tokens={"my_auth": get_auth_token})
{{< /tab >}}
{{< /tabpane >}}
### Specifying tokens for existing tools
{{< tabpane persist=header >}}
{{< tab header="LangChain" lang="Python" >}}
tools = toolbox.load_toolset()
# for a single token
auth_tools = [tool.add_auth_token("my_auth", get_auth_token) for tool in tools]
# OR, if multiple tokens are needed
authorized_tool = tools[0].add_auth_tokens({
"my_auth1": get_auth1_token,
"my_auth2": get_auth2_token,
})
})
{{< /tab >}}
{{< tab header="Llamaindex" lang="Python" >}}
tools = toolbox.load_toolset()
# for a single token
auth_tools = [tool.add_auth_token("my_auth", get_auth_token) for tool in tools]
# OR, if multiple tokens are needed
authorized_tool = tools[0].add_auth_tokens({
"my_auth1": get_auth1_token,
"my_auth2": get_auth2_token,
})
})
{{< /tab >}}
{{< /tabpane >}}

7
docs/en/resources/authServices/google.md
View File

@@ -43,9 +43,14 @@ id-token][provided-claims] can be used for the parameter.
authServices:
my-google-auth:
kind: google
clientId: YOUR_GOOGLE_CLIENT_ID
clientId: ${YOUR_GOOGLE_CLIENT_ID}
```
{{< notice tip >}}
Use environment variable replacement with the format ${ENV_NAME}
instead of hardcoding your secrets into the configuration file.
{{< /notice >}}
## Reference
| **field** | **type** | **required** | **description** |

13
docs/en/resources/sources/_index.md
View File

@@ -11,6 +11,11 @@ Sources as a map in the `sources` section of your `tools.yaml` file. Typically,
a source configuration will contain any information needed to connect with and
interact with the database.
{{< notice tip >}}
Use environment variable replacement with the format ${ENV_NAME}
instead of hardcoding your secrets into the configuration file.
{{< /notice >}}
```yaml
sources:
my-cloud-sql-source:
@@ -19,11 +24,11 @@ sources:
region: us-central1
instance: my-instance-name
database: my_db
user: my-user
password: my-password
user: ${USER_NAME}
password: ${PASSWORD}
```
In implementation, each source is a different connection pool or client that used
to connect to the database and execute the tool.
to connect to the database and execute the tool.
## Available Sources
## Available Sources

21
docs/en/resources/sources/alloydb-pg.md
View File

@@ -75,17 +75,22 @@ a PostgreSQL user][alloydb-users] to login to the database with.
```yaml
sources:
my-alloydb-pg-source:
kind: "alloydb-postgres"
project: "my-project-id"
region: "us-central1"
cluster: "my-cluster"
instance: "my-instance"
database: "my_db"
user: "my-user"
password: "my-password"
kind: alloydb-postgres
project: my-project-id
region: us-central1
cluster: my-cluster
instance: my-instance
database: my_db
user: ${USER_NAME}
password: ${PASSWORD}
# ipType: "public"
```
{{< notice tip >}}
Use environment variable replacement with the format ${ENV_NAME}
instead of hardcoding your secrets into the configuration file.
{{< /notice >}}
## Reference
| **field** | **type** | **required** | **description** |

38
docs/en/resources/sources/bigquery.md
View File

@@ -13,21 +13,21 @@ description: >
# BigQuery Source
[BigQuery][bigquery-docs] is Google Cloud's fully managed, petabyte-scale,
and cost-effective analytics data warehouse that lets you run analytics
over vast amounts of data in near real time. With BigQuery, there's no
infrastructure to set up or manage, letting you focus on finding meaningful
insights using GoogleSQL and taking advantage of flexible pricing models
[BigQuery][bigquery-docs] is Google Cloud's fully managed, petabyte-scale,
and cost-effective analytics data warehouse that lets you run analytics
over vast amounts of data in near real time. With BigQuery, there's no
infrastructure to set up or manage, letting you focus on finding meaningful
insights using GoogleSQL and taking advantage of flexible pricing models
across on-demand and flat-rate options.
If you are new to BigQuery, you can try to
If you are new to BigQuery, you can try to
[load and query data with the bq tool][bigquery-quickstart-cli].
BigQuery uses [GoogleSQL][bigquery-googlesql] for querying data. GoogleSQL
is an ANSI-compliant structured query language (SQL) that is also implemented
for other Google Cloud services. SQL queries are handled by cluster nodes
in the same way as NoSQL data requests. Therefore, the same best practices
apply when creating SQL queries to run against your BigQuery data, such as
BigQuery uses [GoogleSQL][bigquery-googlesql] for querying data. GoogleSQL
is an ANSI-compliant structured query language (SQL) that is also implemented
for other Google Cloud services. SQL queries are handled by cluster nodes
in the same way as NoSQL data requests. Therefore, the same best practices
apply when creating SQL queries to run against your BigQuery data, such as
avoiding full table scans or complex filters.
[bigquery-docs]: https://cloud.google.com/bigquery/docs
@@ -38,16 +38,16 @@ avoiding full table scans or complex filters.
### IAM Permissions
BigQuery uses [Identity and Access Management (IAM)][iam-overview] to control
user and group access to BigQuery resources like projects, datasets, and tables.
Toolbox will use your [Application Default Credentials (ADC)][adc] to authorize
BigQuery uses [Identity and Access Management (IAM)][iam-overview] to control
user and group access to BigQuery resources like projects, datasets, and tables.
Toolbox will use your [Application Default Credentials (ADC)][adc] to authorize
and authenticate when interacting with [BigQuery][bigquery-docs].
In addition to [setting the ADC for your server][set-adc], you need to ensure
the IAM identity has been given the correct IAM permissions for the queries
you intend to run. Common roles include `roles/bigquery.user` (which includes
permissions to run jobs and read data) or `roles/bigquery.dataViewer`. See
[Introduction to BigQuery IAM][grant-permissions] for more information on
In addition to [setting the ADC for your server][set-adc], you need to ensure
the IAM identity has been given the correct IAM permissions for the queries
you intend to run. Common roles include `roles/bigquery.user` (which includes
permissions to run jobs and read data) or `roles/bigquery.dataViewer`. See
[Introduction to BigQuery IAM][grant-permissions] for more information on
applying IAM permissions and roles to an identity.
[iam-overview]: https://cloud.google.com/bigquery/docs/access-control

7
docs/en/resources/sources/cloud-sql-mssql.md
View File

@@ -79,9 +79,16 @@ sources:
instance: my-instance
database: my_db
ipAddress: localhost
user: ${USER_NAME}
password: ${PASSWORD}
# ipType: private
```
{{< notice tip >}}
Use environment variable replacement with the format ${ENV_NAME}
instead of hardcoding your secrets into the configuration file.
{{< /notice >}}
## Reference
| **field** | **type** | **required** | **description** |

19
docs/en/resources/sources/cloud-sql-mysql.md
View File

@@ -74,16 +74,21 @@ a MySQL user][cloud-sql-users] to login to the database with.
```yaml
sources:
my-cloud-sql-mysql-source:
kind: "cloud-sql-mysql"
project: "my-project-id"
region: "us-central1"
instance: "my-instance"
database: "my_db"
user: "my-user"
password: "my-password"
kind: cloud-sql-mysql
project: my-project-id
region: us-central1
instance: my-instance
database: my_db
user: ${USER_NAME}
password: ${PASSWORD}
# ipType: "private"
```
{{< notice tip >}}
Use environment variable replacement with the format ${ENV_NAME}
instead of hardcoding your secrets into the configuration file.
{{< /notice >}}
## Reference
| **field** | **type** | **required** | **description** |

19
docs/en/resources/sources/cloud-sql-pg.md
View File

@@ -79,16 +79,21 @@ a PostgreSQL user][cloud-sql-users] to login to the database with.
```yaml
sources:
my-cloud-sql-pg-source:
kind: "cloud-sql-postgres"
project: "my-project-id"
region: "us-central1"
instance: "my-instance"
database: "my_db"
user: "my-user"
password: "my-password"
kind: cloud-sql-postgres
project: my-project-id
region: us-central1
instance: my-instance
database: my_db
user: ${USER_NAME}
password: ${PASSWORD}
# ipType: "private"
```
{{< notice tip >}}
Use environment variable replacement with the format ${ENV_NAME}
instead of hardcoding your secrets into the configuration file.
{{< /notice >}}
## Reference
| **field** | **type** | **required** | **description** |

21
docs/en/resources/sources/dgraph.md
View File

@@ -9,7 +9,7 @@ description: >
## About
[Dgraph][dgraph-docs] is an open-source graph database. It is designed for real-time workloads, horizontal scalability, and data flexibility. Implemented as a distributed system, Dgraph processes queries in parallel to deliver the fastest result.
[Dgraph][dgraph-docs] is an open-source graph database. It is designed for real-time workloads, horizontal scalability, and data flexibility. Implemented as a distributed system, Dgraph processes queries in parallel to deliver the fastest result.
This source can connect to either a self-managed Dgraph cluster or one hosted on
Dgraph Cloud. If you're new to Dgraph, the fastest way to get started is to
@@ -18,7 +18,7 @@ Dgraph Cloud. If you're new to Dgraph, the fastest way to get started is to
[dgraph-docs]: https://dgraph.io/docs
[dgraph-login]: https://cloud.dgraph.io/login
## Requirements
## Requirements
### Database User
@@ -34,20 +34,25 @@ and user credentials for that namespace.
```yaml
sources:
my-dgraph-source:
kind: "dgraph"
dgraphUrl: "https://xxxx.cloud.dgraph.io"
user: "groot"
password: "password"
apiKey: abc123
kind: dgraph
dgraphUrl: https://xxxx.cloud.dgraph.io
user: ${USER_NAME}
password: ${PASSWORD}
apiKey: ${API_KEY}
namespace : 0
```
{{< notice tip >}}
Use environment variable replacement with the format ${ENV_NAME}
instead of hardcoding your secrets into the configuration file.
{{< /notice >}}
## Reference
| **Field** | **Type** | **Required** | **Description** |
|-------------|:--------:|:------------:|--------------------------------------------------------------------------------------------------|
| kind | string | true | Must be "dgraph". |
| dgraphUrl | string | true | Connection URI (e.g. "https://xxx.cloud.dgraph.io", "https://localhost:8080"). |
| dgraphUrl | string | true | Connection URI (e.g. "<https://xxx.cloud.dgraph.io>", "<https://localhost:8080>"). |
| user | string | false | Name of the Dgraph user to connect as (e.g., "groot"). |
| password | string | false | Password of the Dgraph user (e.g., "password"). |
| apiKey | string | false | API key to connect to a Dgraph Cloud instance. |

7
docs/en/resources/sources/http.md
View File

@@ -22,13 +22,18 @@ sources:
baseUrl: https://api.example.com/data
timeout: 10s # default to 30s
headers:
Authorization: Bearer YOUR_API_TOKEN
Authorization: Bearer ${API_KEY}
Content-Type: application/json
queryParams:
param1: value1
param2: value2
```
{{< notice tip >}}
Use environment variable replacement with the format ${ENV_NAME}
instead of hardcoding your secrets into the configuration file.
{{< /notice >}}
## Reference
| **field** | **type** | **required** | **description** |

21
docs/en/resources/sources/mssql.md
View File

@@ -7,7 +7,7 @@ description: >
---
## About
## About
[SQL Server][mssql-docs] is a relational database management system (RDBMS)
developed by Microsoft that allows users to store, retrieve, and manage large
@@ -20,7 +20,7 @@ amount of data through a structured format.
### Database User
This source only uses standard authentication. You will need to [create a
SQL Server user][mssql-users] to login to the database with.
SQL Server user][mssql-users] to login to the database with.
[mssql-users]: https://learn.microsoft.com/en-us/sql/relational-databases/security/authentication-access/create-a-database-user?view=sql-server-ver16
@@ -29,14 +29,19 @@ SQL Server user][mssql-users] to login to the database with.
```yaml
sources:
my-mssql-source:
kind: "mssql"
host: "127.0.0.1"
port: "1433"
database: "my_db"
user: "my-user"
password: "my-password"
kind: mssql
host: 127.0.0.1
port: 1433
database: my_db
user: ${USER_NAME}
password: ${PASSWORD}
```
{{< notice tip >}}
Use environment variable replacement with the format ${ENV_NAME}
instead of hardcoding your secrets into the configuration file.
{{< /notice >}}
## Reference
| **field** | **type** | **required** | **description** |

21
docs/en/resources/sources/mysql.md
View File

@@ -11,7 +11,7 @@ description: >
[MySQL][mysql-docs] is a relational database management system (RDBMS) that
stores and manages data. It's a popular choice for developers because of its
reliability, performance, and ease of use.
reliability, performance, and ease of use.
[mysql-docs]: https://www.mysql.com/
@@ -20,7 +20,7 @@ reliability, performance, and ease of use.
### Database User
This source only uses standard authentication. You will need to [create a
MySQL user][mysql-users] to login to the database with.
MySQL user][mysql-users] to login to the database with.
[mysql-users]: https://dev.mysql.com/doc/refman/8.4/en/user-names.html
@@ -29,14 +29,19 @@ MySQL user][mysql-users] to login to the database with.
```yaml
sources:
my-mysql-source:
kind: "mysql"
host: "127.0.0.1"
port: "3306"
database: "my_db"
user: "my-user"
password: "my-password"
kind: mysql
host: 127.0.0.1
port: 3306
database: my_db
user: ${USER_NAME}
password: ${PASSWORD}
```
{{< notice tip >}}
Use environment variable replacement with the format ${ENV_NAME}
instead of hardcoding your secrets into the configuration file.
{{< /notice >}}
## Reference
| **field** | **type** | **required** | **description** |

19
docs/en/resources/sources/neo4j.md
View File

@@ -13,13 +13,13 @@ reliability, feature robustness, and performance.
[neo4j-docs]: https://neo4j.com/docs
## Requirements
## Requirements
### Database User
This source only uses standard authentication. You will need to [create a Neo4j
user][neo4j-users] to log in to the database with, or use the default `neo4j`
user if available.
user if available.
[neo4j-users]: https://neo4j.com/docs/operations-manual/current/authentication-authorization/manage-users/
@@ -28,13 +28,18 @@ user if available.
```yaml
sources:
my-neo4j-source:
kind: "neo4j"
uri: "neo4j+s://xxxx.databases.neo4j.io:7687"
user: "neo4j"
password: "my-password"
kind: neo4j
uri: neo4j+s://xxxx.databases.neo4j.io:7687
user: ${USER_NAME}
password: ${PASSWORD}
database: "neo4j"
```
{{< notice tip >}}
Use environment variable replacement with the format ${ENV_NAME}
instead of hardcoding your secrets into the configuration file.
{{< /notice >}}
## Reference
| **field** | **type** | **required** | **description** |
@@ -44,5 +49,3 @@ sources:
| user | string | true | Name of the Neo4j user to connect as (e.g. "neo4j"). |
| password | string | true | Password of the Neo4j user (e.g. "my-password"). |
| database | string | true | Name of the Neo4j database to connect to (e.g. "neo4j"). |

17
docs/en/resources/sources/postgres.md
View File

@@ -29,14 +29,19 @@ PostgreSQL user][pg-users] to login to the database with.
```yaml
sources:
my-pg-source:
kind: "postgres"
host: "127.0.0.1"
port: "5432"
database: "my_db"
user: "my-user"
password: "my-password"
kind: postgres
host: 127.0.0.1
port: 5432
database: my_db
user: ${USER_NAME}
password: ${PASSWORD}
```
{{< notice tip >}}
Use environment variable replacement with the format ${ENV_NAME}
instead of hardcoding your secrets into the configuration file.
{{< /notice >}}
## Reference
| **field** | **type** | **required** | **description** |

4
docs/en/resources/sources/spanner.md
View File

@@ -8,7 +8,7 @@ description: >
---
# Spanner Source
# Spanner Source
[Spanner][spanner-docs] is a fully managed, mission-critical database service
that brings together relational, graph, key-value, and search. It offers
@@ -23,7 +23,7 @@ the Google Cloud console][spanner-quickstart].
[spanner-quickstart]:
https://cloud.google.com/spanner/docs/create-query-database-console
## Requirements
## Requirements
### IAM Permissions

10
internal/tools/http/http_test.go
View File

@@ -60,8 +60,8 @@ func TestParseFromYamlHTTP(t *testing.T) {
field: user_id
requestBody: |
{
"age": {{.age}}
"city": "{{.city}}"
"age": {{.age}},
"city": "{{.city}}",
"food": {{.food}}
}
bodyParams:
@@ -94,8 +94,8 @@ func TestParseFromYamlHTTP(t *testing.T) {
{Name: "other-auth-service", Field: "user_id"}}),
},
RequestBody: `{
"age": {{.age}}
"city": "{{.city}}"
"age": {{.age}},
"city": "{{.city}}",
"food": {{.food}}
}
`,
@@ -159,7 +159,7 @@ func TestFailParseFromYamlHTTP(t *testing.T) {
requestBody: |
{
"age": {{.age}},
"city": "{{.city}}",
"city": "{{.city}}"
}
bodyParams:
- name: age