diff --git a/.env.example b/.env.example index 8bc4defd42..41df435b8f 100644 --- a/.env.example +++ b/.env.example @@ -37,6 +37,16 @@ OPENCLAW_GATEWAY_TOKEN=change-me-to-a-long-random-token # ANTHROPIC_API_KEY=sk-ant-... # GEMINI_API_KEY=... # OPENROUTER_API_KEY=sk-or-... +# OPENCLAW_LIVE_OPENAI_KEY=sk-... +# OPENCLAW_LIVE_ANTHROPIC_KEY=sk-ant-... +# OPENCLAW_LIVE_GEMINI_KEY=... +# OPENAI_API_KEY_1=... +# ANTHROPIC_API_KEY_1=... +# GEMINI_API_KEY_1=... +# GOOGLE_API_KEY=... +# OPENAI_API_KEYS=sk-1,sk-2 +# ANTHROPIC_API_KEYS=sk-ant-1,sk-ant-2 +# GEMINI_API_KEYS=key-1,key-2 # Optional additional providers # ZAI_API_KEY=... diff --git a/docs/concepts/model-providers.md b/docs/concepts/model-providers.md index 6c2c79d850..f59c34b496 100644 --- a/docs/concepts/model-providers.md +++ b/docs/concepts/model-providers.md @@ -17,6 +17,20 @@ For model selection rules, see [/concepts/models](/concepts/models). - If you set `agents.defaults.models`, it becomes the allowlist. - CLI helpers: `openclaw onboard`, `openclaw models list`, `openclaw models set `. +## API key rotation + +- Supports generic provider rotation for selected providers. +- Configure multiple keys via: + - `OPENCLAW_LIVE__KEY` (single live override, highest priority) + - `_API_KEYS` (comma or semicolon list) + - `_API_KEY` (primary key) + - `_API_KEY_*` (numbered list, e.g. `_API_KEY_1`) +- For Google providers, `GOOGLE_API_KEY` is also included as fallback. +- Key selection order preserves priority and deduplicates values. +- Requests are retried with the next key only on rate-limit responses (for example `429`, `rate_limit`, `quota`, `resource exhausted`). +- Non-rate-limit failures fail immediately; no key rotation is attempted. +- When all candidate keys fail, the final error is returned from the last attempt. + ## Built-in providers (pi-ai catalog) OpenClaw ships with the pi‑ai catalog. These providers require **no** @@ -26,6 +40,7 @@ OpenClaw ships with the pi‑ai catalog. These providers require **no** - Provider: `openai` - Auth: `OPENAI_API_KEY` +- Optional rotation: `OPENAI_API_KEYS`, `OPENAI_API_KEY_1`, `OPENAI_API_KEY_2`, plus `OPENCLAW_LIVE_OPENAI_KEY` (single override) - Example model: `openai/gpt-5.1-codex` - CLI: `openclaw onboard --auth-choice openai-api-key` @@ -39,6 +54,7 @@ OpenClaw ships with the pi‑ai catalog. These providers require **no** - Provider: `anthropic` - Auth: `ANTHROPIC_API_KEY` or `claude setup-token` +- Optional rotation: `ANTHROPIC_API_KEYS`, `ANTHROPIC_API_KEY_1`, `ANTHROPIC_API_KEY_2`, plus `OPENCLAW_LIVE_ANTHROPIC_KEY` (single override) - Example model: `anthropic/claude-opus-4-6` - CLI: `openclaw onboard --auth-choice token` (paste setup-token) or `openclaw models auth paste-token --provider anthropic` @@ -78,6 +94,7 @@ OpenClaw ships with the pi‑ai catalog. These providers require **no** - Provider: `google` - Auth: `GEMINI_API_KEY` +- Optional rotation: `GEMINI_API_KEYS`, `GEMINI_API_KEY_1`, `GEMINI_API_KEY_2`, `GOOGLE_API_KEY` fallback, and `OPENCLAW_LIVE_GEMINI_KEY` (single override) - Example model: `google/gemini-3-pro-preview` - CLI: `openclaw onboard --auth-choice gemini-api-key` diff --git a/docs/gateway/authentication.md b/docs/gateway/authentication.md index 9b616084c0..8dd18f8416 100644 --- a/docs/gateway/authentication.md +++ b/docs/gateway/authentication.md @@ -103,6 +103,23 @@ openclaw models status openclaw doctor ``` +## API key rotation behavior (gateway) + +Some providers support retrying a request with alternative keys when an API call +hits a provider rate limit. + +- Priority order: + - `OPENCLAW_LIVE__KEY` (single override) + - `_API_KEYS` + - `_API_KEY` + - `_API_KEY_*` +- Google providers also include `GOOGLE_API_KEY` as an additional fallback. +- The same key list is deduplicated before use. +- OpenClaw retries with the next key only for rate-limit errors (for example + `429`, `rate_limit`, `quota`, `resource exhausted`). +- Non-rate-limit errors are not retried with alternate keys. +- If all keys fail, the final error from the last attempt is returned. + ## Controlling which credential is used ### Per-session (chat command) diff --git a/docs/help/testing.md b/docs/help/testing.md index a0ab38f784..2baa1591df 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -91,7 +91,7 @@ Think of the suites as “increasing realism” (and increasing flakiness/cost): - Costs money / uses rate limits - Prefer running narrowed subsets instead of “everything” - Live runs will source `~/.profile` to pick up missing API keys - - Anthropic key rotation: set `OPENCLAW_LIVE_ANTHROPIC_KEYS="sk-...,sk-..."` (or `OPENCLAW_LIVE_ANTHROPIC_KEY=sk-...`) or multiple `ANTHROPIC_API_KEY*` vars; tests will retry on rate limits +- API key rotation (provider-specific): set `*_API_KEYS` with comma/semicolon format or `*_API_KEY_1`, `*_API_KEY_2` (for example `OPENAI_API_KEYS`, `ANTHROPIC_API_KEYS`, `GEMINI_API_KEYS`) or per-live override via `OPENCLAW_LIVE_*_KEY`; tests retry on rate limit responses. ## Which suite should I run? diff --git a/src/agents/api-key-rotation.ts b/src/agents/api-key-rotation.ts new file mode 100644 index 0000000000..a277da75d5 --- /dev/null +++ b/src/agents/api-key-rotation.ts @@ -0,0 +1,72 @@ +import { formatErrorMessage } from "../infra/errors.js"; +import { collectProviderApiKeys, isApiKeyRateLimitError } from "./live-auth-keys.js"; + +type ApiKeyRetryParams = { + apiKey: string; + error: unknown; + attempt: number; +}; + +type ExecuteWithApiKeyRotationOptions = { + provider: string; + apiKeys: string[]; + execute: (apiKey: string) => Promise; + shouldRetry?: (params: ApiKeyRetryParams & { message: string }) => boolean; + onRetry?: (params: ApiKeyRetryParams & { message: string }) => void; +}; + +function dedupeApiKeys(raw: string[]): string[] { + const seen = new Set(); + const keys: string[] = []; + for (const value of raw) { + const apiKey = value.trim(); + if (!apiKey || seen.has(apiKey)) { + continue; + } + seen.add(apiKey); + keys.push(apiKey); + } + return keys; +} + +export function collectProviderApiKeysForExecution(params: { + provider: string; + primaryApiKey?: string; +}): string[] { + const { primaryApiKey, provider } = params; + return dedupeApiKeys([primaryApiKey?.trim() ?? "", ...collectProviderApiKeys(provider)]); +} + +export async function executeWithApiKeyRotation( + params: ExecuteWithApiKeyRotationOptions, +): Promise { + const keys = dedupeApiKeys(params.apiKeys); + if (keys.length === 0) { + throw new Error(`No API keys configured for provider "${params.provider}".`); + } + + let lastError: unknown; + for (let attempt = 0; attempt < keys.length; attempt += 1) { + const apiKey = keys[attempt]; + try { + return await params.execute(apiKey); + } catch (error) { + lastError = error; + const message = formatErrorMessage(error); + const retryable = params.shouldRetry + ? params.shouldRetry({ apiKey, error, attempt, message }) + : isApiKeyRateLimitError(message); + + if (!retryable || attempt + 1 >= keys.length) { + break; + } + + params.onRetry?.({ apiKey, error, attempt, message }); + } + } + + if (lastError === undefined) { + throw new Error(`Failed to run API request for ${params.provider}.`); + } + throw lastError; +} diff --git a/src/agents/live-auth-keys.ts b/src/agents/live-auth-keys.ts index e272d4cf9f..7732053e79 100644 --- a/src/agents/live-auth-keys.ts +++ b/src/agents/live-auth-keys.ts @@ -1,4 +1,47 @@ +import { normalizeProviderId } from "./model-selection.js"; + const KEY_SPLIT_RE = /[\s,;]+/g; +const GOOGLE_LIVE_SINGLE_KEY = "OPENCLAW_LIVE_GEMINI_KEY"; + +const PROVIDER_PREFIX_OVERRIDES: Record = { + google: "GEMINI", + "google-vertex": "GEMINI", +}; + +type ProviderApiKeyConfig = { + liveSingle?: string; + listVar?: string; + primaryVar?: string; + prefixedVar?: string; + fallbackVars: string[]; +}; + +const PROVIDER_API_KEY_CONFIG: Record> = { + anthropic: { + liveSingle: "OPENCLAW_LIVE_ANTHROPIC_KEY", + listVar: "OPENCLAW_LIVE_ANTHROPIC_KEYS", + primaryVar: "ANTHROPIC_API_KEY", + prefixedVar: "ANTHROPIC_API_KEY_", + }, + google: { + liveSingle: GOOGLE_LIVE_SINGLE_KEY, + listVar: "GEMINI_API_KEYS", + primaryVar: "GEMINI_API_KEY", + prefixedVar: "GEMINI_API_KEY_", + }, + "google-vertex": { + liveSingle: GOOGLE_LIVE_SINGLE_KEY, + listVar: "GEMINI_API_KEYS", + primaryVar: "GEMINI_API_KEY", + prefixedVar: "GEMINI_API_KEY_", + }, + openai: { + liveSingle: "OPENCLAW_LIVE_OPENAI_KEY", + listVar: "OPENAI_API_KEYS", + primaryVar: "OPENAI_API_KEY", + prefixedVar: "OPENAI_API_KEY_", + }, +}; function parseKeyList(raw?: string | null): string[] { if (!raw) { @@ -25,17 +68,53 @@ function collectEnvPrefixedKeys(prefix: string): string[] { return keys; } -export function collectAnthropicApiKeys(): string[] { - const forcedSingle = process.env.OPENCLAW_LIVE_ANTHROPIC_KEY?.trim(); +function resolveProviderApiKeyConfig(provider: string): ProviderApiKeyConfig { + const normalized = normalizeProviderId(provider); + const custom = PROVIDER_API_KEY_CONFIG[normalized]; + const base = PROVIDER_PREFIX_OVERRIDES[normalized] ?? normalized.toUpperCase().replace(/-/g, "_"); + + const liveSingle = custom?.liveSingle ?? `OPENCLAW_LIVE_${base}_KEY`; + const listVar = custom?.listVar ?? `${base}_API_KEYS`; + const primaryVar = custom?.primaryVar ?? `${base}_API_KEY`; + const prefixedVar = custom?.prefixedVar ?? `${base}_API_KEY_`; + + if (normalized === "google" || normalized === "google-vertex") { + return { + liveSingle, + listVar, + primaryVar, + prefixedVar, + fallbackVars: ["GOOGLE_API_KEY"], + }; + } + + return { + liveSingle, + listVar, + primaryVar, + prefixedVar, + fallbackVars: [], + }; +} + +export function collectProviderApiKeys(provider: string): string[] { + const config = resolveProviderApiKeyConfig(provider); + + const forcedSingle = config.liveSingle ? process.env[config.liveSingle]?.trim() : undefined; if (forcedSingle) { return [forcedSingle]; } - const fromList = parseKeyList(process.env.OPENCLAW_LIVE_ANTHROPIC_KEYS); - const fromEnv = collectEnvPrefixedKeys("ANTHROPIC_API_KEY"); - const primary = process.env.ANTHROPIC_API_KEY?.trim(); + const fromList = parseKeyList(config.listVar ? process.env[config.listVar] : undefined); + const primary = config.primaryVar ? process.env[config.primaryVar]?.trim() : undefined; + const fromPrefixed = config.prefixedVar ? collectEnvPrefixedKeys(config.prefixedVar) : []; + + const fallback = config.fallbackVars + .map((envVar) => process.env[envVar]?.trim()) + .filter(Boolean) as string[]; const seen = new Set(); + const add = (value?: string) => { if (!value) { return; @@ -49,17 +128,26 @@ export function collectAnthropicApiKeys(): string[] { for (const value of fromList) { add(value); } - if (primary) { - add(primary); + add(primary); + for (const value of fromPrefixed) { + add(value); } - for (const value of fromEnv) { + for (const value of fallback) { add(value); } return Array.from(seen); } -export function isAnthropicRateLimitError(message: string): boolean { +export function collectAnthropicApiKeys(): string[] { + return collectProviderApiKeys("anthropic"); +} + +export function collectGeminiApiKeys(): string[] { + return collectProviderApiKeys("google"); +} + +export function isApiKeyRateLimitError(message: string): boolean { const lower = message.toLowerCase(); if (lower.includes("rate_limit")) { return true; @@ -70,9 +158,22 @@ export function isAnthropicRateLimitError(message: string): boolean { if (lower.includes("429")) { return true; } + if (lower.includes("quota exceeded") || lower.includes("quota_exceeded")) { + return true; + } + if (lower.includes("resource exhausted") || lower.includes("resource_exhausted")) { + return true; + } + if (lower.includes("too many requests")) { + return true; + } return false; } +export function isAnthropicRateLimitError(message: string): boolean { + return isApiKeyRateLimitError(message); +} + export function isAnthropicBillingError(message: string): boolean { const lower = message.toLowerCase(); if (lower.includes("credit balance")) { @@ -91,7 +192,7 @@ export function isAnthropicBillingError(message: string): boolean { return true; } if ( - /["']?(?:status|code)["']?\s*[:=]\s*402\b|\bhttp\s*402\b|\berror(?:\s+code)?\s*[:=]?\s*402\b|\b(?:got|returned|received)\s+(?:a\s+)?402\b|^\s*402\s+payment/i.test( + /["']?(?:status|code)["']?\s*[:=]\s*402\b|\bhttp\s*402\b|\berror(?:\s+code)?\s*[:=]?\s*402\b|\b(?:got|returned|received)\s+(?:a\s+)?402\b|^\s*402\spayment/i.test( lower, ) ) { diff --git a/src/media-understanding/runner.entries.ts b/src/media-understanding/runner.entries.ts index e1f74bef8b..1df6420167 100644 --- a/src/media-understanding/runner.entries.ts +++ b/src/media-understanding/runner.entries.ts @@ -14,6 +14,10 @@ import type { MediaUnderstandingOutput, MediaUnderstandingProvider, } from "./types.js"; +import { + collectProviderApiKeysForExecution, + executeWithApiKeyRotation, +} from "../agents/api-key-rotation.js"; import { requireApiKey, resolveApiKeyForProvider } from "../agents/model-auth.js"; import { applyTemplate } from "../auto-reply/templating.js"; import { logVerbose, shouldLogVerbose } from "../globals.js"; @@ -392,7 +396,10 @@ export async function runProviderEntry(params: { preferredProfile: entry.preferredProfile, agentDir: params.agentDir, }); - const apiKey = requireApiKey(auth, providerId); + const apiKeys = collectProviderApiKeysForExecution({ + provider: providerId, + primaryApiKey: requireApiKey(auth, providerId), + }); const providerConfig = cfg.models?.providers?.[providerId]; const baseUrl = entry.baseUrl ?? params.config?.baseUrl ?? providerConfig?.baseUrl; const mergedHeaders = { @@ -407,18 +414,23 @@ export async function runProviderEntry(params: { entry, }); const model = entry.model?.trim() || DEFAULT_AUDIO_MODELS[providerId] || entry.model; - const result = await provider.transcribeAudio({ - buffer: media.buffer, - fileName: media.fileName, - mime: media.mime, - apiKey, - baseUrl, - headers, - model, - language: entry.language ?? params.config?.language ?? cfg.tools?.media?.audio?.language, - prompt, - query: providerQuery, - timeoutMs, + const result = await executeWithApiKeyRotation({ + provider: providerId, + apiKeys, + execute: async (apiKey) => + provider.transcribeAudio({ + buffer: media.buffer, + fileName: media.fileName, + mime: media.mime, + apiKey, + baseUrl, + headers, + model, + language: entry.language ?? params.config?.language ?? cfg.tools?.media?.audio?.language, + prompt, + query: providerQuery, + timeoutMs, + }), }); return { kind: "audio.transcription", @@ -452,18 +464,26 @@ export async function runProviderEntry(params: { preferredProfile: entry.preferredProfile, agentDir: params.agentDir, }); - const apiKey = requireApiKey(auth, providerId); + const apiKeys = collectProviderApiKeysForExecution({ + provider: providerId, + primaryApiKey: requireApiKey(auth, providerId), + }); const providerConfig = cfg.models?.providers?.[providerId]; - const result = await provider.describeVideo({ - buffer: media.buffer, - fileName: media.fileName, - mime: media.mime, - apiKey, - baseUrl: providerConfig?.baseUrl, - headers: providerConfig?.headers, - model: entry.model, - prompt, - timeoutMs, + const result = await executeWithApiKeyRotation({ + provider: providerId, + apiKeys, + execute: (apiKey) => + provider.describeVideo({ + buffer: media.buffer, + fileName: media.fileName, + mime: media.mime, + apiKey, + baseUrl: providerConfig?.baseUrl, + headers: providerConfig?.headers, + model: entry.model, + prompt, + timeoutMs, + }), }); return { kind: "video.description", diff --git a/src/memory/embeddings-gemini.ts b/src/memory/embeddings-gemini.ts index 8c4f1a66fb..848c2e1186 100644 --- a/src/memory/embeddings-gemini.ts +++ b/src/memory/embeddings-gemini.ts @@ -1,4 +1,8 @@ import type { EmbeddingProvider, EmbeddingProviderOptions } from "./embeddings.js"; +import { + collectProviderApiKeysForExecution, + executeWithApiKeyRotation, +} from "../agents/api-key-rotation.js"; import { requireApiKey, resolveApiKeyForProvider } from "../agents/model-auth.js"; import { isTruthyEnvValue } from "../infra/env.js"; import { parseGeminiAuth } from "../infra/gemini-auth.js"; @@ -9,6 +13,7 @@ export type GeminiEmbeddingClient = { headers: Record; model: string; modelPath: string; + apiKeys: string[]; }; const DEFAULT_GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"; @@ -74,23 +79,40 @@ export async function createGeminiEmbeddingProvider( const embedUrl = `${baseUrl}/${client.modelPath}:embedContent`; const batchUrl = `${baseUrl}/${client.modelPath}:batchEmbedContents`; - const embedQuery = async (text: string): Promise => { - if (!text.trim()) { - return []; - } - const res = await fetch(embedUrl, { + const fetchWithGeminiAuth = async (apiKey: string, endpoint: string, body: unknown) => { + const authHeaders = parseGeminiAuth(apiKey); + const headers = { + ...authHeaders.headers, + ...client.headers, + }; + const res = await fetch(endpoint, { method: "POST", - headers: client.headers, - body: JSON.stringify({ - content: { parts: [{ text }] }, - taskType: "RETRIEVAL_QUERY", - }), + headers, + body: JSON.stringify(body), }); if (!res.ok) { const payload = await res.text(); throw new Error(`gemini embeddings failed: ${res.status} ${payload}`); } - const payload = (await res.json()) as { embedding?: { values?: number[] } }; + return (await res.json()) as { + embedding?: { values?: number[] }; + embeddings?: Array<{ values?: number[] }>; + }; + }; + + const embedQuery = async (text: string): Promise => { + if (!text.trim()) { + return []; + } + const payload = await executeWithApiKeyRotation({ + provider: "google", + apiKeys: client.apiKeys, + execute: (apiKey) => + fetchWithGeminiAuth(apiKey, embedUrl, { + content: { parts: [{ text }] }, + taskType: "RETRIEVAL_QUERY", + }), + }); return payload.embedding?.values ?? []; }; @@ -103,16 +125,14 @@ export async function createGeminiEmbeddingProvider( content: { parts: [{ text }] }, taskType: "RETRIEVAL_DOCUMENT", })); - const res = await fetch(batchUrl, { - method: "POST", - headers: client.headers, - body: JSON.stringify({ requests }), + const payload = await executeWithApiKeyRotation({ + provider: "google", + apiKeys: client.apiKeys, + execute: (apiKey) => + fetchWithGeminiAuth(apiKey, batchUrl, { + requests, + }), }); - if (!res.ok) { - const payload = await res.text(); - throw new Error(`gemini embeddings failed: ${res.status} ${payload}`); - } - const payload = (await res.json()) as { embeddings?: Array<{ values?: number[] }> }; const embeddings = Array.isArray(payload.embeddings) ? payload.embeddings : []; return texts.map((_, index) => embeddings[index]?.values ?? []); }; @@ -151,11 +171,13 @@ export async function resolveGeminiEmbeddingClient( const rawBaseUrl = remoteBaseUrl || providerConfig?.baseUrl?.trim() || DEFAULT_GEMINI_BASE_URL; const baseUrl = normalizeGeminiBaseUrl(rawBaseUrl); const headerOverrides = Object.assign({}, providerConfig?.headers, remote?.headers); - const authHeaders = parseGeminiAuth(apiKey); const headers: Record = { - ...authHeaders.headers, ...headerOverrides, }; + const apiKeys = collectProviderApiKeysForExecution({ + provider: "google", + primaryApiKey: apiKey, + }); const model = normalizeGeminiModel(options.model); const modelPath = buildGeminiModelPath(model); debugLog("memory embeddings: gemini client", { @@ -166,5 +188,5 @@ export async function resolveGeminiEmbeddingClient( embedEndpoint: `${baseUrl}/${modelPath}:embedContent`, batchEndpoint: `${baseUrl}/${modelPath}:batchEmbedContents`, }); - return { baseUrl, headers, model, modelPath }; + return { baseUrl, headers, model, modelPath, apiKeys }; }