feat(agents): add generic provider api key rotation (#19587)

.env.example

@@ -37,6 +37,16 @@ OPENCLAW_GATEWAY_TOKEN=change-me-to-a-long-random-token
 # ANTHROPIC_API_KEY=sk-ant-...
 # GEMINI_API_KEY=...
 # OPENROUTER_API_KEY=sk-or-...
+# OPENCLAW_LIVE_OPENAI_KEY=sk-...
+# OPENCLAW_LIVE_ANTHROPIC_KEY=sk-ant-...
+# OPENCLAW_LIVE_GEMINI_KEY=...
+# OPENAI_API_KEY_1=...
+# ANTHROPIC_API_KEY_1=...
+# GEMINI_API_KEY_1=...
+# GOOGLE_API_KEY=...
+# OPENAI_API_KEYS=sk-1,sk-2
+# ANTHROPIC_API_KEYS=sk-ant-1,sk-ant-2
+# GEMINI_API_KEYS=key-1,key-2
 
 # Optional additional providers
 # ZAI_API_KEY=...

docs/concepts/model-providers.md

@@ -17,6 +17,20 @@ For model selection rules, see [/concepts/models](/concepts/models).
 - If you set `agents.defaults.models`, it becomes the allowlist.
 - CLI helpers: `openclaw onboard`, `openclaw models list`, `openclaw models set <provider/model>`.
 
+## API key rotation
+
+- Supports generic provider rotation for selected providers.
+- Configure multiple keys via:
+  - `OPENCLAW_LIVE_<PROVIDER>_KEY` (single live override, highest priority)
+  - `<PROVIDER>_API_KEYS` (comma or semicolon list)
+  - `<PROVIDER>_API_KEY` (primary key)
+  - `<PROVIDER>_API_KEY_*` (numbered list, e.g. `<PROVIDER>_API_KEY_1`)
+- For Google providers, `GOOGLE_API_KEY` is also included as fallback.
+- Key selection order preserves priority and deduplicates values.
+- Requests are retried with the next key only on rate-limit responses (for example `429`, `rate_limit`, `quota`, `resource exhausted`).
+- Non-rate-limit failures fail immediately; no key rotation is attempted.
+- When all candidate keys fail, the final error is returned from the last attempt.
+
 ## Built-in providers (pi-ai catalog)
 
 OpenClaw ships with the pi‑ai catalog. These providers require **no**
@@ -26,6 +40,7 @@ OpenClaw ships with the pi‑ai catalog. These providers require **no**
 
 - Provider: `openai`
 - Auth: `OPENAI_API_KEY`
+- Optional rotation: `OPENAI_API_KEYS`, `OPENAI_API_KEY_1`, `OPENAI_API_KEY_2`, plus `OPENCLAW_LIVE_OPENAI_KEY` (single override)
 - Example model: `openai/gpt-5.1-codex`
 - CLI: `openclaw onboard --auth-choice openai-api-key`
 
@@ -39,6 +54,7 @@ OpenClaw ships with the pi‑ai catalog. These providers require **no**
 
 - Provider: `anthropic`
 - Auth: `ANTHROPIC_API_KEY` or `claude setup-token`
+- Optional rotation: `ANTHROPIC_API_KEYS`, `ANTHROPIC_API_KEY_1`, `ANTHROPIC_API_KEY_2`, plus `OPENCLAW_LIVE_ANTHROPIC_KEY` (single override)
 - Example model: `anthropic/claude-opus-4-6`
 - CLI: `openclaw onboard --auth-choice token` (paste setup-token) or `openclaw models auth paste-token --provider anthropic`
 
@@ -78,6 +94,7 @@ OpenClaw ships with the pi‑ai catalog. These providers require **no**
 
 - Provider: `google`
 - Auth: `GEMINI_API_KEY`
+- Optional rotation: `GEMINI_API_KEYS`, `GEMINI_API_KEY_1`, `GEMINI_API_KEY_2`, `GOOGLE_API_KEY` fallback, and `OPENCLAW_LIVE_GEMINI_KEY` (single override)
 - Example model: `google/gemini-3-pro-preview`
 - CLI: `openclaw onboard --auth-choice gemini-api-key`
 

docs/gateway/authentication.md

@@ -103,6 +103,23 @@ openclaw models status
 openclaw doctor
 ```
 
+## API key rotation behavior (gateway)
+
+Some providers support retrying a request with alternative keys when an API call
+hits a provider rate limit.
+
+- Priority order:
+  - `OPENCLAW_LIVE_<PROVIDER>_KEY` (single override)
+  - `<PROVIDER>_API_KEYS`
+  - `<PROVIDER>_API_KEY`
+  - `<PROVIDER>_API_KEY_*`
+- Google providers also include `GOOGLE_API_KEY` as an additional fallback.
+- The same key list is deduplicated before use.
+- OpenClaw retries with the next key only for rate-limit errors (for example
+  `429`, `rate_limit`, `quota`, `resource exhausted`).
+- Non-rate-limit errors are not retried with alternate keys.
+- If all keys fail, the final error from the last attempt is returned.
+
 ## Controlling which credential is used
 
 ### Per-session (chat command)

docs/help/testing.md

@@ -91,7 +91,7 @@ Think of the suites as “increasing realism” (and increasing flakiness/cost):
   - Costs money / uses rate limits
   - Prefer running narrowed subsets instead of “everything”
   - Live runs will source `~/.profile` to pick up missing API keys
-  - Anthropic key rotation: set `OPENCLAW_LIVE_ANTHROPIC_KEYS="sk-...,sk-..."` (or `OPENCLAW_LIVE_ANTHROPIC_KEY=sk-...`) or multiple `ANTHROPIC_API_KEY*` vars; tests will retry on rate limits
+- API key rotation (provider-specific): set `*_API_KEYS` with comma/semicolon format or `*_API_KEY_1`, `*_API_KEY_2` (for example `OPENAI_API_KEYS`, `ANTHROPIC_API_KEYS`, `GEMINI_API_KEYS`) or per-live override via `OPENCLAW_LIVE_*_KEY`; tests retry on rate limit responses.
 
 ## Which suite should I run?
 

src/agents/api-key-rotation.ts

@@ -0,0 +1,72 @@
+import { formatErrorMessage } from "../infra/errors.js";
+import { collectProviderApiKeys, isApiKeyRateLimitError } from "./live-auth-keys.js";
+
+type ApiKeyRetryParams = {
+  apiKey: string;
+  error: unknown;
+  attempt: number;
+};
+
+type ExecuteWithApiKeyRotationOptions<T> = {
+  provider: string;
+  apiKeys: string[];
+  execute: (apiKey: string) => Promise<T>;
+  shouldRetry?: (params: ApiKeyRetryParams & { message: string }) => boolean;
+  onRetry?: (params: ApiKeyRetryParams & { message: string }) => void;
+};
+
+function dedupeApiKeys(raw: string[]): string[] {
+  const seen = new Set<string>();
+  const keys: string[] = [];
+  for (const value of raw) {
+    const apiKey = value.trim();
+    if (!apiKey || seen.has(apiKey)) {
+      continue;
+    }
+    seen.add(apiKey);
+    keys.push(apiKey);
+  }
+  return keys;
+}
+
+export function collectProviderApiKeysForExecution(params: {
+  provider: string;
+  primaryApiKey?: string;
+}): string[] {
+  const { primaryApiKey, provider } = params;
+  return dedupeApiKeys([primaryApiKey?.trim() ?? "", ...collectProviderApiKeys(provider)]);
+}
+
+export async function executeWithApiKeyRotation<T>(
+  params: ExecuteWithApiKeyRotationOptions<T>,
+): Promise<T> {
+  const keys = dedupeApiKeys(params.apiKeys);
+  if (keys.length === 0) {
+    throw new Error(`No API keys configured for provider "${params.provider}".`);
+  }
+
+  let lastError: unknown;
+  for (let attempt = 0; attempt < keys.length; attempt += 1) {
+    const apiKey = keys[attempt];
+    try {
+      return await params.execute(apiKey);
+    } catch (error) {
+      lastError = error;
+      const message = formatErrorMessage(error);
+      const retryable = params.shouldRetry
+        ? params.shouldRetry({ apiKey, error, attempt, message })
+        : isApiKeyRateLimitError(message);
+
+      if (!retryable || attempt + 1 >= keys.length) {
+        break;
+      }
+
+      params.onRetry?.({ apiKey, error, attempt, message });
+    }
+  }
+
+  if (lastError === undefined) {
+    throw new Error(`Failed to run API request for ${params.provider}.`);
+  }
+  throw lastError;
+}

src/agents/live-auth-keys.ts

@@ -1,4 +1,47 @@
+import { normalizeProviderId } from "./model-selection.js";
+
 const KEY_SPLIT_RE = /[\s,;]+/g;
+const GOOGLE_LIVE_SINGLE_KEY = "OPENCLAW_LIVE_GEMINI_KEY";
+
+const PROVIDER_PREFIX_OVERRIDES: Record<string, string> = {
+  google: "GEMINI",
+  "google-vertex": "GEMINI",
+};
+
+type ProviderApiKeyConfig = {
+  liveSingle?: string;
+  listVar?: string;
+  primaryVar?: string;
+  prefixedVar?: string;
+  fallbackVars: string[];
+};
+
+const PROVIDER_API_KEY_CONFIG: Record<string, Omit<ProviderApiKeyConfig, "fallbackVars">> = {
+  anthropic: {
+    liveSingle: "OPENCLAW_LIVE_ANTHROPIC_KEY",
+    listVar: "OPENCLAW_LIVE_ANTHROPIC_KEYS",
+    primaryVar: "ANTHROPIC_API_KEY",
+    prefixedVar: "ANTHROPIC_API_KEY_",
+  },
+  google: {
+    liveSingle: GOOGLE_LIVE_SINGLE_KEY,
+    listVar: "GEMINI_API_KEYS",
+    primaryVar: "GEMINI_API_KEY",
+    prefixedVar: "GEMINI_API_KEY_",
+  },
+  "google-vertex": {
+    liveSingle: GOOGLE_LIVE_SINGLE_KEY,
+    listVar: "GEMINI_API_KEYS",
+    primaryVar: "GEMINI_API_KEY",
+    prefixedVar: "GEMINI_API_KEY_",
+  },
+  openai: {
+    liveSingle: "OPENCLAW_LIVE_OPENAI_KEY",
+    listVar: "OPENAI_API_KEYS",
+    primaryVar: "OPENAI_API_KEY",
+    prefixedVar: "OPENAI_API_KEY_",
+  },
+};
 
 function parseKeyList(raw?: string | null): string[] {
   if (!raw) {
@@ -25,17 +68,53 @@ function collectEnvPrefixedKeys(prefix: string): string[] {
   return keys;
 }
 
-export function collectAnthropicApiKeys(): string[] {
-  const forcedSingle = process.env.OPENCLAW_LIVE_ANTHROPIC_KEY?.trim();
+function resolveProviderApiKeyConfig(provider: string): ProviderApiKeyConfig {
+  const normalized = normalizeProviderId(provider);
+  const custom = PROVIDER_API_KEY_CONFIG[normalized];
+  const base = PROVIDER_PREFIX_OVERRIDES[normalized] ?? normalized.toUpperCase().replace(/-/g, "_");
+
+  const liveSingle = custom?.liveSingle ?? `OPENCLAW_LIVE_${base}_KEY`;
+  const listVar = custom?.listVar ?? `${base}_API_KEYS`;
+  const primaryVar = custom?.primaryVar ?? `${base}_API_KEY`;
+  const prefixedVar = custom?.prefixedVar ?? `${base}_API_KEY_`;
+
+  if (normalized === "google" || normalized === "google-vertex") {
+    return {
+      liveSingle,
+      listVar,
+      primaryVar,
+      prefixedVar,
+      fallbackVars: ["GOOGLE_API_KEY"],
+    };
+  }
+
+  return {
+    liveSingle,
+    listVar,
+    primaryVar,
+    prefixedVar,
+    fallbackVars: [],
+  };
+}
+
+export function collectProviderApiKeys(provider: string): string[] {
+  const config = resolveProviderApiKeyConfig(provider);
+
+  const forcedSingle = config.liveSingle ? process.env[config.liveSingle]?.trim() : undefined;
   if (forcedSingle) {
     return [forcedSingle];
   }
 
-  const fromList = parseKeyList(process.env.OPENCLAW_LIVE_ANTHROPIC_KEYS);
-  const fromEnv = collectEnvPrefixedKeys("ANTHROPIC_API_KEY");
-  const primary = process.env.ANTHROPIC_API_KEY?.trim();
+  const fromList = parseKeyList(config.listVar ? process.env[config.listVar] : undefined);
+  const primary = config.primaryVar ? process.env[config.primaryVar]?.trim() : undefined;
+  const fromPrefixed = config.prefixedVar ? collectEnvPrefixedKeys(config.prefixedVar) : [];
+
+  const fallback = config.fallbackVars
+    .map((envVar) => process.env[envVar]?.trim())
+    .filter(Boolean) as string[];
 
   const seen = new Set<string>();
+
   const add = (value?: string) => {
     if (!value) {
       return;
@@ -49,17 +128,26 @@ export function collectAnthropicApiKeys(): string[] {
   for (const value of fromList) {
     add(value);
   }
-  if (primary) {
-    add(primary);
+  add(primary);
+  for (const value of fromPrefixed) {
+    add(value);
   }
-  for (const value of fromEnv) {
+  for (const value of fallback) {
     add(value);
   }
 
   return Array.from(seen);
 }
 
-export function isAnthropicRateLimitError(message: string): boolean {
+export function collectAnthropicApiKeys(): string[] {
+  return collectProviderApiKeys("anthropic");
+}
+
+export function collectGeminiApiKeys(): string[] {
+  return collectProviderApiKeys("google");
+}
+
+export function isApiKeyRateLimitError(message: string): boolean {
   const lower = message.toLowerCase();
   if (lower.includes("rate_limit")) {
     return true;
@@ -70,9 +158,22 @@ export function isAnthropicRateLimitError(message: string): boolean {
   if (lower.includes("429")) {
     return true;
   }
+  if (lower.includes("quota exceeded") || lower.includes("quota_exceeded")) {
+    return true;
+  }
+  if (lower.includes("resource exhausted") || lower.includes("resource_exhausted")) {
+    return true;
+  }
+  if (lower.includes("too many requests")) {
+    return true;
+  }
   return false;
 }
 
+export function isAnthropicRateLimitError(message: string): boolean {
+  return isApiKeyRateLimitError(message);
+}
+
 export function isAnthropicBillingError(message: string): boolean {
   const lower = message.toLowerCase();
   if (lower.includes("credit balance")) {
@@ -91,7 +192,7 @@ export function isAnthropicBillingError(message: string): boolean {
     return true;
   }
   if (
-    /["']?(?:status|code)["']?\s*[:=]\s*402\b|\bhttp\s*402\b|\berror(?:\s+code)?\s*[:=]?\s*402\b|\b(?:got|returned|received)\s+(?:a\s+)?402\b|^\s*402\s+payment/i.test(
+    /["']?(?:status|code)["']?\s*[:=]\s*402\b|\bhttp\s*402\b|\berror(?:\s+code)?\s*[:=]?\s*402\b|\b(?:got|returned|received)\s+(?:a\s+)?402\b|^\s*402\spayment/i.test(
       lower,
     )
   ) {

src/media-understanding/runner.entries.ts

@@ -2,6 +2,10 @@ import fs from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
 import { requireApiKey, resolveApiKeyForProvider } from "../agents/model-auth.js";
+import {
+  collectProviderApiKeysForExecution,
+  executeWithApiKeyRotation,
+} from "../agents/api-key-rotation.js";
 import type { MsgContext } from "../auto-reply/templating.js";
 import { applyTemplate } from "../auto-reply/templating.js";
 import type { OpenClawConfig } from "../config/config.js";
@@ -408,7 +412,10 @@ export async function runProviderEntry(params: {
       preferredProfile: entry.preferredProfile,
       agentDir: params.agentDir,
     });
-    const apiKey = requireApiKey(auth, providerId);
+    const apiKeys = collectProviderApiKeysForExecution({
+      provider: providerId,
+      primaryApiKey: requireApiKey(auth, providerId),
+    });
     const providerConfig = cfg.models?.providers?.[providerId];
     const baseUrl = entry.baseUrl ?? params.config?.baseUrl ?? providerConfig?.baseUrl;
     const mergedHeaders = {
@@ -423,18 +430,23 @@ export async function runProviderEntry(params: {
       entry,
     });
     const model = entry.model?.trim() || DEFAULT_AUDIO_MODELS[providerId] || entry.model;
-    const result = await provider.transcribeAudio({
-      buffer: media.buffer,
-      fileName: media.fileName,
-      mime: media.mime,
-      apiKey,
-      baseUrl,
-      headers,
-      model,
-      language: entry.language ?? params.config?.language ?? cfg.tools?.media?.audio?.language,
-      prompt,
-      query: providerQuery,
-      timeoutMs,
+    const result = await executeWithApiKeyRotation({
+      provider: providerId,
+      apiKeys,
+      execute: async (apiKey) =>
+        provider.transcribeAudio({
+          buffer: media.buffer,
+          fileName: media.fileName,
+          mime: media.mime,
+          apiKey,
+          baseUrl,
+          headers,
+          model,
+          language: entry.language ?? params.config?.language ?? cfg.tools?.media?.audio?.language,
+          prompt,
+          query: providerQuery,
+          timeoutMs,
+        }),
     });
     return {
       kind: "audio.transcription",
@@ -468,18 +480,26 @@ export async function runProviderEntry(params: {
     preferredProfile: entry.preferredProfile,
     agentDir: params.agentDir,
   });
-  const apiKey = requireApiKey(auth, providerId);
+  const apiKeys = collectProviderApiKeysForExecution({
+    provider: providerId,
+    primaryApiKey: requireApiKey(auth, providerId),
+  });
   const providerConfig = cfg.models?.providers?.[providerId];
-  const result = await provider.describeVideo({
-    buffer: media.buffer,
-    fileName: media.fileName,
-    mime: media.mime,
-    apiKey,
-    baseUrl: providerConfig?.baseUrl,
-    headers: providerConfig?.headers,
-    model: entry.model,
-    prompt,
-    timeoutMs,
+  const result = await executeWithApiKeyRotation({
+    provider: providerId,
+    apiKeys,
+    execute: (apiKey) =>
+      provider.describeVideo({
+        buffer: media.buffer,
+        fileName: media.fileName,
+        mime: media.mime,
+        apiKey,
+        baseUrl: providerConfig?.baseUrl,
+        headers: providerConfig?.headers,
+        model: entry.model,
+        prompt,
+        timeoutMs,
+      }),
   });
   return {
     kind: "video.description",

src/memory/embeddings-gemini.ts

@@ -1,13 +1,18 @@
+import type { EmbeddingProvider, EmbeddingProviderOptions } from "./embeddings.js";
+import {
+  collectProviderApiKeysForExecution,
+  executeWithApiKeyRotation,
+} from "../agents/api-key-rotation.js";
 import { requireApiKey, resolveApiKeyForProvider } from "../agents/model-auth.js";
 import { parseGeminiAuth } from "../infra/gemini-auth.js";
 import { debugEmbeddingsLog } from "./embeddings-debug.js";
-import type { EmbeddingProvider, EmbeddingProviderOptions } from "./embeddings.js";
 
 export type GeminiEmbeddingClient = {
   baseUrl: string;
   headers: Record<string, string>;
   model: string;
   modelPath: string;
+  apiKeys: string[];
 };
 
 const DEFAULT_GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta";
@@ -62,23 +67,40 @@ export async function createGeminiEmbeddingProvider(
   const embedUrl = `${baseUrl}/${client.modelPath}:embedContent`;
   const batchUrl = `${baseUrl}/${client.modelPath}:batchEmbedContents`;
 
-  const embedQuery = async (text: string): Promise<number[]> => {
-    if (!text.trim()) {
-      return [];
-    }
-    const res = await fetch(embedUrl, {
+  const fetchWithGeminiAuth = async (apiKey: string, endpoint: string, body: unknown) => {
+    const authHeaders = parseGeminiAuth(apiKey);
+    const headers = {
+      ...authHeaders.headers,
+      ...client.headers,
+    };
+    const res = await fetch(endpoint, {
       method: "POST",
-      headers: client.headers,
-      body: JSON.stringify({
-        content: { parts: [{ text }] },
-        taskType: "RETRIEVAL_QUERY",
-      }),
+      headers,
+      body: JSON.stringify(body),
     });
     if (!res.ok) {
       const payload = await res.text();
       throw new Error(`gemini embeddings failed: ${res.status} ${payload}`);
     }
-    const payload = (await res.json()) as { embedding?: { values?: number[] } };
+    return (await res.json()) as {
+      embedding?: { values?: number[] };
+      embeddings?: Array<{ values?: number[] }>;
+    };
+  };
+
+  const embedQuery = async (text: string): Promise<number[]> => {
+    if (!text.trim()) {
+      return [];
+    }
+    const payload = await executeWithApiKeyRotation({
+      provider: "google",
+      apiKeys: client.apiKeys,
+      execute: (apiKey) =>
+        fetchWithGeminiAuth(apiKey, embedUrl, {
+          content: { parts: [{ text }] },
+          taskType: "RETRIEVAL_QUERY",
+        }),
+    });
     return payload.embedding?.values ?? [];
   };
 
@@ -91,16 +113,14 @@ export async function createGeminiEmbeddingProvider(
       content: { parts: [{ text }] },
       taskType: "RETRIEVAL_DOCUMENT",
     }));
-    const res = await fetch(batchUrl, {
-      method: "POST",
-      headers: client.headers,
-      body: JSON.stringify({ requests }),
+    const payload = await executeWithApiKeyRotation({
+      provider: "google",
+      apiKeys: client.apiKeys,
+      execute: (apiKey) =>
+        fetchWithGeminiAuth(apiKey, batchUrl, {
+          requests,
+        }),
     });
-    if (!res.ok) {
-      const payload = await res.text();
-      throw new Error(`gemini embeddings failed: ${res.status} ${payload}`);
-    }
-    const payload = (await res.json()) as { embeddings?: Array<{ values?: number[] }> };
     const embeddings = Array.isArray(payload.embeddings) ? payload.embeddings : [];
     return texts.map((_, index) => embeddings[index]?.values ?? []);
   };
@@ -139,11 +159,13 @@ export async function resolveGeminiEmbeddingClient(
   const rawBaseUrl = remoteBaseUrl || providerConfig?.baseUrl?.trim() || DEFAULT_GEMINI_BASE_URL;
   const baseUrl = normalizeGeminiBaseUrl(rawBaseUrl);
   const headerOverrides = Object.assign({}, providerConfig?.headers, remote?.headers);
-  const authHeaders = parseGeminiAuth(apiKey);
   const headers: Record<string, string> = {
-    ...authHeaders.headers,
     ...headerOverrides,
   };
+  const apiKeys = collectProviderApiKeysForExecution({
+    provider: "google",
+    primaryApiKey: apiKey,
+  });
   const model = normalizeGeminiModel(options.model);
   const modelPath = buildGeminiModelPath(model);
   debugEmbeddingsLog("memory embeddings: gemini client", {
@@ -154,5 +176,5 @@ export async function resolveGeminiEmbeddingClient(
     embedEndpoint: `${baseUrl}/${modelPath}:embedContent`,
     batchEndpoint: `${baseUrl}/${modelPath}:batchEmbedContents`,
   });
-  return { baseUrl, headers, model, modelPath };
+  return { baseUrl, headers, model, modelPath, apiKeys };
 }