From c2876b69fbf5b7907ee6b7534284d842c1d2023c Mon Sep 17 00:00:00 2001 From: Josh Avant <830519+joshavant@users.noreply.github.com> Date: Thu, 19 Feb 2026 14:33:02 -0800 Subject: [PATCH] feat(auto-reply): add model fallback lifecycle visibility in status, verbose logs, and WebUI (#20704) --- CHANGELOG.md | 1 + src/auto-reply/fallback-state.test.ts | 123 +++++ src/auto-reply/fallback-state.ts | 180 +++++++ src/auto-reply/model-runtime.ts | 93 ++++ .../reply/agent-runner-execution.ts | 24 + .../reply/agent-runner.runreplyagent.test.ts | 503 +++++++++++++++++- src/auto-reply/reply/agent-runner.ts | 123 ++++- src/auto-reply/reply/commands-status.ts | 28 +- .../reply/directive-handling.impl.ts | 1 + .../reply/directive-handling.model.test.ts | 26 + .../reply/directive-handling.model.ts | 39 +- src/auto-reply/status.test.ts | 65 ++- src/auto-reply/status.ts | 68 ++- src/config/sessions/types.ts | 7 + src/gateway/server-chat.agent-events.test.ts | 137 +++++ src/gateway/server-chat.ts | 17 +- ui/src/styles/components.css | 10 + ui/src/ui/app-render.ts | 1 + ui/src/ui/app-tool-stream.node.test.ts | 139 +++++ ui/src/ui/app-tool-stream.ts | 198 ++++++- ui/src/ui/app-view-state.ts | 3 +- ui/src/ui/app.ts | 2 + ui/src/ui/views/chat.test.ts | 70 +++ ui/src/ui/views/chat.ts | 52 ++ 24 files changed, 1855 insertions(+), 55 deletions(-) create mode 100644 src/auto-reply/fallback-state.test.ts create mode 100644 src/auto-reply/fallback-state.ts create mode 100644 src/auto-reply/model-runtime.ts create mode 100644 ui/src/ui/app-tool-stream.node.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f00192254..7c43210065 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ Docs: https://docs.openclaw.ai ### Changes - iOS/Gateway: stabilize background wake and reconnect behavior with background reconnect suppression/lease windows, BGAppRefresh wake fallback, location wake hook throttling, and APNs wake retry+nudge instrumentation. (#21226) thanks @mbelinky. +- Auto-reply/UI: add model fallback lifecycle visibility in verbose logs, /status active-model context with fallback reason, and cohesive WebUI fallback indicators. (#20704) Thanks @joshavant. ### Fixes diff --git a/src/auto-reply/fallback-state.test.ts b/src/auto-reply/fallback-state.test.ts new file mode 100644 index 0000000000..f15048a5bb --- /dev/null +++ b/src/auto-reply/fallback-state.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, it } from "vitest"; +import { + resolveActiveFallbackState, + resolveFallbackTransition, + type FallbackNoticeState, +} from "./fallback-state.js"; + +const baseAttempt = { + provider: "fireworks", + model: "fireworks/minimax-m2p5", + error: "Provider fireworks is in cooldown (all profiles unavailable)", + reason: "rate_limit" as const, +}; + +describe("fallback-state", () => { + it("treats fallback as active only when state matches selected and active refs", () => { + const state: FallbackNoticeState = { + fallbackNoticeSelectedModel: "fireworks/minimax-m2p5", + fallbackNoticeActiveModel: "deepinfra/moonshotai/Kimi-K2.5", + fallbackNoticeReason: "rate limit", + }; + + const resolved = resolveActiveFallbackState({ + selectedModelRef: "fireworks/minimax-m2p5", + activeModelRef: "deepinfra/moonshotai/Kimi-K2.5", + state, + }); + + expect(resolved.active).toBe(true); + expect(resolved.reason).toBe("rate limit"); + }); + + it("does not treat runtime drift as fallback when persisted state does not match", () => { + const state: FallbackNoticeState = { + fallbackNoticeSelectedModel: "anthropic/claude", + fallbackNoticeActiveModel: "deepinfra/moonshotai/Kimi-K2.5", + fallbackNoticeReason: "rate limit", + }; + + const resolved = resolveActiveFallbackState({ + selectedModelRef: "fireworks/minimax-m2p5", + activeModelRef: "deepinfra/moonshotai/Kimi-K2.5", + state, + }); + + expect(resolved.active).toBe(false); + expect(resolved.reason).toBeUndefined(); + }); + + it("marks fallback transition when selected->active pair changes", () => { + const resolved = resolveFallbackTransition({ + selectedProvider: "fireworks", + selectedModel: "fireworks/minimax-m2p5", + activeProvider: "deepinfra", + activeModel: "moonshotai/Kimi-K2.5", + attempts: [baseAttempt], + state: {}, + }); + + expect(resolved.fallbackActive).toBe(true); + expect(resolved.fallbackTransitioned).toBe(true); + expect(resolved.fallbackCleared).toBe(false); + expect(resolved.stateChanged).toBe(true); + expect(resolved.reasonSummary).toBe("rate limit"); + expect(resolved.nextState.selectedModel).toBe("fireworks/minimax-m2p5"); + expect(resolved.nextState.activeModel).toBe("deepinfra/moonshotai/Kimi-K2.5"); + }); + + it("normalizes fallback reason whitespace for summaries", () => { + const resolved = resolveFallbackTransition({ + selectedProvider: "fireworks", + selectedModel: "fireworks/minimax-m2p5", + activeProvider: "deepinfra", + activeModel: "moonshotai/Kimi-K2.5", + attempts: [{ ...baseAttempt, reason: "rate_limit\n\tburst" }], + state: {}, + }); + + expect(resolved.reasonSummary).toBe("rate limit burst"); + }); + + it("refreshes reason when fallback remains active with same model pair", () => { + const resolved = resolveFallbackTransition({ + selectedProvider: "fireworks", + selectedModel: "fireworks/minimax-m2p5", + activeProvider: "deepinfra", + activeModel: "moonshotai/Kimi-K2.5", + attempts: [{ ...baseAttempt, reason: "timeout" }], + state: { + fallbackNoticeSelectedModel: "fireworks/minimax-m2p5", + fallbackNoticeActiveModel: "deepinfra/moonshotai/Kimi-K2.5", + fallbackNoticeReason: "rate limit", + }, + }); + + expect(resolved.fallbackTransitioned).toBe(false); + expect(resolved.stateChanged).toBe(true); + expect(resolved.nextState.reason).toBe("timeout"); + }); + + it("marks fallback as cleared when runtime returns to selected model", () => { + const resolved = resolveFallbackTransition({ + selectedProvider: "fireworks", + selectedModel: "fireworks/minimax-m2p5", + activeProvider: "fireworks", + activeModel: "fireworks/minimax-m2p5", + attempts: [], + state: { + fallbackNoticeSelectedModel: "fireworks/minimax-m2p5", + fallbackNoticeActiveModel: "deepinfra/moonshotai/Kimi-K2.5", + fallbackNoticeReason: "rate limit", + }, + }); + + expect(resolved.fallbackActive).toBe(false); + expect(resolved.fallbackCleared).toBe(true); + expect(resolved.fallbackTransitioned).toBe(false); + expect(resolved.stateChanged).toBe(true); + expect(resolved.nextState.selectedModel).toBeUndefined(); + expect(resolved.nextState.activeModel).toBeUndefined(); + expect(resolved.nextState.reason).toBeUndefined(); + }); +}); diff --git a/src/auto-reply/fallback-state.ts b/src/auto-reply/fallback-state.ts new file mode 100644 index 0000000000..836cf70d91 --- /dev/null +++ b/src/auto-reply/fallback-state.ts @@ -0,0 +1,180 @@ +import type { SessionEntry } from "../config/sessions.js"; +import { formatProviderModelRef } from "./model-runtime.js"; +import type { RuntimeFallbackAttempt } from "./reply/agent-runner-execution.js"; + +const FALLBACK_REASON_PART_MAX = 80; + +export type FallbackNoticeState = Pick< + SessionEntry, + "fallbackNoticeSelectedModel" | "fallbackNoticeActiveModel" | "fallbackNoticeReason" +>; + +export function normalizeFallbackModelRef(value?: string): string | undefined { + const trimmed = String(value ?? "").trim(); + return trimmed || undefined; +} + +function truncateFallbackReasonPart(value: string, max = FALLBACK_REASON_PART_MAX): string { + const text = String(value ?? "") + .replace(/\s+/g, " ") + .trim(); + if (text.length <= max) { + return text; + } + return `${text.slice(0, Math.max(0, max - 1)).trimEnd()}…`; +} + +export function formatFallbackAttemptReason(attempt: RuntimeFallbackAttempt): string { + const reason = attempt.reason?.trim(); + if (reason) { + return reason.replace(/_/g, " "); + } + const code = attempt.code?.trim(); + if (code) { + return code; + } + if (typeof attempt.status === "number") { + return `HTTP ${attempt.status}`; + } + return truncateFallbackReasonPart(attempt.error || "error"); +} + +function formatFallbackAttemptSummary(attempt: RuntimeFallbackAttempt): string { + return `${formatProviderModelRef(attempt.provider, attempt.model)} ${formatFallbackAttemptReason(attempt)}`; +} + +export function buildFallbackReasonSummary(attempts: RuntimeFallbackAttempt[]): string { + const firstAttempt = attempts[0]; + const firstReason = firstAttempt + ? formatFallbackAttemptReason(firstAttempt) + : "selected model unavailable"; + const moreAttempts = attempts.length > 1 ? ` (+${attempts.length - 1} more attempts)` : ""; + return `${truncateFallbackReasonPart(firstReason)}${moreAttempts}`; +} + +export function buildFallbackAttemptSummaries(attempts: RuntimeFallbackAttempt[]): string[] { + return attempts.map((attempt) => + truncateFallbackReasonPart(formatFallbackAttemptSummary(attempt)), + ); +} + +export function buildFallbackNotice(params: { + selectedProvider: string; + selectedModel: string; + activeProvider: string; + activeModel: string; + attempts: RuntimeFallbackAttempt[]; +}): string | null { + const selected = formatProviderModelRef(params.selectedProvider, params.selectedModel); + const active = formatProviderModelRef(params.activeProvider, params.activeModel); + if (selected === active) { + return null; + } + const reasonSummary = buildFallbackReasonSummary(params.attempts); + return `↪️ Model Fallback: ${active} (selected ${selected}; ${reasonSummary})`; +} + +export function buildFallbackClearedNotice(params: { + selectedProvider: string; + selectedModel: string; + previousActiveModel?: string; +}): string { + const selected = formatProviderModelRef(params.selectedProvider, params.selectedModel); + const previous = normalizeFallbackModelRef(params.previousActiveModel); + if (previous && previous !== selected) { + return `↪️ Model Fallback cleared: ${selected} (was ${previous})`; + } + return `↪️ Model Fallback cleared: ${selected}`; +} + +export function resolveActiveFallbackState(params: { + selectedModelRef: string; + activeModelRef: string; + state?: FallbackNoticeState; +}): { active: boolean; reason?: string } { + const selected = normalizeFallbackModelRef(params.state?.fallbackNoticeSelectedModel); + const active = normalizeFallbackModelRef(params.state?.fallbackNoticeActiveModel); + const reason = normalizeFallbackModelRef(params.state?.fallbackNoticeReason); + const fallbackActive = + params.selectedModelRef !== params.activeModelRef && + selected === params.selectedModelRef && + active === params.activeModelRef; + return { + active: fallbackActive, + reason: fallbackActive ? reason : undefined, + }; +} + +export type ResolvedFallbackTransition = { + selectedModelRef: string; + activeModelRef: string; + fallbackActive: boolean; + fallbackTransitioned: boolean; + fallbackCleared: boolean; + reasonSummary: string; + attemptSummaries: string[]; + previousState: { + selectedModel?: string; + activeModel?: string; + reason?: string; + }; + nextState: { + selectedModel?: string; + activeModel?: string; + reason?: string; + }; + stateChanged: boolean; +}; + +export function resolveFallbackTransition(params: { + selectedProvider: string; + selectedModel: string; + activeProvider: string; + activeModel: string; + attempts: RuntimeFallbackAttempt[]; + state?: FallbackNoticeState; +}): ResolvedFallbackTransition { + const selectedModelRef = formatProviderModelRef(params.selectedProvider, params.selectedModel); + const activeModelRef = formatProviderModelRef(params.activeProvider, params.activeModel); + const previousState = { + selectedModel: normalizeFallbackModelRef(params.state?.fallbackNoticeSelectedModel), + activeModel: normalizeFallbackModelRef(params.state?.fallbackNoticeActiveModel), + reason: normalizeFallbackModelRef(params.state?.fallbackNoticeReason), + }; + const fallbackActive = selectedModelRef !== activeModelRef; + const fallbackTransitioned = + fallbackActive && + (previousState.selectedModel !== selectedModelRef || + previousState.activeModel !== activeModelRef); + const fallbackCleared = + !fallbackActive && Boolean(previousState.selectedModel || previousState.activeModel); + const reasonSummary = buildFallbackReasonSummary(params.attempts); + const attemptSummaries = buildFallbackAttemptSummaries(params.attempts); + const nextState = fallbackActive + ? { + selectedModel: selectedModelRef, + activeModel: activeModelRef, + reason: reasonSummary, + } + : { + selectedModel: undefined, + activeModel: undefined, + reason: undefined, + }; + const stateChanged = + previousState.selectedModel !== nextState.selectedModel || + previousState.activeModel !== nextState.activeModel || + previousState.reason !== nextState.reason; + return { + selectedModelRef, + activeModelRef, + fallbackActive, + fallbackTransitioned, + fallbackCleared, + reasonSummary, + attemptSummaries, + previousState, + nextState, + stateChanged, + }; +} diff --git a/src/auto-reply/model-runtime.ts b/src/auto-reply/model-runtime.ts new file mode 100644 index 0000000000..e43bd66305 --- /dev/null +++ b/src/auto-reply/model-runtime.ts @@ -0,0 +1,93 @@ +import type { SessionEntry } from "../config/sessions.js"; + +export function formatProviderModelRef(providerRaw: string, modelRaw: string): string { + const provider = String(providerRaw ?? "").trim(); + const model = String(modelRaw ?? "").trim(); + if (!provider) { + return model; + } + if (!model) { + return provider; + } + const prefix = `${provider}/`; + if (model.toLowerCase().startsWith(prefix.toLowerCase())) { + const normalizedModel = model.slice(prefix.length).trim(); + if (normalizedModel) { + return `${provider}/${normalizedModel}`; + } + } + return `${provider}/${model}`; +} + +type ModelRef = { + provider: string; + model: string; + label: string; +}; + +function normalizeModelWithinProvider(provider: string, modelRaw: string): string { + const model = String(modelRaw ?? "").trim(); + if (!provider || !model) { + return model; + } + const prefix = `${provider}/`; + if (model.toLowerCase().startsWith(prefix.toLowerCase())) { + const withoutPrefix = model.slice(prefix.length).trim(); + if (withoutPrefix) { + return withoutPrefix; + } + } + return model; +} + +function normalizeModelRef( + rawModel: string, + fallbackProvider: string, + parseEmbeddedProvider = false, +): ModelRef { + const trimmed = String(rawModel ?? "").trim(); + const slashIndex = parseEmbeddedProvider ? trimmed.indexOf("/") : -1; + if (slashIndex > 0) { + const provider = trimmed.slice(0, slashIndex).trim(); + const model = trimmed.slice(slashIndex + 1).trim(); + if (provider && model) { + return { + provider, + model, + label: `${provider}/${model}`, + }; + } + } + const provider = String(fallbackProvider ?? "").trim(); + const dedupedModel = normalizeModelWithinProvider(provider, trimmed); + return { + provider, + model: dedupedModel || trimmed, + label: provider ? formatProviderModelRef(provider, dedupedModel || trimmed) : trimmed, + }; +} + +export function resolveSelectedAndActiveModel(params: { + selectedProvider: string; + selectedModel: string; + sessionEntry?: Pick; +}): { + selected: ModelRef; + active: ModelRef; + activeDiffers: boolean; +} { + const selected = normalizeModelRef(params.selectedModel, params.selectedProvider); + const runtimeModel = params.sessionEntry?.model?.trim(); + const runtimeProvider = params.sessionEntry?.modelProvider?.trim(); + + const active = runtimeModel + ? normalizeModelRef(runtimeModel, runtimeProvider || selected.provider, !runtimeProvider) + : selected; + const activeDiffers = active.provider !== selected.provider || active.model !== selected.model; + + return { + selected, + active, + activeDiffers, + }; +} diff --git a/src/auto-reply/reply/agent-runner-execution.ts b/src/auto-reply/reply/agent-runner-execution.ts index b9cf70b17c..1bc0d9ed0f 100644 --- a/src/auto-reply/reply/agent-runner-execution.ts +++ b/src/auto-reply/reply/agent-runner-execution.ts @@ -40,12 +40,23 @@ import type { FollowupRun } from "./queue.js"; import { createBlockReplyDeliveryHandler } from "./reply-delivery.js"; import type { TypingSignaler } from "./typing-mode.js"; +export type RuntimeFallbackAttempt = { + provider: string; + model: string; + error: string; + reason?: string; + status?: number; + code?: string; +}; + export type AgentRunLoopResult = | { kind: "success"; + runId: string; runResult: Awaited>; fallbackProvider?: string; fallbackModel?: string; + fallbackAttempts: RuntimeFallbackAttempt[]; didLogHeartbeatStrip: boolean; autoCompactionCompleted: boolean; /** Payload keys sent directly (not via pipeline) during tool flush. */ @@ -106,6 +117,7 @@ export async function runAgentTurnWithFallback(params: { let runResult: Awaited>; let fallbackProvider = params.followupRun.run.provider; let fallbackModel = params.followupRun.run.model; + let fallbackAttempts: RuntimeFallbackAttempt[] = []; let didResetAfterCompactionFailure = false; let didRetryTransientHttpError = false; @@ -397,6 +409,16 @@ export async function runAgentTurnWithFallback(params: { runResult = fallbackResult.result; fallbackProvider = fallbackResult.provider; fallbackModel = fallbackResult.model; + fallbackAttempts = Array.isArray(fallbackResult.attempts) + ? fallbackResult.attempts.map((attempt) => ({ + provider: String(attempt.provider ?? ""), + model: String(attempt.model ?? ""), + error: String(attempt.error ?? ""), + reason: attempt.reason ? String(attempt.reason) : undefined, + status: typeof attempt.status === "number" ? attempt.status : undefined, + code: attempt.code ? String(attempt.code) : undefined, + })) + : []; // Some embedded runs surface context overflow as an error payload instead of throwing. // Treat those as a session-level failure and auto-recover by starting a fresh session. @@ -543,9 +565,11 @@ export async function runAgentTurnWithFallback(params: { return { kind: "success", + runId, runResult, fallbackProvider, fallbackModel, + fallbackAttempts, didLogHeartbeatStrip, autoCompactionCompleted, directlySentBlockKeys: directlySentBlockKeys.size > 0 ? directlySentBlockKeys : undefined, diff --git a/src/auto-reply/reply/agent-runner.runreplyagent.test.ts b/src/auto-reply/reply/agent-runner.runreplyagent.test.ts index 0263c8a15f..f87f8279b9 100644 --- a/src/auto-reply/reply/agent-runner.runreplyagent.test.ts +++ b/src/auto-reply/reply/agent-runner.runreplyagent.test.ts @@ -54,6 +54,7 @@ vi.mock("../../agents/model-fallback.js", () => ({ result: await run(provider, model), provider, model, + attempts: [], }), })); @@ -508,6 +509,30 @@ describe("runReplyAgent typing (heartbeat)", () => { expect(onToolResult).not.toHaveBeenCalled(); }); + it("retries transient HTTP failures once with timer-driven backoff", async () => { + vi.useFakeTimers(); + let calls = 0; + state.runEmbeddedPiAgentMock.mockImplementation(async () => { + calls += 1; + if (calls === 1) { + throw new Error("502 Bad Gateway"); + } + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const { run } = createMinimalRun({ + typingMode: "message", + }); + const runPromise = run(); + + await vi.advanceTimersByTimeAsync(2_499); + expect(calls).toBe(1); + await vi.advanceTimersByTimeAsync(1); + await runPromise; + expect(calls).toBe(2); + vi.useRealTimers(); + }); + it("announces auto-compaction in verbose mode and tracks count", async () => { await withTempStateDir(async (stateDir) => { const storePath = path.join(stateDir, "sessions", "sessions.json"); @@ -538,12 +563,482 @@ describe("runReplyAgent typing (heartbeat)", () => { }); }); + it("announces model fallback in verbose mode", async () => { + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: Date.now(), + }; + const sessionStore = { main: sessionEntry }; + state.runEmbeddedPiAgentMock.mockResolvedValueOnce({ payloads: [{ text: "final" }], meta: {} }); + const modelFallback = await import("../../agents/model-fallback.js"); + vi.spyOn(modelFallback, "runWithModelFallback").mockImplementationOnce( + async ({ run }: { run: (provider: string, model: string) => Promise }) => ({ + result: await run("deepinfra", "moonshotai/Kimi-K2.5"), + provider: "deepinfra", + model: "moonshotai/Kimi-K2.5", + attempts: [ + { + provider: "fireworks", + model: "fireworks/minimax-m2p5", + error: "Provider fireworks is in cooldown (all profiles unavailable)", + reason: "rate_limit", + }, + ], + }), + ); + + const { run } = createMinimalRun({ + resolvedVerboseLevel: "on", + sessionEntry, + sessionStore, + sessionKey: "main", + }); + const res = await run(); + expect(Array.isArray(res)).toBe(true); + const payloads = res as { text?: string }[]; + expect(payloads[0]?.text).toContain("Model Fallback:"); + expect(payloads[0]?.text).toContain("deepinfra/moonshotai/Kimi-K2.5"); + expect(sessionEntry.fallbackNoticeReason).toBe("rate limit"); + }); + + it("does not announce model fallback when verbose is off", async () => { + const { onAgentEvent } = await import("../../infra/agent-events.js"); + state.runEmbeddedPiAgentMock.mockResolvedValueOnce({ payloads: [{ text: "final" }], meta: {} }); + const modelFallback = await import("../../agents/model-fallback.js"); + vi.spyOn(modelFallback, "runWithModelFallback").mockImplementationOnce( + async ({ run }: { run: (provider: string, model: string) => Promise }) => ({ + result: await run("deepinfra", "moonshotai/Kimi-K2.5"), + provider: "deepinfra", + model: "moonshotai/Kimi-K2.5", + attempts: [ + { + provider: "fireworks", + model: "fireworks/minimax-m2p5", + error: "Provider fireworks is in cooldown (all profiles unavailable)", + reason: "rate_limit", + }, + ], + }), + ); + + const { run } = createMinimalRun({ + resolvedVerboseLevel: "off", + }); + const phases: string[] = []; + const off = onAgentEvent((evt) => { + const phase = typeof evt.data?.phase === "string" ? evt.data.phase : null; + if (evt.stream === "lifecycle" && phase) { + phases.push(phase); + } + }); + const res = await run(); + off(); + const payload = Array.isArray(res) ? (res[0] as { text?: string }) : (res as { text?: string }); + expect(payload.text).not.toContain("Model Fallback:"); + expect(phases.filter((phase) => phase === "fallback")).toHaveLength(1); + }); + + it("announces model fallback only once per active fallback state", async () => { + const { onAgentEvent } = await import("../../infra/agent-events.js"); + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: Date.now(), + }; + const sessionStore = { main: sessionEntry }; + + state.runEmbeddedPiAgentMock.mockResolvedValue({ + payloads: [{ text: "final" }], + meta: {}, + }); + const modelFallback = await import("../../agents/model-fallback.js"); + const fallbackSpy = vi + .spyOn(modelFallback, "runWithModelFallback") + .mockImplementation( + async ({ run }: { run: (provider: string, model: string) => Promise }) => ({ + result: await run("deepinfra", "moonshotai/Kimi-K2.5"), + provider: "deepinfra", + model: "moonshotai/Kimi-K2.5", + attempts: [ + { + provider: "fireworks", + model: "fireworks/minimax-m2p5", + error: "Provider fireworks is in cooldown (all profiles unavailable)", + reason: "rate_limit", + }, + ], + }), + ); + try { + const { run } = createMinimalRun({ + resolvedVerboseLevel: "on", + sessionEntry, + sessionStore, + sessionKey: "main", + }); + const fallbackEvents: Array> = []; + const off = onAgentEvent((evt) => { + if (evt.stream === "lifecycle" && evt.data?.phase === "fallback") { + fallbackEvents.push(evt.data); + } + }); + const first = await run(); + const second = await run(); + off(); + + const firstText = Array.isArray(first) ? first[0]?.text : first?.text; + const secondText = Array.isArray(second) ? second[0]?.text : second?.text; + expect(firstText).toContain("Model Fallback:"); + expect(secondText).not.toContain("Model Fallback:"); + expect(fallbackEvents).toHaveLength(1); + } finally { + fallbackSpy.mockRestore(); + } + }); + + it("re-announces model fallback after returning to selected model", async () => { + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: Date.now(), + }; + const sessionStore = { main: sessionEntry }; + let callCount = 0; + + state.runEmbeddedPiAgentMock.mockResolvedValue({ + payloads: [{ text: "final" }], + meta: {}, + }); + const modelFallback = await import("../../agents/model-fallback.js"); + const fallbackSpy = vi + .spyOn(modelFallback, "runWithModelFallback") + .mockImplementation( + async ({ + provider, + model, + run, + }: { + provider: string; + model: string; + run: (provider: string, model: string) => Promise; + }) => { + callCount += 1; + if (callCount === 2) { + return { + result: await run(provider, model), + provider, + model, + attempts: [], + }; + } + return { + result: await run("deepinfra", "moonshotai/Kimi-K2.5"), + provider: "deepinfra", + model: "moonshotai/Kimi-K2.5", + attempts: [ + { + provider: "fireworks", + model: "fireworks/minimax-m2p5", + error: "Provider fireworks is in cooldown (all profiles unavailable)", + reason: "rate_limit", + }, + ], + }; + }, + ); + try { + const { run } = createMinimalRun({ + resolvedVerboseLevel: "on", + sessionEntry, + sessionStore, + sessionKey: "main", + }); + const first = await run(); + const second = await run(); + const third = await run(); + + const firstText = Array.isArray(first) ? first[0]?.text : first?.text; + const secondText = Array.isArray(second) ? second[0]?.text : second?.text; + const thirdText = Array.isArray(third) ? third[0]?.text : third?.text; + expect(firstText).toContain("Model Fallback:"); + expect(secondText).not.toContain("Model Fallback:"); + expect(thirdText).toContain("Model Fallback:"); + } finally { + fallbackSpy.mockRestore(); + } + }); + + it("announces fallback-cleared once when runtime returns to selected model", async () => { + const { onAgentEvent } = await import("../../infra/agent-events.js"); + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: Date.now(), + }; + const sessionStore = { main: sessionEntry }; + let callCount = 0; + + state.runEmbeddedPiAgentMock.mockResolvedValue({ + payloads: [{ text: "final" }], + meta: {}, + }); + const modelFallback = await import("../../agents/model-fallback.js"); + const fallbackSpy = vi + .spyOn(modelFallback, "runWithModelFallback") + .mockImplementation( + async ({ + provider, + model, + run, + }: { + provider: string; + model: string; + run: (provider: string, model: string) => Promise; + }) => { + callCount += 1; + if (callCount === 1) { + return { + result: await run("deepinfra", "moonshotai/Kimi-K2.5"), + provider: "deepinfra", + model: "moonshotai/Kimi-K2.5", + attempts: [ + { + provider: "fireworks", + model: "fireworks/minimax-m2p5", + error: "Provider fireworks is in cooldown (all profiles unavailable)", + reason: "rate_limit", + }, + ], + }; + } + return { + result: await run(provider, model), + provider, + model, + attempts: [], + }; + }, + ); + try { + const { run } = createMinimalRun({ + resolvedVerboseLevel: "on", + sessionEntry, + sessionStore, + sessionKey: "main", + }); + const phases: string[] = []; + const off = onAgentEvent((evt) => { + const phase = typeof evt.data?.phase === "string" ? evt.data.phase : null; + if (evt.stream === "lifecycle" && phase) { + phases.push(phase); + } + }); + const first = await run(); + const second = await run(); + const third = await run(); + off(); + + const firstText = Array.isArray(first) ? first[0]?.text : first?.text; + const secondText = Array.isArray(second) ? second[0]?.text : second?.text; + const thirdText = Array.isArray(third) ? third[0]?.text : third?.text; + expect(firstText).toContain("Model Fallback:"); + expect(secondText).toContain("Model Fallback cleared:"); + expect(thirdText).not.toContain("Model Fallback cleared:"); + expect(phases.filter((phase) => phase === "fallback")).toHaveLength(1); + expect(phases.filter((phase) => phase === "fallback_cleared")).toHaveLength(1); + } finally { + fallbackSpy.mockRestore(); + } + }); + + it("emits fallback lifecycle events while verbose is off", async () => { + const { onAgentEvent } = await import("../../infra/agent-events.js"); + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: Date.now(), + }; + const sessionStore = { main: sessionEntry }; + let callCount = 0; + + state.runEmbeddedPiAgentMock.mockResolvedValue({ + payloads: [{ text: "final" }], + meta: {}, + }); + const modelFallback = await import("../../agents/model-fallback.js"); + const fallbackSpy = vi + .spyOn(modelFallback, "runWithModelFallback") + .mockImplementation( + async ({ + provider, + model, + run, + }: { + provider: string; + model: string; + run: (provider: string, model: string) => Promise; + }) => { + callCount += 1; + if (callCount === 1) { + return { + result: await run("deepinfra", "moonshotai/Kimi-K2.5"), + provider: "deepinfra", + model: "moonshotai/Kimi-K2.5", + attempts: [ + { + provider: "fireworks", + model: "fireworks/minimax-m2p5", + error: "Provider fireworks is in cooldown (all profiles unavailable)", + reason: "rate_limit", + }, + ], + }; + } + return { + result: await run(provider, model), + provider, + model, + attempts: [], + }; + }, + ); + try { + const { run } = createMinimalRun({ + resolvedVerboseLevel: "off", + sessionEntry, + sessionStore, + sessionKey: "main", + }); + const phases: string[] = []; + const off = onAgentEvent((evt) => { + const phase = typeof evt.data?.phase === "string" ? evt.data.phase : null; + if (evt.stream === "lifecycle" && phase) { + phases.push(phase); + } + }); + const first = await run(); + const second = await run(); + off(); + + const firstText = Array.isArray(first) ? first[0]?.text : first?.text; + const secondText = Array.isArray(second) ? second[0]?.text : second?.text; + expect(firstText).not.toContain("Model Fallback:"); + expect(secondText).not.toContain("Model Fallback cleared:"); + expect(phases.filter((phase) => phase === "fallback")).toHaveLength(1); + expect(phases.filter((phase) => phase === "fallback_cleared")).toHaveLength(1); + } finally { + fallbackSpy.mockRestore(); + } + }); + + it("backfills fallback reason when fallback is already active", async () => { + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: Date.now(), + fallbackNoticeSelectedModel: "anthropic/claude", + fallbackNoticeActiveModel: "deepinfra/moonshotai/Kimi-K2.5", + modelProvider: "deepinfra", + model: "moonshotai/Kimi-K2.5", + }; + const sessionStore = { main: sessionEntry }; + + state.runEmbeddedPiAgentMock.mockResolvedValue({ + payloads: [{ text: "final" }], + meta: {}, + }); + const modelFallback = await import("../../agents/model-fallback.js"); + const fallbackSpy = vi + .spyOn(modelFallback, "runWithModelFallback") + .mockImplementation( + async ({ run }: { run: (provider: string, model: string) => Promise }) => ({ + result: await run("deepinfra", "moonshotai/Kimi-K2.5"), + provider: "deepinfra", + model: "moonshotai/Kimi-K2.5", + attempts: [ + { + provider: "anthropic", + model: "claude", + error: "Provider anthropic is in cooldown (all profiles unavailable)", + reason: "rate_limit", + }, + ], + }), + ); + try { + const { run } = createMinimalRun({ + resolvedVerboseLevel: "on", + sessionEntry, + sessionStore, + sessionKey: "main", + }); + const res = await run(); + const firstText = Array.isArray(res) ? res[0]?.text : res?.text; + expect(firstText).not.toContain("Model Fallback:"); + expect(sessionEntry.fallbackNoticeReason).toBe("rate limit"); + } finally { + fallbackSpy.mockRestore(); + } + }); + + it("refreshes fallback reason summary while fallback stays active", async () => { + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: Date.now(), + fallbackNoticeSelectedModel: "anthropic/claude", + fallbackNoticeActiveModel: "deepinfra/moonshotai/Kimi-K2.5", + fallbackNoticeReason: "rate limit", + modelProvider: "deepinfra", + model: "moonshotai/Kimi-K2.5", + }; + const sessionStore = { main: sessionEntry }; + + state.runEmbeddedPiAgentMock.mockResolvedValue({ + payloads: [{ text: "final" }], + meta: {}, + }); + const modelFallback = await import("../../agents/model-fallback.js"); + const fallbackSpy = vi + .spyOn(modelFallback, "runWithModelFallback") + .mockImplementation( + async ({ run }: { run: (provider: string, model: string) => Promise }) => ({ + result: await run("deepinfra", "moonshotai/Kimi-K2.5"), + provider: "deepinfra", + model: "moonshotai/Kimi-K2.5", + attempts: [ + { + provider: "anthropic", + model: "claude", + error: "Provider anthropic is in cooldown (all profiles unavailable)", + reason: "timeout", + }, + ], + }), + ); + try { + const { run } = createMinimalRun({ + resolvedVerboseLevel: "on", + sessionEntry, + sessionStore, + sessionKey: "main", + }); + const res = await run(); + const firstText = Array.isArray(res) ? res[0]?.text : res?.text; + expect(firstText).not.toContain("Model Fallback:"); + expect(sessionEntry.fallbackNoticeReason).toBe("timeout"); + } finally { + fallbackSpy.mockRestore(); + } + }); + it("retries after compaction failure by resetting the session", async () => { await withTempStateDir(async (stateDir) => { const sessionId = "session"; const storePath = path.join(stateDir, "sessions", "sessions.json"); const transcriptPath = sessions.resolveSessionTranscriptPath(sessionId); - const sessionEntry = { sessionId, updatedAt: Date.now(), sessionFile: transcriptPath }; + const sessionEntry = { + sessionId, + updatedAt: Date.now(), + sessionFile: transcriptPath, + fallbackNoticeSelectedModel: "fireworks/minimax-m2p5", + fallbackNoticeActiveModel: "deepinfra/moonshotai/Kimi-K2.5", + fallbackNoticeReason: "rate limit", + }; const sessionStore = { main: sessionEntry }; await fs.mkdir(path.dirname(storePath), { recursive: true }); @@ -575,9 +1070,15 @@ describe("runReplyAgent typing (heartbeat)", () => { } expect(payload.text?.toLowerCase()).toContain("reset"); expect(sessionStore.main.sessionId).not.toBe(sessionId); + expect(sessionStore.main.fallbackNoticeSelectedModel).toBeUndefined(); + expect(sessionStore.main.fallbackNoticeActiveModel).toBeUndefined(); + expect(sessionStore.main.fallbackNoticeReason).toBeUndefined(); const persisted = JSON.parse(await fs.readFile(storePath, "utf-8")); expect(persisted.main.sessionId).toBe(sessionStore.main.sessionId); + expect(persisted.main.fallbackNoticeSelectedModel).toBeUndefined(); + expect(persisted.main.fallbackNoticeActiveModel).toBeUndefined(); + expect(persisted.main.fallbackNoticeReason).toBeUndefined(); }); }); diff --git a/src/auto-reply/reply/agent-runner.ts b/src/auto-reply/reply/agent-runner.ts index 57e71dc3ae..e110170929 100644 --- a/src/auto-reply/reply/agent-runner.ts +++ b/src/auto-reply/reply/agent-runner.ts @@ -15,10 +15,16 @@ import { updateSessionStoreEntry, } from "../../config/sessions.js"; import type { TypingMode } from "../../config/types.js"; +import { emitAgentEvent } from "../../infra/agent-events.js"; import { emitDiagnosticEvent, isDiagnosticsEnabled } from "../../infra/diagnostic-events.js"; import { enqueueSystemEvent } from "../../infra/system-events.js"; import { defaultRuntime } from "../../runtime.js"; import { estimateUsageCost, resolveModelCostConfig } from "../../utils/usage-format.js"; +import { + buildFallbackClearedNotice, + buildFallbackNotice, + resolveFallbackTransition, +} from "../fallback-state.js"; import type { OriginatingChannelType, TemplateContext } from "../templating.js"; import { resolveResponseUsageMode, type VerboseLevel } from "../thinking.js"; import type { GetReplyOptions, ReplyPayload } from "../types.js"; @@ -290,6 +296,9 @@ export async function runReplyAgent(params: { updatedAt: Date.now(), systemSent: false, abortedLastRun: false, + fallbackNoticeSelectedModel: undefined, + fallbackNoticeActiveModel: undefined, + fallbackNoticeReason: undefined, }; const agentId = resolveAgentIdFromSessionKey(sessionKey); const nextSessionFile = resolveSessionTranscriptPath( @@ -373,7 +382,14 @@ export async function runReplyAgent(params: { return finalizeWithFollowup(runOutcome.payload, queueKey, runFollowupTurn); } - const { runResult, fallbackProvider, fallbackModel, directlySentBlockKeys } = runOutcome; + const { + runId, + runResult, + fallbackProvider, + fallbackModel, + fallbackAttempts, + directlySentBlockKeys, + } = runOutcome; let { didLogHeartbeatStrip, autoCompactionCompleted } = runOutcome; if ( @@ -414,6 +430,42 @@ export async function runReplyAgent(params: { const modelUsed = runResult.meta?.agentMeta?.model ?? fallbackModel ?? defaultModel; const providerUsed = runResult.meta?.agentMeta?.provider ?? fallbackProvider ?? followupRun.run.provider; + const verboseEnabled = resolvedVerboseLevel !== "off"; + const selectedProvider = followupRun.run.provider; + const selectedModel = followupRun.run.model; + const fallbackStateEntry = + activeSessionEntry ?? (sessionKey ? activeSessionStore?.[sessionKey] : undefined); + const fallbackTransition = resolveFallbackTransition({ + selectedProvider, + selectedModel, + activeProvider: providerUsed, + activeModel: modelUsed, + attempts: fallbackAttempts, + state: fallbackStateEntry, + }); + if (fallbackTransition.stateChanged) { + if (fallbackStateEntry) { + fallbackStateEntry.fallbackNoticeSelectedModel = fallbackTransition.nextState.selectedModel; + fallbackStateEntry.fallbackNoticeActiveModel = fallbackTransition.nextState.activeModel; + fallbackStateEntry.fallbackNoticeReason = fallbackTransition.nextState.reason; + fallbackStateEntry.updatedAt = Date.now(); + activeSessionEntry = fallbackStateEntry; + } + if (sessionKey && fallbackStateEntry && activeSessionStore) { + activeSessionStore[sessionKey] = fallbackStateEntry; + } + if (sessionKey && storePath) { + await updateSessionStoreEntry({ + storePath, + sessionKey, + update: async () => ({ + fallbackNoticeSelectedModel: fallbackTransition.nextState.selectedModel, + fallbackNoticeActiveModel: fallbackTransition.nextState.activeModel, + fallbackNoticeReason: fallbackTransition.nextState.reason, + }), + }); + } + } const cliSessionId = isCliProvider(providerUsed, cfg) ? runResult.meta?.agentMeta?.sessionId?.trim() : undefined; @@ -546,9 +598,68 @@ export async function runReplyAgent(params: { } } - // If verbose is enabled and this is a new session, prepend a session hint. + // If verbose is enabled, prepend operational run notices. let finalPayloads = guardedReplyPayloads; - const verboseEnabled = resolvedVerboseLevel !== "off"; + const verboseNotices: ReplyPayload[] = []; + + if (verboseEnabled && activeIsNewSession) { + verboseNotices.push({ text: `🧭 New session: ${followupRun.run.sessionId}` }); + } + + if (fallbackTransition.fallbackTransitioned) { + emitAgentEvent({ + runId, + sessionKey, + stream: "lifecycle", + data: { + phase: "fallback", + selectedProvider, + selectedModel, + activeProvider: providerUsed, + activeModel: modelUsed, + reasonSummary: fallbackTransition.reasonSummary, + attemptSummaries: fallbackTransition.attemptSummaries, + attempts: fallbackAttempts, + }, + }); + if (verboseEnabled) { + const fallbackNotice = buildFallbackNotice({ + selectedProvider, + selectedModel, + activeProvider: providerUsed, + activeModel: modelUsed, + attempts: fallbackAttempts, + }); + if (fallbackNotice) { + verboseNotices.push({ text: fallbackNotice }); + } + } + } + if (fallbackTransition.fallbackCleared) { + emitAgentEvent({ + runId, + sessionKey, + stream: "lifecycle", + data: { + phase: "fallback_cleared", + selectedProvider, + selectedModel, + activeProvider: providerUsed, + activeModel: modelUsed, + previousActiveModel: fallbackTransition.previousState.activeModel, + }, + }); + if (verboseEnabled) { + verboseNotices.push({ + text: buildFallbackClearedNotice({ + selectedProvider, + selectedModel, + previousActiveModel: fallbackTransition.previousState.activeModel, + }), + }); + } + } + if (autoCompactionCompleted) { const count = await incrementRunCompactionCount({ sessionEntry: activeSessionEntry, @@ -578,11 +689,11 @@ export async function runReplyAgent(params: { if (verboseEnabled) { const suffix = typeof count === "number" ? ` (count ${count})` : ""; - finalPayloads = [{ text: `🧹 Auto-compaction complete${suffix}.` }, ...finalPayloads]; + verboseNotices.push({ text: `🧹 Auto-compaction complete${suffix}.` }); } } - if (verboseEnabled && activeIsNewSession) { - finalPayloads = [{ text: `🧭 New session: ${followupRun.run.sessionId}` }, ...finalPayloads]; + if (verboseNotices.length > 0) { + finalPayloads = [...verboseNotices, ...finalPayloads]; } if (responseUsageLine) { finalPayloads = appendUsageLine(finalPayloads, responseUsageLine); diff --git a/src/auto-reply/reply/commands-status.ts b/src/auto-reply/reply/commands-status.ts index 08aff7e056..fee7efdee7 100644 --- a/src/auto-reply/reply/commands-status.ts +++ b/src/auto-reply/reply/commands-status.ts @@ -19,6 +19,7 @@ import { } from "../../infra/provider-usage.js"; import type { MediaUnderstandingDecision } from "../../media-understanding/types.js"; import { normalizeGroupActivation } from "../group-activation.js"; +import { resolveSelectedAndActiveModel } from "../model-runtime.js"; import { buildStatusMessage } from "../status.js"; import type { ElevatedLevel, ReasoningLevel, ThinkLevel, VerboseLevel } from "../thinking.js"; import type { ReplyPayload } from "../types.js"; @@ -136,6 +137,25 @@ export async function buildStatusReply(params: { const groupActivation = isGroup ? (normalizeGroupActivation(sessionEntry?.groupActivation) ?? defaultGroupActivation()) : undefined; + const modelRefs = resolveSelectedAndActiveModel({ + selectedProvider: provider, + selectedModel: model, + sessionEntry, + }); + const selectedModelAuth = resolveModelAuthLabel({ + provider, + cfg, + sessionEntry, + agentDir: statusAgentDir, + }); + const activeModelAuth = modelRefs.activeDiffers + ? resolveModelAuthLabel({ + provider: modelRefs.active.provider, + cfg, + sessionEntry, + agentDir: statusAgentDir, + }) + : selectedModelAuth; const agentDefaults = cfg.agents?.defaults ?? {}; const statusText = buildStatusMessage({ config: cfg, @@ -160,12 +180,8 @@ export async function buildStatusReply(params: { resolvedVerbose: resolvedVerboseLevel, resolvedReasoning: resolvedReasoningLevel, resolvedElevated: resolvedElevatedLevel, - modelAuth: resolveModelAuthLabel({ - provider, - cfg, - sessionEntry, - agentDir: statusAgentDir, - }), + modelAuth: selectedModelAuth, + activeModelAuth, usageLine: usageLine ?? undefined, queue: { mode: queueSettings.mode, diff --git a/src/auto-reply/reply/directive-handling.impl.ts b/src/auto-reply/reply/directive-handling.impl.ts index cd250cc78b..156109b1c0 100644 --- a/src/auto-reply/reply/directive-handling.impl.ts +++ b/src/auto-reply/reply/directive-handling.impl.ts @@ -106,6 +106,7 @@ export async function handleDirectiveOnly( allowedModelCatalog, resetModelOverride, surface: params.surface, + sessionEntry, }); if (modelInfo) { return modelInfo; diff --git a/src/auto-reply/reply/directive-handling.model.test.ts b/src/auto-reply/reply/directive-handling.model.test.ts index 0b842fc0c7..9e47d5dffc 100644 --- a/src/auto-reply/reply/directive-handling.model.test.ts +++ b/src/auto-reply/reply/directive-handling.model.test.ts @@ -63,6 +63,32 @@ describe("/model chat UX", () => { expect(reply?.text).toContain("Switch: /model "); }); + it("shows active runtime model when different from selected model", async () => { + const directives = parseInlineDirectives("/model"); + const cfg = { commands: { text: true } } as unknown as OpenClawConfig; + + const reply = await maybeHandleModelDirectiveInfo({ + directives, + cfg, + agentDir: "/tmp/agent", + activeAgentId: "main", + provider: "fireworks", + model: "fireworks/minimax-m2p5", + defaultProvider: "fireworks", + defaultModel: "fireworks/minimax-m2p5", + aliasIndex: baseAliasIndex(), + allowedModelCatalog: [], + resetModelOverride: false, + sessionEntry: { + modelProvider: "deepinfra", + model: "moonshotai/Kimi-K2.5", + }, + }); + + expect(reply?.text).toContain("Current: fireworks/minimax-m2p5 (selected)"); + expect(reply?.text).toContain("Active: deepinfra/moonshotai/Kimi-K2.5 (runtime)"); + }); + it("auto-applies closest match for typos", () => { const directives = parseInlineDirectives("/model anthropic/claud-opus-4-5"); const cfg = { commands: { text: true } } as unknown as OpenClawConfig; diff --git a/src/auto-reply/reply/directive-handling.model.ts b/src/auto-reply/reply/directive-handling.model.ts index 40be896e34..e05b7044ed 100644 --- a/src/auto-reply/reply/directive-handling.model.ts +++ b/src/auto-reply/reply/directive-handling.model.ts @@ -7,8 +7,10 @@ import { resolveModelRefFromString, } from "../../agents/model-selection.js"; import type { OpenClawConfig } from "../../config/config.js"; +import type { SessionEntry } from "../../config/sessions.js"; import { buildBrowseProvidersButton } from "../../telegram/model-buttons.js"; import { shortenHomePath } from "../../utils.js"; +import { resolveSelectedAndActiveModel } from "../model-runtime.js"; import type { ReplyPayload } from "../types.js"; import { resolveModelsCommandReply } from "./commands-models.js"; import { @@ -198,6 +200,7 @@ export async function maybeHandleModelDirectiveInfo(params: { allowedModelCatalog: Array<{ provider: string; id?: string; name?: string }>; resetModelOverride: boolean; surface?: string; + sessionEntry?: Pick; }): Promise { if (!params.directives.hasModelDirective) { return undefined; @@ -233,31 +236,45 @@ export async function maybeHandleModelDirectiveInfo(params: { } if (wantsSummary) { - const current = `${params.provider}/${params.model}`; + const modelRefs = resolveSelectedAndActiveModel({ + selectedProvider: params.provider, + selectedModel: params.model, + sessionEntry: params.sessionEntry, + }); + const current = modelRefs.selected.label; const isTelegram = params.surface === "telegram"; + const activeRuntimeLine = modelRefs.activeDiffers + ? `Active: ${modelRefs.active.label} (runtime)` + : null; if (isTelegram) { const buttons = buildBrowseProvidersButton(); return { text: [ - `Current: ${current}`, + `Current: ${current}${modelRefs.activeDiffers ? " (selected)" : ""}`, + activeRuntimeLine, "", "Tap below to browse models, or use:", "/model to switch", "/model status for details", - ].join("\n"), + ] + .filter(Boolean) + .join("\n"), channelData: { telegram: { buttons } }, }; } return { text: [ - `Current: ${current}`, + `Current: ${current}${modelRefs.activeDiffers ? " (selected)" : ""}`, + activeRuntimeLine, "", "Switch: /model ", "Browse: /models (providers) or /models (models)", "More: /model status", - ].join("\n"), + ] + .filter(Boolean) + .join("\n"), }; } @@ -284,14 +301,20 @@ export async function maybeHandleModelDirectiveInfo(params: { authByProvider.set(provider, formatAuthLabel(auth)); } - const current = `${params.provider}/${params.model}`; + const modelRefs = resolveSelectedAndActiveModel({ + selectedProvider: params.provider, + selectedModel: params.model, + sessionEntry: params.sessionEntry, + }); + const current = modelRefs.selected.label; const defaultLabel = `${params.defaultProvider}/${params.defaultModel}`; const lines = [ - `Current: ${current}`, + `Current: ${current}${modelRefs.activeDiffers ? " (selected)" : ""}`, + modelRefs.activeDiffers ? `Active: ${modelRefs.active.label} (runtime)` : null, `Default: ${defaultLabel}`, `Agent: ${params.activeAgentId}`, `Auth file: ${formatPath(resolveAuthStorePathForDisplay(params.agentDir))}`, - ]; + ].filter((line): line is string => Boolean(line)); if (params.resetModelOverride) { lines.push(`(previous selection reset to default)`); } diff --git a/src/auto-reply/status.test.ts b/src/auto-reply/status.test.ts index 6e65d3b12a..f66c39f312 100644 --- a/src/auto-reply/status.test.ts +++ b/src/auto-reply/status.test.ts @@ -190,7 +190,7 @@ describe("buildStatusMessage", () => { expect(optionsLine).not.toContain("elevated"); }); - it("prefers model overrides over last-run model", () => { + it("shows selected model and active runtime model when they differ", () => { const text = buildStatusMessage({ agent: { model: "anthropic/claude-opus-4-5", @@ -203,15 +203,76 @@ describe("buildStatusMessage", () => { modelOverride: "gpt-4.1-mini", modelProvider: "anthropic", model: "claude-haiku-4-5", + fallbackNoticeSelectedModel: "openai/gpt-4.1-mini", + fallbackNoticeActiveModel: "anthropic/claude-haiku-4-5", + fallbackNoticeReason: "rate limit", contextTokens: 32_000, }, sessionKey: "agent:main:main", sessionScope: "per-sender", queue: { mode: "collect", depth: 0 }, modelAuth: "api-key", + activeModelAuth: "api-key di_123…abc (deepinfra:default)", + }); + + const normalized = normalizeTestText(text); + expect(normalized).toContain("Model: openai/gpt-4.1-mini"); + expect(normalized).toContain("Fallback: anthropic/claude-haiku-4-5"); + expect(normalized).toContain("(rate limit)"); + expect(normalized).not.toContain(" - Reason:"); + expect(normalized).not.toContain("Active:"); + expect(normalized).toContain("di_123...abc"); + }); + + it("omits active fallback details when runtime drift does not match fallback state", () => { + const text = buildStatusMessage({ + agent: { + model: "openai/gpt-4.1-mini", + contextTokens: 32_000, + }, + sessionEntry: { + sessionId: "runtime-drift-only", + updatedAt: 0, + modelProvider: "anthropic", + model: "claude-haiku-4-5", + fallbackNoticeSelectedModel: "fireworks/minimax-m2p5", + fallbackNoticeActiveModel: "deepinfra/moonshotai/Kimi-K2.5", + fallbackNoticeReason: "rate limit", + }, + sessionKey: "agent:main:main", + sessionScope: "per-sender", + queue: { mode: "collect", depth: 0 }, + modelAuth: "api-key", + activeModelAuth: "api-key di_123…abc (deepinfra:default)", + }); + + const normalized = normalizeTestText(text); + expect(normalized).toContain("Model: openai/gpt-4.1-mini"); + expect(normalized).not.toContain("Fallback:"); + expect(normalized).not.toContain("(rate limit)"); + }); + + it("omits active lines when runtime matches selected model", () => { + const text = buildStatusMessage({ + agent: { + model: "openai/gpt-4.1-mini", + contextTokens: 32_000, + }, + sessionEntry: { + sessionId: "selected-active-same", + updatedAt: 0, + modelProvider: "openai", + model: "gpt-4.1-mini", + fallbackNoticeReason: "unknown", + }, + sessionKey: "agent:main:main", + sessionScope: "per-sender", + queue: { mode: "collect", depth: 0 }, + modelAuth: "api-key", }); - expect(normalizeTestText(text)).toContain("Model: openai/gpt-4.1-mini"); + const normalized = normalizeTestText(text); + expect(normalized).not.toContain("Fallback:"); }); it("keeps provider prefix from configured model", () => { diff --git a/src/auto-reply/status.ts b/src/auto-reply/status.ts index 5ad02e40bb..5ce0f30618 100644 --- a/src/auto-reply/status.ts +++ b/src/auto-reply/status.ts @@ -40,6 +40,8 @@ import { type ChatCommandDefinition, } from "./commands-registry.js"; import type { CommandCategory } from "./commands-registry.types.js"; +import { resolveActiveFallbackState } from "./fallback-state.js"; +import { formatProviderModelRef, resolveSelectedAndActiveModel } from "./model-runtime.js"; import type { ElevatedLevel, ReasoningLevel, ThinkLevel, VerboseLevel } from "./thinking.js"; type AgentDefaults = NonNullable["defaults"]>; @@ -72,6 +74,7 @@ type StatusArgs = { resolvedReasoning?: ReasoningLevel; resolvedElevated?: ElevatedLevel; modelAuth?: string; + activeModelAuth?: string; usageLine?: string; timeLine?: string; queue?: QueueStatus; @@ -339,12 +342,19 @@ export function buildStatusMessage(args: StatusArgs): string { defaultProvider: DEFAULT_PROVIDER, defaultModel: DEFAULT_MODEL, }); - const provider = entry?.providerOverride ?? resolved.provider ?? DEFAULT_PROVIDER; - let model = entry?.modelOverride ?? resolved.model ?? DEFAULT_MODEL; + const selectedProvider = entry?.providerOverride ?? resolved.provider ?? DEFAULT_PROVIDER; + const selectedModel = entry?.modelOverride ?? resolved.model ?? DEFAULT_MODEL; + const modelRefs = resolveSelectedAndActiveModel({ + selectedProvider, + selectedModel, + sessionEntry: entry, + }); + let activeProvider = modelRefs.active.provider; + let activeModel = modelRefs.active.model; let contextTokens = entry?.contextTokens ?? args.agent?.contextTokens ?? - lookupContextTokens(model) ?? + lookupContextTokens(activeModel) ?? DEFAULT_CONTEXT_TOKENS; let inputTokens = entry?.inputTokens; @@ -366,8 +376,18 @@ export function buildStatusMessage(args: StatusArgs): string { if (!totalTokens || totalTokens === 0 || candidate > totalTokens) { totalTokens = candidate; } - if (!model) { - model = logUsage.model ?? model; + if (!entry?.model && logUsage.model) { + const slashIndex = logUsage.model.indexOf("/"); + if (slashIndex > 0) { + const provider = logUsage.model.slice(0, slashIndex).trim(); + const model = logUsage.model.slice(slashIndex + 1).trim(); + if (provider && model) { + activeProvider = provider; + activeModel = model; + } + } else { + activeModel = logUsage.model; + } } if (!contextTokens && logUsage.model) { contextTokens = lookupContextTokens(logUsage.model) ?? contextTokens; @@ -440,14 +460,21 @@ export function buildStatusMessage(args: StatusArgs): string { ]; const activationLine = activationParts.filter(Boolean).join(" · "); - const authMode = resolveModelAuthMode(provider, args.config); - const authLabelValue = - args.modelAuth ?? (authMode && authMode !== "unknown" ? authMode : undefined); - const showCost = authLabelValue === "api-key" || authLabelValue === "mixed"; + const activeAuthMode = resolveModelAuthMode(activeProvider, args.config); + const selectedAuthLabelValue = + args.modelAuth ?? + (() => { + const selectedAuthMode = resolveModelAuthMode(selectedProvider, args.config); + return selectedAuthMode && selectedAuthMode !== "unknown" ? selectedAuthMode : undefined; + })(); + const activeAuthLabelValue = + args.activeModelAuth ?? + (activeAuthMode && activeAuthMode !== "unknown" ? activeAuthMode : undefined); + const showCost = activeAuthLabelValue === "api-key" || activeAuthLabelValue === "mixed"; const costConfig = showCost ? resolveModelCostConfig({ - provider, - model, + provider: activeProvider, + model: activeModel, config: args.config, }) : undefined; @@ -464,9 +491,21 @@ export function buildStatusMessage(args: StatusArgs): string { : undefined; const costLabel = showCost && hasUsage ? formatUsd(cost) : undefined; - const modelLabel = model ? `${provider}/${model}` : "unknown"; - const authLabel = authLabelValue ? ` · 🔑 ${authLabelValue}` : ""; - const modelLine = `🧠 Model: ${modelLabel}${authLabel}`; + const selectedModelLabel = modelRefs.selected.label || "unknown"; + const activeModelLabel = formatProviderModelRef(activeProvider, activeModel) || "unknown"; + const fallbackState = resolveActiveFallbackState({ + selectedModelRef: selectedModelLabel, + activeModelRef: activeModelLabel, + state: entry, + }); + const selectedAuthLabel = selectedAuthLabelValue ? ` · 🔑 ${selectedAuthLabelValue}` : ""; + const modelLine = `🧠 Model: ${selectedModelLabel}${selectedAuthLabel}`; + const showFallbackAuth = activeAuthLabelValue && activeAuthLabelValue !== selectedAuthLabelValue; + const fallbackLine = fallbackState.active + ? `↪️ Fallback: ${activeModelLabel}${ + showFallbackAuth ? ` · 🔑 ${activeAuthLabelValue}` : "" + } (${fallbackState.reason ?? "selected model unavailable"})` + : null; const commit = resolveCommitHash(); const versionLine = `🦞 OpenClaw ${VERSION}${commit ? ` (${commit})` : ""}`; const usagePair = formatUsagePair(inputTokens, outputTokens); @@ -480,6 +519,7 @@ export function buildStatusMessage(args: StatusArgs): string { versionLine, args.timeLine, modelLine, + fallbackLine, usageCostLine, `📚 ${contextLine}`, mediaLine, diff --git a/src/config/sessions/types.ts b/src/config/sessions/types.ts index f729bedf5c..60fcffee29 100644 --- a/src/config/sessions/types.ts +++ b/src/config/sessions/types.ts @@ -80,6 +80,13 @@ export type SessionEntry = { totalTokensFresh?: boolean; modelProvider?: string; model?: string; + /** + * Last selected/runtime model pair for which a fallback notice was emitted. + * Used to avoid repeating the same fallback notice every turn. + */ + fallbackNoticeSelectedModel?: string; + fallbackNoticeActiveModel?: string; + fallbackNoticeReason?: string; contextTokens?: number; compactionCount?: number; memoryFlushAt?: number; diff --git a/src/gateway/server-chat.agent-events.test.ts b/src/gateway/server-chat.agent-events.test.ts index 143bdd003d..10db72bdcc 100644 --- a/src/gateway/server-chat.agent-events.test.ts +++ b/src/gateway/server-chat.agent-events.test.ts @@ -256,4 +256,141 @@ describe("agent event handler", () => { expect(payload.data?.result).toEqual(result); resetAgentRunContextForTest(); }); + + it("broadcasts fallback events to agent subscribers and node session", () => { + const { broadcast, broadcastToConnIds, nodeSendToSession, handler } = createHarness({ + resolveSessionKeyForRun: () => "session-fallback", + }); + + handler({ + runId: "run-fallback", + seq: 1, + stream: "lifecycle", + ts: Date.now(), + data: { + phase: "fallback", + selectedProvider: "fireworks", + selectedModel: "fireworks/minimax-m2p5", + activeProvider: "deepinfra", + activeModel: "moonshotai/Kimi-K2.5", + }, + }); + + expect(broadcastToConnIds).not.toHaveBeenCalled(); + const broadcastAgentCalls = broadcast.mock.calls.filter(([event]) => event === "agent"); + expect(broadcastAgentCalls).toHaveLength(1); + const payload = broadcastAgentCalls[0]?.[1] as { + sessionKey?: string; + stream?: string; + data?: Record; + }; + expect(payload.stream).toBe("lifecycle"); + expect(payload.data?.phase).toBe("fallback"); + expect(payload.sessionKey).toBe("session-fallback"); + expect(payload.data?.activeProvider).toBe("deepinfra"); + + const nodeCalls = nodeSendToSession.mock.calls.filter(([, event]) => event === "agent"); + expect(nodeCalls).toHaveLength(1); + }); + + it("remaps chat-linked lifecycle runId to client runId", () => { + const { broadcast, nodeSendToSession, chatRunState, handler } = createHarness({ + resolveSessionKeyForRun: () => "session-fallback", + }); + chatRunState.registry.add("run-fallback-internal", { + sessionKey: "session-fallback", + clientRunId: "run-fallback-client", + }); + + handler({ + runId: "run-fallback-internal", + seq: 1, + stream: "lifecycle", + ts: Date.now(), + data: { + phase: "fallback", + selectedProvider: "fireworks", + selectedModel: "fireworks/minimax-m2p5", + activeProvider: "deepinfra", + activeModel: "moonshotai/Kimi-K2.5", + }, + }); + + const broadcastAgentCalls = broadcast.mock.calls.filter(([event]) => event === "agent"); + expect(broadcastAgentCalls).toHaveLength(1); + const payload = broadcastAgentCalls[0]?.[1] as { + runId?: string; + sessionKey?: string; + stream?: string; + data?: Record; + }; + expect(payload.runId).toBe("run-fallback-client"); + expect(payload.stream).toBe("lifecycle"); + expect(payload.data?.phase).toBe("fallback"); + + const nodeCalls = nodeSendToSession.mock.calls.filter(([, event]) => event === "agent"); + expect(nodeCalls).toHaveLength(1); + const nodePayload = nodeCalls[0]?.[2] as { runId?: string }; + expect(nodePayload.runId).toBe("run-fallback-client"); + }); + + it("uses agent event sessionKey when run-context lookup cannot resolve", () => { + const { broadcast, handler } = createHarness({ + resolveSessionKeyForRun: () => undefined, + }); + + handler({ + runId: "run-fallback-session-key", + seq: 1, + stream: "lifecycle", + ts: Date.now(), + sessionKey: "session-from-event", + data: { + phase: "fallback", + selectedProvider: "fireworks", + selectedModel: "fireworks/minimax-m2p5", + activeProvider: "deepinfra", + activeModel: "moonshotai/Kimi-K2.5", + }, + }); + + const broadcastAgentCalls = broadcast.mock.calls.filter(([event]) => event === "agent"); + expect(broadcastAgentCalls).toHaveLength(1); + const payload = broadcastAgentCalls[0]?.[1] as { sessionKey?: string }; + expect(payload.sessionKey).toBe("session-from-event"); + }); + + it("remaps chat-linked tool runId for non-full verbose payloads", () => { + const { broadcastToConnIds, chatRunState, toolEventRecipients, handler } = createHarness({ + resolveSessionKeyForRun: () => "session-tool-remap", + }); + + chatRunState.registry.add("run-tool-internal", { + sessionKey: "session-tool-remap", + clientRunId: "run-tool-client", + }); + registerAgentRunContext("run-tool-internal", { + sessionKey: "session-tool-remap", + verboseLevel: "on", + }); + toolEventRecipients.add("run-tool-internal", "conn-1"); + + handler({ + runId: "run-tool-internal", + seq: 1, + stream: "tool", + ts: Date.now(), + data: { + phase: "result", + name: "exec", + toolCallId: "tool-remap-1", + result: { content: [{ type: "text", text: "secret" }] }, + }, + }); + + expect(broadcastToConnIds).toHaveBeenCalledTimes(1); + const payload = broadcastToConnIds.mock.calls[0]?.[1] as { runId?: string }; + expect(payload.runId).toBe("run-tool-client"); + resetAgentRunContextForTest(); + }); }); diff --git a/src/gateway/server-chat.ts b/src/gateway/server-chat.ts index eff7455953..a40353e5ba 100644 --- a/src/gateway/server-chat.ts +++ b/src/gateway/server-chat.ts @@ -325,12 +325,17 @@ export function createAgentEventHandler({ return (evt: AgentEventPayload) => { const chatLink = chatRunState.registry.peek(evt.runId); - const sessionKey = chatLink?.sessionKey ?? resolveSessionKeyForRun(evt.runId); + const eventSessionKey = + typeof evt.sessionKey === "string" && evt.sessionKey.trim() ? evt.sessionKey : undefined; + const sessionKey = + chatLink?.sessionKey ?? eventSessionKey ?? resolveSessionKeyForRun(evt.runId); const clientRunId = chatLink?.clientRunId ?? evt.runId; + const eventRunId = chatLink?.clientRunId ?? evt.runId; + const eventForClients = chatLink ? { ...evt, runId: eventRunId } : evt; const isAborted = chatRunState.abortedRuns.has(clientRunId) || chatRunState.abortedRuns.has(evt.runId); // Include sessionKey so Control UI can filter tool streams per session. - const agentPayload = sessionKey ? { ...evt, sessionKey } : evt; + const agentPayload = sessionKey ? { ...eventForClients, sessionKey } : eventForClients; const last = agentRunSeq.get(evt.runId) ?? 0; const isToolEvent = evt.stream === "tool"; const toolVerbose = isToolEvent ? resolveToolVerboseLevel(evt.runId, sessionKey) : "off"; @@ -341,12 +346,14 @@ export function createAgentEventHandler({ const data = evt.data ? { ...evt.data } : {}; delete data.result; delete data.partialResult; - return sessionKey ? { ...evt, sessionKey, data } : { ...evt, data }; + return sessionKey + ? { ...eventForClients, sessionKey, data } + : { ...eventForClients, data }; })() : agentPayload; if (evt.seq !== last + 1) { broadcast("agent", { - runId: evt.runId, + runId: eventRunId, stream: "error", ts: Date.now(), sessionKey, @@ -399,7 +406,7 @@ export function createAgentEventHandler({ } else { emitChatFinal( sessionKey, - evt.runId, + eventRunId, evt.seq, lifecyclePhase === "error" ? "error" : "done", evt.data?.error, diff --git a/ui/src/styles/components.css b/ui/src/styles/components.css index f38e31896c..670fc417cc 100644 --- a/ui/src/styles/components.css +++ b/ui/src/styles/components.css @@ -635,6 +635,16 @@ border-color: rgba(34, 197, 94, 0.35); } +.compaction-indicator--fallback { + color: #d97706; + border-color: rgba(217, 119, 6, 0.35); +} + +.compaction-indicator--fallback-cleared { + color: var(--ok); + border-color: rgba(34, 197, 94, 0.35); +} + @keyframes compaction-spin { to { transform: rotate(360deg); diff --git a/ui/src/ui/app-render.ts b/ui/src/ui/app-render.ts index a9ebc1d7cb..a87f9a8059 100644 --- a/ui/src/ui/app-render.ts +++ b/ui/src/ui/app-render.ts @@ -824,6 +824,7 @@ export function renderApp(state: AppViewState) { loading: state.chatLoading, sending: state.chatSending, compactionStatus: state.compactionStatus, + fallbackStatus: state.fallbackStatus, assistantAvatarUrl: chatAvatarUrl, messages: state.chatMessages, toolMessages: state.chatToolMessages, diff --git a/ui/src/ui/app-tool-stream.node.test.ts b/ui/src/ui/app-tool-stream.node.test.ts new file mode 100644 index 0000000000..4c948ecb75 --- /dev/null +++ b/ui/src/ui/app-tool-stream.node.test.ts @@ -0,0 +1,139 @@ +import { beforeAll, describe, expect, it, vi } from "vitest"; +import { handleAgentEvent, type FallbackStatus, type ToolStreamEntry } from "./app-tool-stream.ts"; + +type ToolStreamHost = Parameters[0]; +type MutableHost = ToolStreamHost & { + compactionStatus?: unknown; + compactionClearTimer?: number | null; + fallbackStatus?: FallbackStatus | null; + fallbackClearTimer?: number | null; +}; + +function createHost(overrides?: Partial): MutableHost { + return { + sessionKey: "main", + chatRunId: null, + toolStreamById: new Map(), + toolStreamOrder: [], + chatToolMessages: [], + toolStreamSyncTimer: null, + compactionStatus: null, + compactionClearTimer: null, + fallbackStatus: null, + fallbackClearTimer: null, + ...overrides, + }; +} + +describe("app-tool-stream fallback lifecycle handling", () => { + beforeAll(() => { + const globalWithWindow = globalThis as typeof globalThis & { + window?: Window & typeof globalThis; + }; + if (!globalWithWindow.window) { + globalWithWindow.window = globalThis as unknown as Window & typeof globalThis; + } + }); + + it("accepts session-scoped fallback lifecycle events when no run is active", () => { + vi.useFakeTimers(); + const host = createHost(); + + handleAgentEvent(host, { + runId: "run-1", + seq: 1, + stream: "lifecycle", + ts: Date.now(), + sessionKey: "main", + data: { + phase: "fallback", + selectedProvider: "fireworks", + selectedModel: "fireworks/minimax-m2p5", + activeProvider: "deepinfra", + activeModel: "moonshotai/Kimi-K2.5", + reasonSummary: "rate limit", + }, + }); + + expect(host.fallbackStatus?.selected).toBe("fireworks/minimax-m2p5"); + expect(host.fallbackStatus?.active).toBe("deepinfra/moonshotai/Kimi-K2.5"); + expect(host.fallbackStatus?.reason).toBe("rate limit"); + vi.useRealTimers(); + }); + + it("rejects idle fallback lifecycle events for other sessions", () => { + vi.useFakeTimers(); + const host = createHost(); + + handleAgentEvent(host, { + runId: "run-1", + seq: 1, + stream: "lifecycle", + ts: Date.now(), + sessionKey: "agent:other:main", + data: { + phase: "fallback", + selectedProvider: "fireworks", + selectedModel: "fireworks/minimax-m2p5", + activeProvider: "deepinfra", + activeModel: "moonshotai/Kimi-K2.5", + }, + }); + + expect(host.fallbackStatus).toBeNull(); + vi.useRealTimers(); + }); + + it("auto-clears fallback status after toast duration", () => { + vi.useFakeTimers(); + const host = createHost(); + + handleAgentEvent(host, { + runId: "run-1", + seq: 1, + stream: "lifecycle", + ts: Date.now(), + sessionKey: "main", + data: { + phase: "fallback", + selectedProvider: "fireworks", + selectedModel: "fireworks/minimax-m2p5", + activeProvider: "deepinfra", + activeModel: "moonshotai/Kimi-K2.5", + }, + }); + + expect(host.fallbackStatus).not.toBeNull(); + vi.advanceTimersByTime(7_999); + expect(host.fallbackStatus).not.toBeNull(); + vi.advanceTimersByTime(1); + expect(host.fallbackStatus).toBeNull(); + vi.useRealTimers(); + }); + + it("builds previous fallback label from provider + model on fallback_cleared", () => { + vi.useFakeTimers(); + const host = createHost(); + + handleAgentEvent(host, { + runId: "run-1", + seq: 1, + stream: "lifecycle", + ts: Date.now(), + sessionKey: "main", + data: { + phase: "fallback_cleared", + selectedProvider: "fireworks", + selectedModel: "fireworks/minimax-m2p5", + activeProvider: "fireworks", + activeModel: "fireworks/minimax-m2p5", + previousActiveProvider: "deepinfra", + previousActiveModel: "moonshotai/Kimi-K2.5", + }, + }); + + expect(host.fallbackStatus?.phase).toBe("cleared"); + expect(host.fallbackStatus?.previous).toBe("deepinfra/moonshotai/Kimi-K2.5"); + vi.useRealTimers(); + }); +}); diff --git a/ui/src/ui/app-tool-stream.ts b/ui/src/ui/app-tool-stream.ts index 3c7c175beb..c7f3f9085b 100644 --- a/ui/src/ui/app-tool-stream.ts +++ b/ui/src/ui/app-tool-stream.ts @@ -34,6 +34,82 @@ type ToolStreamHost = { toolStreamSyncTimer: number | null; }; +function toTrimmedString(value: unknown): string | null { + if (typeof value !== "string") { + return null; + } + const trimmed = value.trim(); + return trimmed ? trimmed : null; +} + +function resolveModelLabel(provider: unknown, model: unknown): string | null { + const modelValue = toTrimmedString(model); + if (!modelValue) { + return null; + } + const providerValue = toTrimmedString(provider); + if (providerValue) { + const prefix = `${providerValue}/`; + if (modelValue.toLowerCase().startsWith(prefix.toLowerCase())) { + const trimmedModel = modelValue.slice(prefix.length).trim(); + if (trimmedModel) { + return `${providerValue}/${trimmedModel}`; + } + } + return `${providerValue}/${modelValue}`; + } + const slashIndex = modelValue.indexOf("/"); + if (slashIndex > 0) { + const p = modelValue.slice(0, slashIndex).trim(); + const m = modelValue.slice(slashIndex + 1).trim(); + if (p && m) { + return `${p}/${m}`; + } + } + return modelValue; +} + +type FallbackAttempt = { + provider: string; + model: string; + reason: string; +}; + +function parseFallbackAttemptSummaries(value: unknown): string[] { + if (!Array.isArray(value)) { + return []; + } + return value + .map((entry) => toTrimmedString(entry)) + .filter((entry): entry is string => Boolean(entry)); +} + +function parseFallbackAttempts(value: unknown): FallbackAttempt[] { + if (!Array.isArray(value)) { + return []; + } + const out: FallbackAttempt[] = []; + for (const entry of value) { + if (!entry || typeof entry !== "object") { + continue; + } + const item = entry as Record; + const provider = toTrimmedString(item.provider); + const model = toTrimmedString(item.model); + if (!provider || !model) { + continue; + } + const reason = + toTrimmedString(item.reason)?.replace(/_/g, " ") ?? + toTrimmedString(item.code) ?? + (typeof item.status === "number" ? `HTTP ${item.status}` : null) ?? + toTrimmedString(item.error) ?? + "error"; + out.push({ provider, model, reason }); + } + return out; +} + function extractToolOutputText(value: unknown): string | null { if (!value || typeof value !== "object") { return null; @@ -167,12 +243,25 @@ export type CompactionStatus = { completedAt: number | null; }; +export type FallbackStatus = { + phase?: "active" | "cleared"; + selected: string; + active: string; + previous?: string; + reason?: string; + attempts: string[]; + occurredAt: number; +}; + type CompactionHost = ToolStreamHost & { compactionStatus?: CompactionStatus | null; compactionClearTimer?: number | null; + fallbackStatus?: FallbackStatus | null; + fallbackClearTimer?: number | null; }; const COMPACTION_TOAST_DURATION_MS = 5000; +const FALLBACK_TOAST_DURATION_MS = 8000; export function handleCompactionEvent(host: CompactionHost, payload: AgentEventPayload) { const data = payload.data ?? {}; @@ -204,6 +293,95 @@ export function handleCompactionEvent(host: CompactionHost, payload: AgentEventP } } +function resolveAcceptedSession( + host: ToolStreamHost, + payload: AgentEventPayload, + options?: { + allowSessionScopedWhenIdle?: boolean; + }, +): { accepted: boolean; sessionKey?: string } { + const sessionKey = typeof payload.sessionKey === "string" ? payload.sessionKey : undefined; + if (sessionKey && sessionKey !== host.sessionKey) { + return { accepted: false }; + } + if (!host.chatRunId && options?.allowSessionScopedWhenIdle && sessionKey) { + return { accepted: true, sessionKey }; + } + // Fallback: only accept session-less events for the active run. + if (!sessionKey && host.chatRunId && payload.runId !== host.chatRunId) { + return { accepted: false }; + } + if (host.chatRunId && payload.runId !== host.chatRunId) { + return { accepted: false }; + } + if (!host.chatRunId) { + return { accepted: false }; + } + return { accepted: true, sessionKey }; +} + +function handleLifecycleFallbackEvent(host: CompactionHost, payload: AgentEventPayload) { + const data = payload.data ?? {}; + const phase = payload.stream === "fallback" ? "fallback" : toTrimmedString(data.phase); + if (payload.stream === "lifecycle" && phase !== "fallback" && phase !== "fallback_cleared") { + return; + } + + const accepted = resolveAcceptedSession(host, payload, { allowSessionScopedWhenIdle: true }); + if (!accepted.accepted) { + return; + } + + const selected = + resolveModelLabel(data.selectedProvider, data.selectedModel) ?? + resolveModelLabel(data.fromProvider, data.fromModel); + const active = + resolveModelLabel(data.activeProvider, data.activeModel) ?? + resolveModelLabel(data.toProvider, data.toModel); + const previous = + resolveModelLabel(data.previousActiveProvider, data.previousActiveModel) ?? + toTrimmedString(data.previousActiveModel); + if (!selected || !active) { + return; + } + if (phase === "fallback" && selected === active) { + return; + } + + const reason = toTrimmedString(data.reasonSummary) ?? toTrimmedString(data.reason); + const attempts = (() => { + const summaries = parseFallbackAttemptSummaries(data.attemptSummaries); + if (summaries.length > 0) { + return summaries; + } + return parseFallbackAttempts(data.attempts).map((attempt) => { + const modelRef = resolveModelLabel(attempt.provider, attempt.model); + return `${modelRef ?? `${attempt.provider}/${attempt.model}`}: ${attempt.reason}`; + }); + })(); + + if (host.fallbackClearTimer != null) { + window.clearTimeout(host.fallbackClearTimer); + host.fallbackClearTimer = null; + } + host.fallbackStatus = { + phase: phase === "fallback_cleared" ? "cleared" : "active", + selected, + active: phase === "fallback_cleared" ? selected : active, + previous: + phase === "fallback_cleared" + ? (previous ?? (active !== selected ? active : undefined)) + : undefined, + reason: reason ?? undefined, + attempts, + occurredAt: Date.now(), + }; + host.fallbackClearTimer = window.setTimeout(() => { + host.fallbackStatus = null; + host.fallbackClearTimer = null; + }, FALLBACK_TOAST_DURATION_MS); +} + export function handleAgentEvent(host: ToolStreamHost, payload?: AgentEventPayload) { if (!payload) { return; @@ -215,23 +393,19 @@ export function handleAgentEvent(host: ToolStreamHost, payload?: AgentEventPaylo return; } + if (payload.stream === "lifecycle" || payload.stream === "fallback") { + handleLifecycleFallbackEvent(host as CompactionHost, payload); + return; + } + if (payload.stream !== "tool") { return; } - const sessionKey = typeof payload.sessionKey === "string" ? payload.sessionKey : undefined; - if (sessionKey && sessionKey !== host.sessionKey) { - return; - } - // Fallback: only accept session-less events for the active run. - if (!sessionKey && host.chatRunId && payload.runId !== host.chatRunId) { - return; - } - if (host.chatRunId && payload.runId !== host.chatRunId) { - return; - } - if (!host.chatRunId) { + const accepted = resolveAcceptedSession(host, payload); + if (!accepted.accepted) { return; } + const sessionKey = accepted.sessionKey; const data = payload.data ?? {}; const toolCallId = typeof data.toolCallId === "string" ? data.toolCallId : ""; diff --git a/ui/src/ui/app-view-state.ts b/ui/src/ui/app-view-state.ts index a484208fe3..e7c7735c8b 100644 --- a/ui/src/ui/app-view-state.ts +++ b/ui/src/ui/app-view-state.ts @@ -1,5 +1,5 @@ import type { EventLogEntry } from "./app-events.ts"; -import type { CompactionStatus } from "./app-tool-stream.ts"; +import type { CompactionStatus, FallbackStatus } from "./app-tool-stream.ts"; import type { DevicePairingList } from "./controllers/devices.ts"; import type { ExecApprovalRequest } from "./controllers/exec-approval.ts"; import type { ExecApprovalsFile, ExecApprovalsSnapshot } from "./controllers/exec-approvals.ts"; @@ -61,6 +61,7 @@ export type AppViewState = { chatStreamStartedAt: number | null; chatRunId: string | null; compactionStatus: CompactionStatus | null; + fallbackStatus: FallbackStatus | null; chatAvatarUrl: string | null; chatThinkingLevel: string | null; chatQueue: ChatQueueItem[]; diff --git a/ui/src/ui/app.ts b/ui/src/ui/app.ts index b03b6659e3..db4b290b10 100644 --- a/ui/src/ui/app.ts +++ b/ui/src/ui/app.ts @@ -48,6 +48,7 @@ import { resetToolStream as resetToolStreamInternal, type ToolStreamEntry, type CompactionStatus, + type FallbackStatus, } from "./app-tool-stream.ts"; import type { AppViewState } from "./app-view-state.ts"; import { normalizeAssistantIdentity } from "./assistant-identity.ts"; @@ -140,6 +141,7 @@ export class OpenClawApp extends LitElement { @state() chatStreamStartedAt: number | null = null; @state() chatRunId: string | null = null; @state() compactionStatus: CompactionStatus | null = null; + @state() fallbackStatus: FallbackStatus | null = null; @state() chatAvatarUrl: string | null = null; @state() chatThinkingLevel: string | null = null; @state() chatQueue: ChatQueueItem[] = []; diff --git a/ui/src/ui/views/chat.test.ts b/ui/src/ui/views/chat.test.ts index b690bb1bb1..8c3828a133 100644 --- a/ui/src/ui/views/chat.test.ts +++ b/ui/src/ui/views/chat.test.ts @@ -23,6 +23,7 @@ function createProps(overrides: Partial = {}): ChatProps { sending: false, canAbort: false, compactionStatus: null, + fallbackStatus: null, messages: [], toolMessages: [], stream: null, @@ -111,6 +112,75 @@ describe("chat view", () => { nowSpy.mockRestore(); }); + it("renders fallback indicator shortly after fallback event", () => { + const container = document.createElement("div"); + const nowSpy = vi.spyOn(Date, "now").mockReturnValue(1_000); + render( + renderChat( + createProps({ + fallbackStatus: { + selected: "fireworks/minimax-m2p5", + active: "deepinfra/moonshotai/Kimi-K2.5", + attempts: ["fireworks/minimax-m2p5: rate limit"], + occurredAt: 900, + }, + }), + ), + container, + ); + + const indicator = container.querySelector(".compaction-indicator--fallback"); + expect(indicator).not.toBeNull(); + expect(indicator?.textContent).toContain("Fallback active: deepinfra/moonshotai/Kimi-K2.5"); + nowSpy.mockRestore(); + }); + + it("hides stale fallback indicator", () => { + const container = document.createElement("div"); + const nowSpy = vi.spyOn(Date, "now").mockReturnValue(20_000); + render( + renderChat( + createProps({ + fallbackStatus: { + selected: "fireworks/minimax-m2p5", + active: "deepinfra/moonshotai/Kimi-K2.5", + attempts: [], + occurredAt: 0, + }, + }), + ), + container, + ); + + expect(container.querySelector(".compaction-indicator--fallback")).toBeNull(); + nowSpy.mockRestore(); + }); + + it("renders fallback-cleared indicator shortly after transition", () => { + const container = document.createElement("div"); + const nowSpy = vi.spyOn(Date, "now").mockReturnValue(1_000); + render( + renderChat( + createProps({ + fallbackStatus: { + phase: "cleared", + selected: "fireworks/minimax-m2p5", + active: "fireworks/minimax-m2p5", + previous: "deepinfra/moonshotai/Kimi-K2.5", + attempts: [], + occurredAt: 900, + }, + }), + ), + container, + ); + + const indicator = container.querySelector(".compaction-indicator--fallback-cleared"); + expect(indicator).not.toBeNull(); + expect(indicator?.textContent).toContain("Fallback cleared: fireworks/minimax-m2p5"); + nowSpy.mockRestore(); + }); + it("shows a stop button when aborting is available", () => { const container = document.createElement("div"); const onAbort = vi.fn(); diff --git a/ui/src/ui/views/chat.ts b/ui/src/ui/views/chat.ts index 1ef9224525..e63f56c25f 100644 --- a/ui/src/ui/views/chat.ts +++ b/ui/src/ui/views/chat.ts @@ -21,6 +21,16 @@ export type CompactionIndicatorStatus = { completedAt: number | null; }; +export type FallbackIndicatorStatus = { + phase?: "active" | "cleared"; + selected: string; + active: string; + previous?: string; + reason?: string; + attempts: string[]; + occurredAt: number; +}; + export type ChatProps = { sessionKey: string; onSessionKeyChange: (next: string) => void; @@ -30,6 +40,7 @@ export type ChatProps = { sending: boolean; canAbort?: boolean; compactionStatus?: CompactionIndicatorStatus | null; + fallbackStatus?: FallbackIndicatorStatus | null; messages: unknown[]; toolMessages: unknown[]; stream: string | null; @@ -72,6 +83,7 @@ export type ChatProps = { }; const COMPACTION_TOAST_DURATION_MS = 5000; +const FALLBACK_TOAST_DURATION_MS = 8000; function adjustTextareaHeight(el: HTMLTextAreaElement) { el.style.height = "auto"; @@ -107,6 +119,45 @@ function renderCompactionIndicator(status: CompactionIndicatorStatus | null | un return nothing; } +function renderFallbackIndicator(status: FallbackIndicatorStatus | null | undefined) { + if (!status) { + return nothing; + } + const phase = status.phase ?? "active"; + const elapsed = Date.now() - status.occurredAt; + if (elapsed >= FALLBACK_TOAST_DURATION_MS) { + return nothing; + } + const details = [ + `Selected: ${status.selected}`, + phase === "cleared" ? `Active: ${status.selected}` : `Active: ${status.active}`, + phase === "cleared" && status.previous ? `Previous fallback: ${status.previous}` : null, + status.reason ? `Reason: ${status.reason}` : null, + status.attempts.length > 0 ? `Attempts: ${status.attempts.slice(0, 3).join(" | ")}` : null, + ] + .filter(Boolean) + .join(" • "); + const message = + phase === "cleared" + ? `Fallback cleared: ${status.selected}` + : `Fallback active: ${status.active}`; + const className = + phase === "cleared" + ? "compaction-indicator compaction-indicator--fallback-cleared" + : "compaction-indicator compaction-indicator--fallback"; + const icon = phase === "cleared" ? icons.check : icons.brain; + return html` +
+ ${icon} ${message} +
+ `; +} + function generateAttachmentId(): string { return `att-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`; } @@ -352,6 +403,7 @@ export function renderChat(props: ChatProps) { : nothing } + ${renderFallbackIndicator(props.fallbackStatus)} ${renderCompactionIndicator(props.compactionStatus)} ${