github/OpenHands
1
1
Fork 0
mirror of https://github.com/All-Hands-AI/OpenHands.git synced 2026-01-09 14:57:59 -05:00
Code Issues Packages Projects Releases Wiki Activity

chore(frontend): Refactor chat interface-related event handling (#8403)

Browse Source
This commit is contained in:
sp.wack
2025-05-19 19:15:09 +04:00
committed by GitHub
parent b244138ec5
commit 14334040f1
37 changed files with 1081 additions and 1387 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
frontend/__tests__/components/chat-message.test.tsx
View File

@@ -10,11 +10,7 @@ describe("ChatMessage", () => {
expect(screen.getByText("Hello, World!")).toBeInTheDocument();
});
it("should render an assistant message", () => {
render(<ChatMessage type="assistant" message="Hello, World!" />);
expect(screen.getByTestId("assistant-message")).toBeInTheDocument();
expect(screen.getByText("Hello, World!")).toBeInTheDocument();
});
it.todo("should render an assistant message");
it.skip("should support code syntax highlighting", () => {
const code = "```js\nconsole.log('Hello, World!')\n```";
@@ -66,10 +62,7 @@ describe("ChatMessage", () => {
it("should apply correct styles to inline code", () => {
render(
<ChatMessage
type="assistant"
message="Here is some `inline code` text"
/>,
<ChatMessage type="agent" message="Here is some `inline code` text" />,
);
const codeElement = screen.getByText("inline code");

80
frontend/__tests__/components/chat/chat-interface.test.tsx
View File

@@ -1,11 +1,9 @@
import { afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import { act, screen, waitFor, within } from "@testing-library/react";
import { screen, waitFor, within } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { renderWithProviders } from "test-utils";
import type { Message } from "#/message";
import { addUserMessage } from "#/state/chat-slice";
import { SUGGESTIONS } from "#/utils/suggestions";
import * as ChatSlice from "#/state/chat-slice";
import { WsClientProviderStatus } from "#/context/ws-client-provider";
import { ChatInterface } from "#/components/features/chat/chat-interface";
@@ -42,51 +40,10 @@ describe("Empty state", () => {
vi.clearAllMocks();
});
it("should render suggestions if empty", () => {
const { store } = renderWithProviders(<ChatInterface />, {
preloadedState: {
chat: {
messages: [],
systemMessage: {
content: "",
tools: [],
openhands_version: null,
agent_class: null
}
},
},
});
expect(screen.getByTestId("suggestions")).toBeInTheDocument();
act(() => {
store.dispatch(
addUserMessage({
content: "Hello",
imageUrls: [],
timestamp: new Date().toISOString(),
pending: true,
}),
);
});
expect(screen.queryByTestId("suggestions")).not.toBeInTheDocument();
});
it.todo("should render suggestions if empty");
it("should render the default suggestions", () => {
renderWithProviders(<ChatInterface />, {
preloadedState: {
chat: {
messages: [],
systemMessage: {
content: "",
tools: [],
openhands_version: null,
agent_class: null
}
},
},
});
renderWithProviders(<ChatInterface />);
const suggestions = screen.getByTestId("suggestions");
const repoSuggestions = Object.keys(SUGGESTIONS.repo);
@@ -110,21 +67,8 @@ describe("Empty state", () => {
status: WsClientProviderStatus.CONNECTED,
isLoadingMessages: false,
}));
const addUserMessageSpy = vi.spyOn(ChatSlice, "addUserMessage");
const user = userEvent.setup();
const { store } = renderWithProviders(<ChatInterface />, {
preloadedState: {
chat: {
messages: [],
systemMessage: {
content: "",
tools: [],
openhands_version: null,
agent_class: null
}
},
},
});
renderWithProviders(<ChatInterface />);
const suggestions = screen.getByTestId("suggestions");
const displayedSuggestions = within(suggestions).getAllByRole("button");
@@ -133,9 +77,7 @@ describe("Empty state", () => {
await user.click(displayedSuggestions[0]);
// user message loaded to input
expect(addUserMessageSpy).not.toHaveBeenCalled();
expect(screen.queryByTestId("suggestions")).toBeInTheDocument();
expect(store.getState().chat.messages).toHaveLength(0);
expect(input).toHaveValue(displayedSuggestions[0].textContent);
},
);
@@ -149,19 +91,7 @@ describe("Empty state", () => {
isLoadingMessages: false,
}));
const user = userEvent.setup();
const { rerender } = renderWithProviders(<ChatInterface />, {
preloadedState: {
chat: {
messages: [],
systemMessage: {
content: "",
tools: [],
openhands_version: null,
agent_class: null
}
},
},
});
const { rerender } = renderWithProviders(<ChatInterface />);
const suggestions = screen.getByTestId("suggestions");
const displayedSuggestions = within(suggestions).getAllByRole("button");

91
frontend/__tests__/components/file-operations.test.tsx
View File

@@ -1,92 +1,11 @@
import { render, screen } from "@testing-library/react";
import { describe, it, expect, vi } from "vitest";
import { Messages } from "#/components/features/chat/messages";
import type { Message } from "#/message";
import { renderWithProviders } from "test-utils";
// Mock the useParams hook to provide a conversationId
vi.mock("react-router", async () => {
const actual = await vi.importActual<typeof import("react-router")>("react-router");
return {
...actual,
useParams: () => ({ conversationId: "test-conversation-id" }),
};
});
import { describe, it } from "vitest";
describe("File Operations Messages", () => {
it("should show success indicator for successful file read operation", () => {
const messages: Message[] = [
{
type: "action",
translationID: "read_file_contents",
content: "Successfully read file contents",
success: true,
sender: "assistant",
timestamp: new Date().toISOString(),
},
];
it.todo("should show success indicator for successful file read operation");
renderWithProviders(<Messages messages={messages} isAwaitingUserConfirmation={false} />);
it.todo("should show failure indicator for failed file read operation");
const statusIcon = screen.getByTestId("status-icon");
expect(statusIcon).toBeInTheDocument();
expect(statusIcon.closest("svg")).toHaveClass("fill-success");
});
it.todo("should show success indicator for successful file edit operation");
it("should show failure indicator for failed file read operation", () => {
const messages: Message[] = [
{
type: "action",
translationID: "read_file_contents",
content: "Failed to read file contents",
success: false,
sender: "assistant",
timestamp: new Date().toISOString(),
},
];
renderWithProviders(<Messages messages={messages} isAwaitingUserConfirmation={false} />);
const statusIcon = screen.getByTestId("status-icon");
expect(statusIcon).toBeInTheDocument();
expect(statusIcon.closest("svg")).toHaveClass("fill-danger");
});
it("should show success indicator for successful file edit operation", () => {
const messages: Message[] = [
{
type: "action",
translationID: "edit_file_contents",
content: "Successfully edited file contents",
success: true,
sender: "assistant",
timestamp: new Date().toISOString(),
},
];
renderWithProviders(<Messages messages={messages} isAwaitingUserConfirmation={false} />);
const statusIcon = screen.getByTestId("status-icon");
expect(statusIcon).toBeInTheDocument();
expect(statusIcon.closest("svg")).toHaveClass("fill-success");
});
it("should show failure indicator for failed file edit operation", () => {
const messages: Message[] = [
{
type: "action",
translationID: "edit_file_contents",
content: "Failed to edit file contents",
success: false,
sender: "assistant",
timestamp: new Date().toISOString(),
},
];
renderWithProviders(<Messages messages={messages} isAwaitingUserConfirmation={false} />);
const statusIcon = screen.getByTestId("status-icon");
expect(statusIcon).toBeInTheDocument();
expect(statusIcon.closest("svg")).toHaveClass("fill-danger");
});
it.todo("should show failure indicator for failed file edit operation");
});

32
frontend/__tests__/context/ws-client-provider.test.tsx
View File

@@ -2,7 +2,6 @@ import { describe, it, expect, vi, beforeEach } from "vitest";
import { render, waitFor } from "@testing-library/react";
import React from "react";
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
import * as ChatSlice from "#/state/chat-slice";
import {
updateStatusWhenErrorMessagePresent,
WsClientProvider,
@@ -11,42 +10,15 @@ import {
describe("Propagate error message", () => {
it("should do nothing when no message was passed from server", () => {
const addErrorMessageSpy = vi.spyOn(ChatSlice, "addErrorMessage");
updateStatusWhenErrorMessagePresent(null);
updateStatusWhenErrorMessagePresent(undefined);
updateStatusWhenErrorMessagePresent({});
updateStatusWhenErrorMessagePresent({ message: null });
expect(addErrorMessageSpy).not.toHaveBeenCalled();
});
it("should display error to user when present", () => {
const message = "We have a problem!";
const addErrorMessageSpy = vi.spyOn(ChatSlice, "addErrorMessage");
updateStatusWhenErrorMessagePresent({ message });
it.todo("should display error to user when present");
expect(addErrorMessageSpy).toHaveBeenCalledWith({
message,
status_update: true,
type: "error",
});
});
it("should display error including translation id when present", () => {
const message = "We have a problem!";
const addErrorMessageSpy = vi.spyOn(ChatSlice, "addErrorMessage");
updateStatusWhenErrorMessagePresent({
message,
data: { msg_id: "..id.." },
});
expect(addErrorMessageSpy).toHaveBeenCalledWith({
message,
id: "..id..",
status_update: true,
type: "error",
});
});
it.todo("should display error including translation id when present");
});
// Create a mock for socket.io-client

146
frontend/__tests__/services/actions.test.ts
View File

@@ -1,146 +0,0 @@
import { describe, it, expect, vi, beforeEach } from "vitest";
import { handleStatusMessage, handleActionMessage } from "#/services/actions";
import store from "#/store";
import { trackError } from "#/utils/error-handler";
import ActionType from "#/types/action-type";
import { ActionMessage } from "#/types/message";
// Mock dependencies
vi.mock("#/utils/error-handler", () => ({
trackError: vi.fn(),
}));
vi.mock("#/store", () => ({
default: {
dispatch: vi.fn(),
},
}));
describe("Actions Service", () => {
beforeEach(() => {
vi.clearAllMocks();
});
describe("handleStatusMessage", () => {
it("should dispatch info messages to status state", () => {
const message = {
type: "info",
message: "Runtime is not available",
id: "runtime.unavailable",
status_update: true as const,
};
handleStatusMessage(message);
expect(store.dispatch).toHaveBeenCalledWith(expect.objectContaining({
payload: message,
}));
});
it("should log error messages and display them in chat", () => {
const message = {
type: "error",
message: "Runtime connection failed",
id: "runtime.connection.failed",
status_update: true as const,
};
handleStatusMessage(message);
expect(trackError).toHaveBeenCalledWith({
message: "Runtime connection failed",
source: "chat",
metadata: { msgId: "runtime.connection.failed" },
});
expect(store.dispatch).toHaveBeenCalledWith(expect.objectContaining({
payload: message,
}));
});
});
describe("handleActionMessage", () => {
it("should use first-person perspective for task completion messages", () => {
// Test partial completion
const messagePartial: ActionMessage = {
id: 1,
action: ActionType.FINISH,
source: "agent",
message: "",
timestamp: new Date().toISOString(),
args: {
final_thought: "",
task_completed: "partial",
outputs: "",
thought: ""
}
};
// Mock implementation to capture the message
let capturedPartialMessage = "";
(store.dispatch as any).mockImplementation((action: any) => {
if (action.type === "chat/addAssistantMessage" &&
action.payload.includes("believe that the task was **completed partially**")) {
capturedPartialMessage = action.payload;
}
});
handleActionMessage(messagePartial);
expect(capturedPartialMessage).toContain("I believe that the task was **completed partially**");
// Test not completed
const messageNotCompleted: ActionMessage = {
id: 2,
action: ActionType.FINISH,
source: "agent",
message: "",
timestamp: new Date().toISOString(),
args: {
final_thought: "",
task_completed: "false",
outputs: "",
thought: ""
}
};
// Mock implementation to capture the message
let capturedNotCompletedMessage = "";
(store.dispatch as any).mockImplementation((action: any) => {
if (action.type === "chat/addAssistantMessage" &&
action.payload.includes("believe that the task was **not completed**")) {
capturedNotCompletedMessage = action.payload;
}
});
handleActionMessage(messageNotCompleted);
expect(capturedNotCompletedMessage).toContain("I believe that the task was **not completed**");
// Test completed successfully
const messageCompleted: ActionMessage = {
id: 3,
action: ActionType.FINISH,
source: "agent",
message: "",
timestamp: new Date().toISOString(),
args: {
final_thought: "",
task_completed: "true",
outputs: "",
thought: ""
}
};
// Mock implementation to capture the message
let capturedCompletedMessage = "";
(store.dispatch as any).mockImplementation((action: any) => {
if (action.type === "chat/addAssistantMessage" &&
action.payload.includes("believe that the task was **completed successfully**")) {
capturedCompletedMessage = action.payload;
}
});
handleActionMessage(messageCompleted);
expect(capturedCompletedMessage).toContain("I believe that the task was **completed successfully**");
});
});
});

51
frontend/__tests__/services/observations.test.ts
View File

@@ -1,51 +0,0 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { handleObservationMessage } from "#/services/observations";
import store from "#/store";
import { ObservationMessage } from "#/types/message";
// Mock dependencies
vi.mock("#/store", () => ({
default: {
dispatch: vi.fn(),
},
}));
describe("Observations Service", () => {
beforeEach(() => {
vi.clearAllMocks();
});
describe("handleObservationMessage", () => {
const createErrorMessage = (): ObservationMessage => ({
id: 14,
timestamp: "2025-04-14T13:37:54.451843",
message: "The action has not been executed.",
cause: 12,
observation: "error",
content: "The action has not been executed.",
extras: {
error_id: "",
metadata: {},
},
});
it("should dispatch error messages exactly once", () => {
const errorMessage = createErrorMessage();
handleObservationMessage(errorMessage);
expect(store.dispatch).toHaveBeenCalledTimes(1);
expect(store.dispatch).toHaveBeenCalledWith({
type: "chat/addAssistantObservation",
payload: expect.objectContaining({
observation: "error",
content: "The action has not been executed.",
source: "user",
extras: {
error_id: "",
},
}),
});
});
});
});

48
frontend/__tests__/services/observations.test.tsx
View File

@@ -1,8 +1,4 @@
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
import { handleObservationMessage } from "#/services/observations";
import { setScreenshotSrc, setUrl } from "#/state/browser-slice";
import ObservationType from "#/types/observation-type";
import store from "#/store";
import { describe, it, vi, beforeEach, afterEach } from "vitest";
// Mock the store module
vi.mock("#/store", () => ({
@@ -20,43 +16,9 @@ describe("handleObservationMessage", () => {
vi.resetAllMocks();
});
it("updates browser state when receiving a browse observation", () => {
const message = {
id: "test-id",
cause: "test-cause",
observation: ObservationType.BROWSE,
content: "test content",
message: "test message",
extras: {
url: "https://example.com",
screenshot: "base64-screenshot-data",
},
};
it.todo("updates browser state when receiving a browse observation");
handleObservationMessage(message);
// Check that setScreenshotSrc and setUrl were called with the correct values
expect(store.dispatch).toHaveBeenCalledWith(setScreenshotSrc("base64-screenshot-data"));
expect(store.dispatch).toHaveBeenCalledWith(setUrl("https://example.com"));
});
it("updates browser state when receiving a browse_interactive observation", () => {
const message = {
id: "test-id",
cause: "test-cause",
observation: ObservationType.BROWSE_INTERACTIVE,
content: "test content",
message: "test message",
extras: {
url: "https://example.com",
screenshot: "base64-screenshot-data",
},
};
handleObservationMessage(message);
// Check that setScreenshotSrc and setUrl were called with the correct values
expect(store.dispatch).toHaveBeenCalledWith(setScreenshotSrc("base64-screenshot-data"));
expect(store.dispatch).toHaveBeenCalledWith(setUrl("https://example.com"));
});
it.todo(
"updates browser state when receiving a browse_interactive observation",
);
});

37
frontend/src/components/features/chat/chat-interface.tsx
View File

@@ -1,4 +1,4 @@
import { useDispatch, useSelector } from "react-redux";
import { useSelector } from "react-redux";
import React from "react";
import posthog from "posthog-js";
import { useParams } from "react-router";
@@ -8,7 +8,6 @@ import { convertImageToBase64 } from "#/utils/convert-image-to-base-64";
import { TrajectoryActions } from "../trajectory/trajectory-actions";
import { createChatMessage } from "#/services/chat-service";
import { InteractiveChatBox } from "./interactive-chat-box";
import { addUserMessage } from "#/state/chat-slice";
import { RootState } from "#/store";
import { AgentState } from "#/types/agent-state";
import { generateAgentStateChangeEvent } from "#/services/agent-state-service";
@@ -25,6 +24,11 @@ import { LoadingSpinner } from "#/components/shared/loading-spinner";
import { useGetTrajectory } from "#/hooks/mutation/use-get-trajectory";
import { downloadTrajectory } from "#/utils/download-trajectory";
import { displayErrorToast } from "#/utils/custom-toast-handlers";
import { useOptimisticUserMessage } from "#/hooks/use-optimistic-user-message";
import { useWSErrorMessage } from "#/hooks/use-ws-error-message";
import i18n from "#/i18n";
import { ErrorMessageBanner } from "./error-message-banner";
import { shouldRenderEvent } from "./event-content-helpers/should-render-event";
function getEntryPoint(
hasRepository: boolean | null,
@@ -36,14 +40,15 @@ function getEntryPoint(
}
export function ChatInterface() {
const { send, isLoadingMessages } = useWsClient();
const dispatch = useDispatch();
const { getErrorMessage } = useWSErrorMessage();
const { send, isLoadingMessages, parsedEvents } = useWsClient();
const { setOptimisticUserMessage, getOptimisticUserMessage } =
useOptimisticUserMessage();
const { t } = useTranslation();
const scrollRef = React.useRef<HTMLDivElement>(null);
const { scrollDomToBottom, onChatBodyScroll, hitBottom } =
useScrollToBottom(scrollRef);
const { messages } = useSelector((state: RootState) => state.chat);
const { curAgentState } = useSelector((state: RootState) => state.agent);
const [feedbackPolarity, setFeedbackPolarity] = React.useState<
@@ -57,8 +62,13 @@ export function ChatInterface() {
const params = useParams();
const { mutate: getTrajectory } = useGetTrajectory();
const optimisticUserMessage = getOptimisticUserMessage();
const errorMessage = getErrorMessage();
const events = parsedEvents.filter(shouldRenderEvent);
const handleSendMessage = async (content: string, files: File[]) => {
if (messages.length === 0) {
if (events.length === 0) {
posthog.capture("initial_query_submitted", {
entry_point: getEntryPoint(
selectedRepository !== null,
@@ -69,7 +79,7 @@ export function ChatInterface() {
});
} else {
posthog.capture("user_message_sent", {
session_message_count: messages.length,
session_message_count: events.length,
current_message_length: content.length,
});
}
@@ -77,9 +87,8 @@ export function ChatInterface() {
const imageUrls = await Promise.all(promises);
const timestamp = new Date().toISOString();
const pending = true;
dispatch(addUserMessage({ content, imageUrls, timestamp, pending }));
send(createChatMessage(content, imageUrls, timestamp));
setOptimisticUserMessage(content);
setMessageToSend(null);
};
@@ -120,7 +129,7 @@ export function ChatInterface() {
return (
<div className="h-full flex flex-col justify-between">
{messages.length === 0 && (
{events.length === 0 && !optimisticUserMessage && (
<ChatSuggestions onSuggestionsClick={setMessageToSend} />
)}
@@ -137,7 +146,7 @@ export function ChatInterface() {
{!isLoadingMessages && (
<Messages
messages={messages}
messages={events}
isAwaitingUserConfirmation={
curAgentState === AgentState.AWAITING_USER_CONFIRMATION
}
@@ -170,6 +179,12 @@ export function ChatInterface() {
{!hitBottom && <ScrollToBottomButton onClick={scrollDomToBottom} />}
</div>
{errorMessage && (
<ErrorMessageBanner
message={i18n.exists(errorMessage) ? t(errorMessage) : errorMessage}
/>
)}
<InteractiveChatBox
onSubmit={handleSendMessage}
onStop={handleStop}

5
frontend/src/components/features/chat/chat-message.tsx
View File

@@ -6,10 +6,11 @@ import { cn } from "#/utils/utils";
import { ul, ol } from "../markdown/list";
import { CopyToClipboardButton } from "#/components/shared/buttons/copy-to-clipboard-button";
import { anchor } from "../markdown/anchor";
import { OpenHandsSourceType } from "#/types/core/base";
import { paragraph } from "../markdown/paragraph";
interface ChatMessageProps {
type: "user" | "assistant";
type: OpenHandsSourceType;
message: string;
}
@@ -49,7 +50,7 @@ export function ChatMessage({
"rounded-xl relative",
"flex flex-col gap-2",
type === "user" && " max-w-[305px] p-4 bg-tertiary self-end",
type === "assistant" && "mt-6 max-w-full bg-transparent",
type === "agent" && "mt-6 max-w-full bg-transparent",
)}
>
<CopyToClipboardButton

11
frontend/src/components/features/chat/error-message-banner.tsx Normal file
View File

@@ -0,0 +1,11 @@
interface ErrorMessageBannerProps {
message: string;
}
export function ErrorMessageBanner({ message }: ErrorMessageBannerProps) {
return (
<div className="w-full rounded-lg p-2 text-black border border-red-800 bg-red-500">
{message}
</div>
);
}

56
frontend/src/components/features/chat/error-message.tsx Normal file
View File

@@ -0,0 +1,56 @@
import React from "react";
import Markdown from "react-markdown";
import remarkGfm from "remark-gfm";
import { useTranslation } from "react-i18next";
import { code } from "../markdown/code";
import { ol, ul } from "../markdown/list";
import ArrowDown from "#/icons/angle-down-solid.svg?react";
import ArrowUp from "#/icons/angle-up-solid.svg?react";
import i18n from "#/i18n";
interface ErrorMessageProps {
errorId?: string;
defaultMessage: string;
}
export function ErrorMessage({ errorId, defaultMessage }: ErrorMessageProps) {
const { t } = useTranslation();
const [showDetails, setShowDetails] = React.useState(false);
const hasValidTranslationId = !!errorId && i18n.exists(errorId);
const errorKey = hasValidTranslationId
? errorId
: "CHAT_INTERFACE$AGENT_ERROR_MESSAGE";
return (
<div className="flex flex-col gap-2 border-l-2 pl-2 my-2 py-2 border-danger text-sm w-full">
<div className="font-bold text-danger">
{t(errorKey)}
<button
type="button"
onClick={() => setShowDetails((prev) => !prev)}
className="cursor-pointer text-left"
>
{showDetails ? (
<ArrowUp className="h-4 w-4 ml-2 inline fill-danger" />
) : (
<ArrowDown className="h-4 w-4 ml-2 inline fill-danger" />
)}
</button>
</div>
{showDetails && (
<Markdown
components={{
code,
ul,
ol,
}}
remarkPlugins={[remarkGfm]}
>
{defaultMessage}
</Markdown>
)}
</div>
);
}

125
frontend/src/components/features/chat/event-content-helpers/get-action-content.ts Normal file
View File

@@ -0,0 +1,125 @@
import { ActionSecurityRisk } from "#/state/security-analyzer-slice";
import {
FileWriteAction,
CommandAction,
IPythonAction,
BrowseAction,
BrowseInteractiveAction,
MCPAction,
ThinkAction,
OpenHandsAction,
FinishAction,
} from "#/types/core/actions";
import { getDefaultEventContent, MAX_CONTENT_LENGTH } from "./shared";
const getRiskText = (risk: ActionSecurityRisk) => {
switch (risk) {
case ActionSecurityRisk.LOW:
return "Low Risk";
case ActionSecurityRisk.MEDIUM:
return "Medium Risk";
case ActionSecurityRisk.HIGH:
return "High Risk";
case ActionSecurityRisk.UNKNOWN:
default:
return "Unknown Risk";
}
};
const getWriteActionContent = (event: FileWriteAction): string => {
let { content } = event.args;
if (content.length > MAX_CONTENT_LENGTH) {
content = `${event.args.content.slice(0, MAX_CONTENT_LENGTH)}...`;
}
return `${event.args.path}\n${content}`;
};
const getRunActionContent = (event: CommandAction): string => {
let content = `Command:\n\`${event.args.command}\``;
if (event.args.confirmation_state === "awaiting_confirmation") {
content += `\n\n${getRiskText(event.args.security_risk)}`;
}
return content;
};
const getIPythonActionContent = (event: IPythonAction): string => {
let content = `\`\`\`\n${event.args.code}\n\`\`\``;
if (event.args.confirmation_state === "awaiting_confirmation") {
content += `\n\n${getRiskText(event.args.security_risk)}`;
}
return content;
};
const getBrowseActionContent = (event: BrowseAction): string =>
`Browsing ${event.args.url}`;
const getBrowseInteractiveActionContent = (event: BrowseInteractiveAction) =>
`**Action:**\n\n\`\`\`python\n${event.args.browser_actions}\n\`\`\``;
const getMcpActionContent = (event: MCPAction): string => {
// Format MCP action with name and arguments
const name = event.args.name || "";
const args = event.args.arguments || {};
let details = `**MCP Tool Call:** ${name}\n\n`;
// Include thought if available
if (event.args.thought) {
details += `\n\n**Thought:**\n${event.args.thought}`;
}
details += `\n\n**Arguments:**\n\`\`\`json\n${JSON.stringify(args, null, 2)}\n\`\`\``;
return details;
};
const getThinkActionContent = (event: ThinkAction): string =>
event.args.thought;
const getFinishActionContent = (event: FinishAction): string => {
let content = event.args.final_thought;
switch (event.args.task_completed) {
case "success":
content +=
"\n\n\nI believe that the task was **completed successfully**.";
break;
case "failure":
content += "\n\n\nI believe that the task was **not completed**.";
break;
case "partial":
default:
content += "\n\n\nI believe that the task was **completed partially**.";
break;
}
return content.trim();
};
const getNoContentActionContent = (): string => "";
export const getActionContent = (event: OpenHandsAction): string => {
switch (event.action) {
case "read":
case "edit":
return getNoContentActionContent();
case "write":
return getWriteActionContent(event);
case "run":
return getRunActionContent(event);
case "run_ipython":
return getIPythonActionContent(event);
case "browse":
return getBrowseActionContent(event);
case "browse_interactive":
return getBrowseInteractiveActionContent(event);
case "call_tool_mcp":
return getMcpActionContent(event);
case "think":
return getThinkActionContent(event);
case "finish":
return getFinishActionContent(event);
default:
return getDefaultEventContent(event);
}
};

70
frontend/src/components/features/chat/event-content-helpers/get-event-content.tsx Normal file
View File

@@ -0,0 +1,70 @@
import { Trans } from "react-i18next";
import { OpenHandsAction } from "#/types/core/actions";
import { isOpenHandsAction, isOpenHandsObservation } from "#/types/core/guards";
import { OpenHandsObservation } from "#/types/core/observations";
import { MonoComponent } from "../mono-component";
import { PathComponent } from "../path-component";
import { getActionContent } from "./get-action-content";
import { getObservationContent } from "./get-observation-content";
const hasPathProperty = (
obj: Record<string, unknown>,
): obj is { path: string } => typeof obj.path === "string";
const hasCommandProperty = (
obj: Record<string, unknown>,
): obj is { command: string } => typeof obj.command === "string";
const trimText = (text: string, maxLength: number): string => {
if (!text) return "";
return text.length > maxLength ? `${text.substring(0, maxLength)}...` : text;
};
export const getEventContent = (
event: OpenHandsAction | OpenHandsObservation,
) => {
let title: React.ReactNode = "";
let details: string = "";
if (isOpenHandsAction(event)) {
title = (
<Trans
i18nKey={`ACTION_MESSAGE$${event.action.toUpperCase()}`}
values={{
path: hasPathProperty(event.args) && event.args.path,
command:
hasCommandProperty(event.args) && trimText(event.args.command, 80),
}}
components={{
path: <PathComponent />,
cmd: <MonoComponent />,
}}
/>
);
details = getActionContent(event);
}
if (isOpenHandsObservation(event)) {
title = (
<Trans
i18nKey={`OBSERVATION_MESSAGE$${event.observation.toUpperCase()}`}
values={{
path: hasPathProperty(event.extras) && event.extras.path,
command:
hasCommandProperty(event.extras) &&
trimText(event.extras.command, 80),
}}
components={{
path: <PathComponent />,
cmd: <MonoComponent />,
}}
/>
);
details = getObservationContent(event);
}
return {
title: title ?? "Unknown event",
details: details ?? "Unknown event",
};
};

133
frontend/src/components/features/chat/event-content-helpers/get-observation-content.ts Normal file
View File

@@ -0,0 +1,133 @@
import {
ReadObservation,
CommandObservation,
IPythonObservation,
EditObservation,
BrowseObservation,
OpenHandsObservation,
RecallObservation,
} from "#/types/core/observations";
import { getObservationResult } from "./get-observation-result";
import { getDefaultEventContent, MAX_CONTENT_LENGTH } from "./shared";
const getReadObservationContent = (event: ReadObservation): string =>
`\`\`\`\n${event.content}\n\`\`\``;
const getCommandObservationContent = (
event: CommandObservation | IPythonObservation,
): string => {
let { content } = event;
if (content.length > MAX_CONTENT_LENGTH) {
content = `${content.slice(0, MAX_CONTENT_LENGTH)}...`;
}
return `Output:\n\`\`\`sh\n${content.trim() || "[Command finished execution with no output]"}\n\`\`\``;
};
const getEditObservationContent = (
event: EditObservation,
successMessage: boolean,
): string => {
if (successMessage) {
return `\`\`\`diff\n${event.extras.diff}\n\`\`\``; // Content is already truncated by the ACI
}
return event.content;
};
const getBrowseObservationContent = (event: BrowseObservation) => {
let contentDetails = `**URL:** ${event.extras.url}\n`;
if (event.extras.error) {
contentDetails += `\n\n**Error:**\n${event.extras.error}\n`;
}
contentDetails += `\n\n**Output:**\n${event.content}`;
if (contentDetails.length > MAX_CONTENT_LENGTH) {
contentDetails = `${contentDetails.slice(0, MAX_CONTENT_LENGTH)}...(truncated)`;
}
return contentDetails;
};
const getMcpObservationContent = (event: OpenHandsObservation): string => {
let { content } = event;
if (content.length > MAX_CONTENT_LENGTH) {
content = `${content.slice(0, MAX_CONTENT_LENGTH)}...`;
}
return `**Output:**\n\`\`\`\n${content.trim() || "[MCP Tool finished execution with no output]"}\n\`\`\``;
};
const getRecallObservationContent = (event: RecallObservation): string => {
let content = "";
if (event.extras.recall_type === "workspace_context") {
if (event.extras.repo_name) {
content += `\n\n**Repository:** ${event.extras.repo_name}`;
}
if (event.extras.repo_directory) {
content += `\n\n**Directory:** ${event.extras.repo_directory}`;
}
if (event.extras.date) {
content += `\n\n**Date:** ${event.extras.date}`;
}
if (
event.extras.runtime_hosts &&
Object.keys(event.extras.runtime_hosts).length > 0
) {
content += `\n\n**Available Hosts**`;
for (const [host, port] of Object.entries(event.extras.runtime_hosts)) {
content += `\n\n- ${host} (port ${port})`;
}
}
if (event.extras.repo_instructions) {
content += `\n\n**Repository Instructions:**\n\n${event.extras.repo_instructions}`;
}
if (event.extras.additional_agent_instructions) {
content += `\n\n**Additional Instructions:**\n\n${event.extras.additional_agent_instructions}`;
}
}
// Handle microagent knowledge
if (
event.extras.microagent_knowledge &&
event.extras.microagent_knowledge.length > 0
) {
content += `\n\n**Triggered Microagent Knowledge:**`;
for (const knowledge of event.extras.microagent_knowledge) {
content += `\n\n- **${knowledge.name}** (triggered by keyword: ${knowledge.trigger})\n\n\`\`\`\n${knowledge.content}\n\`\`\``;
}
}
if (
event.extras.custom_secrets_descriptions &&
Object.keys(event.extras.custom_secrets_descriptions).length > 0
) {
content += `\n\n**Custom Secrets**`;
for (const [name, description] of Object.entries(
event.extras.custom_secrets_descriptions,
)) {
content += `\n\n- $${name}: ${description}`;
}
}
return content;
};
export const getObservationContent = (event: OpenHandsObservation): string => {
switch (event.observation) {
case "read":
return getReadObservationContent(event);
case "edit":
return getEditObservationContent(
event,
getObservationResult(event) === "success",
);
case "run_ipython":
case "run":
return getCommandObservationContent(event);
case "browse":
return getBrowseObservationContent(event);
case "mcp":
return getMcpObservationContent(event);
case "recall":
return getRecallObservationContent(event);
default:
return getDefaultEventContent(event);
}
};

26
frontend/src/components/features/chat/event-content-helpers/get-observation-result.ts Normal file
View File

@@ -0,0 +1,26 @@
import { OpenHandsObservation } from "#/types/core/observations";
export type ObservationResultStatus = "success" | "error" | "timeout";
export const getObservationResult = (event: OpenHandsObservation) => {
const hasContent = event.content.length > 0;
const contentIncludesError = event.content.toLowerCase().includes("error:");
switch (event.observation) {
case "run": {
const exitCode = event.extras.metadata.exit_code;
if (exitCode === -1) return "timeout"; // Command timed out
if (exitCode === 0) return "success"; // Command executed successfully
return "error"; // Command failed
}
case "run_ipython":
case "read":
case "edit":
case "mcp":
if (!hasContent || contentIncludesError) return "error";
return "success"; // Content is valid
default:
return "success";
}
};

8
frontend/src/components/features/chat/event-content-helpers/shared.ts Normal file
View File

@@ -0,0 +1,8 @@
import { OpenHandsAction } from "#/types/core/actions";
import { OpenHandsObservation } from "#/types/core/observations";
export const MAX_CONTENT_LENGTH = 1000;
export const getDefaultEventContent = (
event: OpenHandsAction | OpenHandsObservation,
): string => `\`\`\`json\n${JSON.stringify(event, null, 2)}\n\`\`\``;

27
frontend/src/components/features/chat/event-content-helpers/should-render-event.ts Normal file
View File

@@ -0,0 +1,27 @@
import { OpenHandsAction } from "#/types/core/actions";
import { OpenHandsEventType } from "#/types/core/base";
import { isOpenHandsAction, isOpenHandsObservation } from "#/types/core/guards";
import { OpenHandsObservation } from "#/types/core/observations";
const COMMON_NO_RENDER_LIST: OpenHandsEventType[] = [
"system",
"agent_state_changed",
"change_agent_state",
];
const ACTION_NO_RENDER_LIST: OpenHandsEventType[] = ["recall"];
export const shouldRenderEvent = (
event: OpenHandsAction | OpenHandsObservation,
) => {
if (isOpenHandsAction(event)) {
const noRenderList = COMMON_NO_RENDER_LIST.concat(ACTION_NO_RENDER_LIST);
return !noRenderList.includes(event.action);
}
if (isOpenHandsObservation(event)) {
return !COMMON_NO_RENDER_LIST.includes(event.observation);
}
return true;
};

123
frontend/src/components/features/chat/event-message.tsx Normal file
View File

@@ -0,0 +1,123 @@
import { ConfirmationButtons } from "#/components/shared/buttons/confirmation-buttons";
import { I18nKey } from "#/i18n/declaration";
import { OpenHandsAction } from "#/types/core/actions";
import {
isUserMessage,
isErrorObservation,
isAssistantMessage,
isOpenHandsAction,
isOpenHandsObservation,
isFinishAction,
isRejectObservation,
} from "#/types/core/guards";
import { OpenHandsObservation } from "#/types/core/observations";
import { ImageCarousel } from "../images/image-carousel";
import { ChatMessage } from "./chat-message";
import { ErrorMessage } from "./error-message";
import { getObservationResult } from "./event-content-helpers/get-observation-result";
import { getEventContent } from "./event-content-helpers/get-event-content";
import { ExpandableMessage } from "./expandable-message";
import { GenericEventMessage } from "./generic-event-message";
const hasThoughtProperty = (
obj: Record<string, unknown>,
): obj is { thought: string } => "thought" in obj && !!obj.thought;
interface EventMessageProps {
event: OpenHandsAction | OpenHandsObservation;
hasObservationPair: boolean;
isFirstMessageWithResolverTrigger: boolean;
isAwaitingUserConfirmation: boolean;
isLastMessage: boolean;
}
export function EventMessage({
event,
hasObservationPair,
isFirstMessageWithResolverTrigger,
isAwaitingUserConfirmation,
isLastMessage,
}: EventMessageProps) {
const shouldShowConfirmationButtons =
isLastMessage && event.source === "agent" && isAwaitingUserConfirmation;
const isFirstUserMessageWithResolverTrigger =
isFirstMessageWithResolverTrigger && isUserMessage(event);
// Special case: First user message with resolver trigger
if (isFirstUserMessageWithResolverTrigger) {
return (
<div>
<ExpandableMessage
type="action"
message={event.args.content}
id={I18nKey.CHAT$RESOLVER_INSTRUCTIONS}
/>
{event.args.image_urls && event.args.image_urls.length > 0 && (
<ImageCarousel size="small" images={event.args.image_urls} />
)}
</div>
);
}
if (isErrorObservation(event)) {
return (
<ErrorMessage
errorId={event.extras.error_id}
defaultMessage={event.message}
/>
);
}
if (
hasObservationPair &&
isOpenHandsAction(event) &&
hasThoughtProperty(event.args)
) {
return <ChatMessage type="agent" message={event.args.thought} />;
}
if (isFinishAction(event)) {
return (
<ChatMessage type="agent" message={getEventContent(event).details} />
);
}
if (isUserMessage(event) || isAssistantMessage(event)) {
return (
<ChatMessage
type={event.source}
message={isUserMessage(event) ? event.args.content : event.message}
>
{event.args.image_urls && event.args.image_urls.length > 0 && (
<ImageCarousel size="small" images={event.args.image_urls} />
)}
{shouldShowConfirmationButtons && <ConfirmationButtons />}
</ChatMessage>
);
}
if (isRejectObservation(event)) {
return <ChatMessage type="agent" message={event.content} />;
}
return (
<div>
{isOpenHandsAction(event) && hasThoughtProperty(event.args) && (
<ChatMessage type="agent" message={event.args.thought} />
)}
<GenericEventMessage
title={getEventContent(event).title}
details={getEventContent(event).details}
success={
isOpenHandsObservation(event)
? getObservationResult(event)
: undefined
}
/>
{shouldShowConfirmationButtons && <ConfirmationButtons />}
</div>
);
}

61
frontend/src/components/features/chat/generic-event-message.tsx Normal file
View File

@@ -0,0 +1,61 @@
import React from "react";
import Markdown from "react-markdown";
import remarkGfm from "remark-gfm";
import { code } from "../markdown/code";
import { ol, ul } from "../markdown/list";
import ArrowDown from "#/icons/angle-down-solid.svg?react";
import ArrowUp from "#/icons/angle-up-solid.svg?react";
import { SuccessIndicator } from "./success-indicator";
import { ObservationResultStatus } from "./event-content-helpers/get-observation-result";
interface GenericEventMessageProps {
title: React.ReactNode;
details: string;
success?: ObservationResultStatus;
}
export function GenericEventMessage({
title,
details,
success,
}: GenericEventMessageProps) {
const [showDetails, setShowDetails] = React.useState(false);
return (
<div className="flex flex-col gap-2 border-l-2 pl-2 my-2 py-2 border-neutral-300 text-sm w-full">
<div className="flex items-center justify-between font-bold text-neutral-300">
<div>
{title}
{details && (
<button
type="button"
onClick={() => setShowDetails((prev) => !prev)}
className="cursor-pointer text-left"
>
{showDetails ? (
<ArrowUp className="h-4 w-4 ml-2 inline fill-neutral-300" />
) : (
<ArrowDown className="h-4 w-4 ml-2 inline fill-neutral-300" />
)}
</button>
)}
</div>
{success && <SuccessIndicator status={success} />}
</div>
{showDetails && (
<Markdown
components={{
code,
ul,
ol,
}}
remarkPlugins={[remarkGfm]}
>
{details}
</Markdown>
)}
</div>
);
}

118
frontend/src/components/features/chat/messages.tsx
View File

@@ -1,80 +1,82 @@
import React from "react";
import type { Message } from "#/message";
import { ChatMessage } from "#/components/features/chat/chat-message";
import { ConfirmationButtons } from "#/components/shared/buttons/confirmation-buttons";
import { ImageCarousel } from "../images/image-carousel";
import { ExpandableMessage } from "./expandable-message";
import { useUserConversation } from "#/hooks/query/use-user-conversation";
import { useConversation } from "#/context/conversation-context";
import { I18nKey } from "#/i18n/declaration";
import { OpenHandsAction } from "#/types/core/actions";
import { OpenHandsObservation } from "#/types/core/observations";
import { isOpenHandsAction, isOpenHandsObservation } from "#/types/core/guards";
import { OpenHandsEventType } from "#/types/core/base";
import { EventMessage } from "./event-message";
import { ChatMessage } from "./chat-message";
import { useOptimisticUserMessage } from "#/hooks/use-optimistic-user-message";
const COMMON_NO_RENDER_LIST: OpenHandsEventType[] = [
"system",
"agent_state_changed",
"change_agent_state",
];
const ACTION_NO_RENDER_LIST: OpenHandsEventType[] = ["recall"];
const shouldRenderEvent = (event: OpenHandsAction | OpenHandsObservation) => {
if (isOpenHandsAction(event)) {
const noRenderList = COMMON_NO_RENDER_LIST.concat(ACTION_NO_RENDER_LIST);
return !noRenderList.includes(event.action);
}
if (isOpenHandsObservation(event)) {
return !COMMON_NO_RENDER_LIST.includes(event.observation);
}
return true;
};
interface MessagesProps {
messages: Message[];
messages: (OpenHandsAction | OpenHandsObservation)[];
isAwaitingUserConfirmation: boolean;
}
export const Messages: React.FC<MessagesProps> = React.memo(
({ messages, isAwaitingUserConfirmation }) => {
const { getOptimisticUserMessage } = useOptimisticUserMessage();
const { conversationId } = useConversation();
const { data: conversation } = useUserConversation(conversationId || null);
const optimisticUserMessage = getOptimisticUserMessage();
// Check if conversation metadata has trigger=resolver
const isResolverTrigger = conversation?.trigger === "resolver";
return messages.map((message, index) => {
const shouldShowConfirmationButtons =
messages.length - 1 === index &&
message.sender === "assistant" &&
isAwaitingUserConfirmation;
const actionHasObservationPair = React.useCallback(
(event: OpenHandsAction | OpenHandsObservation): boolean => {
if (isOpenHandsAction(event)) {
return !!messages.some(
(msg) => isOpenHandsObservation(msg) && msg.cause === event.id,
);
}
const isFirstUserMessageWithResolverTrigger =
index === 0 && message.sender === "user" && isResolverTrigger;
return false;
},
[messages],
);
// Special case: First user message with resolver trigger
if (isFirstUserMessageWithResolverTrigger) {
return (
<div key={index}>
<ExpandableMessage
type="action"
message={message.content}
id={I18nKey.CHAT$RESOLVER_INSTRUCTIONS}
/>
{message.imageUrls && message.imageUrls.length > 0 && (
<ImageCarousel size="small" images={message.imageUrls} />
)}
</div>
);
}
return (
<>
{messages.filter(shouldRenderEvent).map((message, index) => (
<EventMessage
key={index}
event={message}
hasObservationPair={actionHasObservationPair(message)}
isFirstMessageWithResolverTrigger={index === 0 && isResolverTrigger}
isAwaitingUserConfirmation={isAwaitingUserConfirmation}
isLastMessage={messages.length - 1 === index}
/>
))}
if (message.type === "error" || message.type === "action") {
return (
<div key={index}>
<ExpandableMessage
type={message.type}
id={message.translationID}
message={message.content}
success={message.success}
observation={message.observation}
action={message.action}
/>
{shouldShowConfirmationButtons && <ConfirmationButtons />}
</div>
);
}
return (
<ChatMessage
key={index}
type={message.sender}
message={message.content}
>
{message.imageUrls && message.imageUrls.length > 0 && (
<ImageCarousel size="small" images={message.imageUrls} />
)}
{shouldShowConfirmationButtons && <ConfirmationButtons />}
</ChatMessage>
);
});
{optimisticUserMessage && (
<ChatMessage type="user" message={optimisticUserMessage} />
)}
</>
);
},
);

35
frontend/src/components/features/chat/success-indicator.tsx Normal file
View File

@@ -0,0 +1,35 @@
import { FaClock } from "react-icons/fa";
import CheckCircle from "#/icons/check-circle-solid.svg?react";
import XCircle from "#/icons/x-circle-solid.svg?react";
import { ObservationResultStatus } from "./event-content-helpers/get-observation-result";
interface SuccessIndicatorProps {
status: ObservationResultStatus;
}
export function SuccessIndicator({ status }: SuccessIndicatorProps) {
return (
<span className="flex-shrink-0">
{status === "success" && (
<CheckCircle
data-testid="status-icon"
className="h-4 w-4 ml-2 inline fill-success"
/>
)}
{status === "error" && (
<XCircle
data-testid="status-icon"
className="h-4 w-4 ml-2 inline fill-danger"
/>
)}
{status === "timeout" && (
<FaClock
data-testid="status-icon"
className="h-4 w-4 ml-2 inline fill-yellow-500"
/>
)}
</span>
);
}

9
frontend/src/components/features/conversation-panel/conversation-card.tsx
View File

@@ -15,8 +15,9 @@ import { cn } from "#/utils/utils";
import { BaseModal } from "../../shared/modals/base-modal/base-modal";
import { RootState } from "#/store";
import { I18nKey } from "#/i18n/declaration";
import { selectSystemMessage } from "#/state/chat-slice";
import { transformVSCodeUrl } from "#/utils/vscode-url-helper";
import { useWsClient } from "#/context/ws-client-provider";
import { isSystemMessage } from "#/types/core/guards";
interface ConversationCardProps {
onClick?: () => void;
@@ -52,15 +53,17 @@ export function ConversationCard({
conversationId,
}: ConversationCardProps) {
const { t } = useTranslation();
const { parsedEvents } = useWsClient();
const [contextMenuVisible, setContextMenuVisible] = React.useState(false);
const [titleMode, setTitleMode] = React.useState<"view" | "edit">("view");
const [metricsModalVisible, setMetricsModalVisible] = React.useState(false);
const [systemModalVisible, setSystemModalVisible] = React.useState(false);
const inputRef = React.useRef<HTMLInputElement>(null);
const systemMessage = parsedEvents.find(isSystemMessage);
// Subscribe to metrics data from Redux store
const metrics = useSelector((state: RootState) => state.metrics);
const systemMessage = useSelector(selectSystemMessage);
const handleBlur = () => {
if (inputRef.current?.value) {
@@ -365,7 +368,7 @@ export function ConversationCard({
<SystemMessageModal
isOpen={systemModalVisible}
onClose={() => setSystemModalVisible(false)}
systemMessage={systemMessage}
systemMessage={systemMessage ? systemMessage.args : null}
/>
</>
);

3
frontend/src/components/features/home/tasks/task-card.tsx
View File

@@ -6,6 +6,7 @@ import { cn } from "#/utils/utils";
import { useUserRepositories } from "#/hooks/query/use-user-repositories";
import { TaskIssueNumber } from "./task-issue-number";
import { Provider } from "#/types/settings";
import { useOptimisticUserMessage } from "#/hooks/use-optimistic-user-message";
const getTaskTypeMap = (
t: (key: string) => string,
@@ -21,6 +22,7 @@ interface TaskCardProps {
}
export function TaskCard({ task }: TaskCardProps) {
const { setOptimisticUserMessage } = useOptimisticUserMessage();
const { data: repositories } = useUserRepositories();
const { mutate: createConversation, isPending } = useCreateConversation();
const isCreatingConversation = useIsCreatingConversation();
@@ -38,6 +40,7 @@ export function TaskCard({ task }: TaskCardProps) {
const handleLaunchConversation = () => {
const repo = getRepo(task.repo, task.git_provider);
setOptimisticUserMessage("Addressing task...");
return createConversation({
selectedRepository: repo,

66
frontend/src/context/ws-client-provider.tsx
View File

@@ -3,7 +3,7 @@ import { io, Socket } from "socket.io-client";
import { useQueryClient } from "@tanstack/react-query";
import EventLogger from "#/utils/event-logger";
import { handleAssistantMessage } from "#/services/actions";
import { showChatError } from "#/utils/error-handler";
import { showChatError, trackError } from "#/utils/error-handler";
import { useRate } from "#/hooks/use-rate";
import { OpenHandsParsedEvent } from "#/types/core";
import {
@@ -11,10 +11,26 @@ import {
CommandAction,
FileEditAction,
FileWriteAction,
OpenHandsAction,
UserMessageAction,
} from "#/types/core/actions";
import { Conversation } from "#/api/open-hands.types";
import { useUserProviders } from "#/hooks/use-user-providers";
import { OpenHandsObservation } from "#/types/core/observations";
import {
isErrorObservation,
isOpenHandsAction,
isOpenHandsObservation,
isUserMessage,
} from "#/types/core/guards";
import { useOptimisticUserMessage } from "#/hooks/use-optimistic-user-message";
import { useWSErrorMessage } from "#/hooks/use-ws-error-message";
const hasValidMessageProperty = (obj: unknown): obj is { message: string } =>
typeof obj === "object" &&
obj !== null &&
"message" in obj &&
typeof obj.message === "string";
const isOpenHandsEvent = (event: unknown): event is OpenHandsParsedEvent =>
typeof event === "object" &&
@@ -35,14 +51,6 @@ const isFileEditAction = (
const isCommandAction = (event: OpenHandsParsedEvent): event is CommandAction =>
"action" in event && event.action === "run";
const isUserMessage = (
event: OpenHandsParsedEvent,
): event is UserMessageAction =>
"source" in event &&
"type" in event &&
event.source === "user" &&
event.type === "message";
const isAssistantMessage = (
event: OpenHandsParsedEvent,
): event is AssistantMessageAction =>
@@ -65,6 +73,7 @@ interface UseWsClient {
status: WsClientProviderStatus;
isLoadingMessages: boolean;
events: Record<string, unknown>[];
parsedEvents: (OpenHandsAction | OpenHandsObservation)[];
send: (event: Record<string, unknown>) => void;
}
@@ -72,6 +81,7 @@ const WsClientContext = React.createContext<UseWsClient>({
status: WsClientProviderStatus.DISCONNECTED,
isLoadingMessages: true,
events: [],
parsedEvents: [],
send: () => {
throw new Error("not connected");
},
@@ -121,12 +131,17 @@ export function WsClientProvider({
conversationId,
children,
}: React.PropsWithChildren<WsClientProviderProps>) {
const { removeOptimisticUserMessage } = useOptimisticUserMessage();
const { setErrorMessage, removeErrorMessage } = useWSErrorMessage();
const queryClient = useQueryClient();
const sioRef = React.useRef<Socket | null>(null);
const [status, setStatus] = React.useState(
WsClientProviderStatus.DISCONNECTED,
);
const [events, setEvents] = React.useState<Record<string, unknown>[]>([]);
const [parsedEvents, setParsedEvents] = React.useState<
(OpenHandsAction | OpenHandsObservation)[]
>([]);
const lastEventRef = React.useRef<Record<string, unknown> | null>(null);
const { providers } = useUserProviders();
@@ -146,6 +161,24 @@ export function WsClientProvider({
function handleMessage(event: Record<string, unknown>) {
if (isOpenHandsEvent(event)) {
if (isOpenHandsAction(event) || isOpenHandsObservation(event)) {
setParsedEvents((prevEvents) => [...prevEvents, event]);
}
if (isErrorObservation(event)) {
trackError({
message: event.message,
source: "chat",
metadata: { msgId: event.id },
});
} else {
removeErrorMessage();
}
if (isUserMessage(event)) {
removeOptimisticUserMessage();
}
if (isMessageAction(event)) {
messageRateHandler.record(new Date().getTime());
}
@@ -202,11 +235,23 @@ export function WsClientProvider({
sio.io.opts.query = sio.io.opts.query || {};
sio.io.opts.query.latest_event_id = lastEventRef.current?.id;
updateStatusWhenErrorMessagePresent(data);
setErrorMessage(
hasValidMessageProperty(data)
? data.message
: "The WebSocket connection was closed.",
);
}
function handleError(data: unknown) {
setStatus(WsClientProviderStatus.DISCONNECTED);
updateStatusWhenErrorMessagePresent(data);
setErrorMessage(
hasValidMessageProperty(data)
? data.message
: "An unknown error occurred on the WebSocket connection.",
);
}
React.useEffect(() => {
@@ -267,9 +312,10 @@ export function WsClientProvider({
status,
isLoadingMessages: messageRateHandler.isUnderThreshold,
events,
parsedEvents,
send,
}),
[status, messageRateHandler.isUnderThreshold, events],
[status, messageRateHandler.isUnderThreshold, events, parsedEvents],
);
return <WsClientContext value={value}>{children}</WsClientContext>;

16
frontend/src/hooks/use-handle-ws-events.ts
View File

@@ -1,10 +1,7 @@
import React from "react";
import { useDispatch } from "react-redux";
import { useWsClient } from "#/context/ws-client-provider";
import { generateAgentStateChangeEvent } from "#/services/agent-state-service";
import { addErrorMessage } from "#/state/chat-slice";
import { AgentState } from "#/types/agent-state";
import { ErrorObservation } from "#/types/core/observations";
import { displayErrorToast } from "#/utils/custom-toast-handlers";
interface ServerError {
@@ -15,12 +12,8 @@ interface ServerError {
const isServerError = (data: object): data is ServerError => "error" in data;
const isErrorObservation = (data: object): data is ErrorObservation =>
"observation" in data && data.observation === "error";
export const useHandleWSEvents = () => {
const { events, send } = useWsClient();
const dispatch = useDispatch();
React.useEffect(() => {
if (!events.length) {
@@ -49,14 +42,5 @@ export const useHandleWSEvents = () => {
send(generateAgentStateChangeEvent(AgentState.PAUSED));
}
}
if (isErrorObservation(event)) {
dispatch(
addErrorMessage({
id: event.extras?.error_id,
message: event.message,
}),
);
}
}, [events.length]);
};

23
frontend/src/hooks/use-optimistic-user-message.ts Normal file
View File

@@ -0,0 +1,23 @@
import { useQueryClient } from "@tanstack/react-query";
export const useOptimisticUserMessage = () => {
const queryKey = ["optimistic_user_message"] as const;
const queryClient = useQueryClient();
const setOptimisticUserMessage = (message: string) => {
queryClient.setQueryData<string>(queryKey, message);
};
const getOptimisticUserMessage = () =>
queryClient.getQueryData<string>(queryKey);
const removeOptimisticUserMessage = () => {
queryClient.removeQueries({ queryKey });
};
return {
setOptimisticUserMessage,
getOptimisticUserMessage,
removeOptimisticUserMessage,
};
};

22
frontend/src/hooks/use-ws-error-message.ts Normal file
View File

@@ -0,0 +1,22 @@
import { useQueryClient } from "@tanstack/react-query";
export const useWSErrorMessage = () => {
const queryClient = useQueryClient();
const setErrorMessage = (message: string) => {
queryClient.setQueryData<string>(["error_message"], message);
};
const getErrorMessage = () =>
queryClient.getQueryData<string>(["error_message"]);
const removeErrorMessage = () => {
queryClient.removeQueries({ queryKey: ["error_message"] });
};
return {
setErrorMessage,
getErrorMessage,
removeErrorMessage,
};
};

224
frontend/src/i18n/translation.json
View File

@@ -6384,20 +6384,20 @@
"uk": "Завантажити файл"
},
"ACTION_MESSAGE$RUN": {
"en": "Running <cmd>{{action.payload.args.command}}</cmd>",
"zh-CN": "运行 <cmd>{{action.payload.args.command}}</cmd>",
"zh-TW": "執行 <cmd>{{action.payload.args.command}}</cmd>",
"ko-KR": "실행 <cmd>{{action.payload.args.command}}</cmd>",
"ja": "実行 <cmd>{{action.payload.args.command}}</cmd>",
"no": "Kjører <cmd>{{action.payload.args.command}}</cmd>",
"ar": "تشغيل <cmd>{{action.payload.args.command}}</cmd>",
"de": "Führt <cmd>{{action.payload.args.command}}</cmd> aus",
"fr": "Exécution de <cmd>{{action.payload.args.command}}</cmd>",
"it": "Esecuzione di <cmd>{{action.payload.args.command}}</cmd>",
"pt": "Executando <cmd>{{action.payload.args.command}}</cmd>",
"es": "Ejecutando <cmd>{{action.payload.args.command}}</cmd>",
"tr": "<cmd>{{action.payload.args.command}}</cmd> çalıştırılıyor",
"uk": "Виконую <cmd>{{action.payload.args.command}}</cmd>"
"en": "Running <cmd>{{command}}</cmd>",
"zh-CN": "运行 <cmd>{{command}}</cmd>",
"zh-TW": "執行 <cmd>{{command}}</cmd>",
"ko-KR": "실행 <cmd>{{command}}</cmd>",
"ja": "実行 <cmd>{{command}}</cmd>",
"no": "Kjører <cmd>{{command}}</cmd>",
"ar": "تشغيل <cmd>{{command}}</cmd>",
"de": "Führt <cmd>{{command}}</cmd> aus",
"fr": "Exécution de <cmd>{{command}}</cmd>",
"it": "Esecuzione di <cmd>{{command}}</cmd>",
"pt": "Executando <cmd>{{command}}</cmd>",
"es": "Ejecutando <cmd>{{command}}</cmd>",
"tr": "<cmd>{{command}}</cmd> çalıştırılıyor",
"uk": "Виконую <cmd>{{command}}</cmd>"
},
"ACTION_MESSAGE$RUN_IPYTHON": {
"en": "Running a Python command",
@@ -6432,52 +6432,52 @@
"uk": "Викликаю інструмент MCP: {{action.payload.args.name}}"
},
"ACTION_MESSAGE$READ": {
"en": "Reading <path>{{action.payload.args.path}}</path>",
"zh-CN": "读取 <path>{{action.payload.args.path}}</path>",
"zh-TW": "讀取 <path>{{action.payload.args.path}}</path>",
"ko-KR": "읽기 <path>{{action.payload.args.path}}</path>",
"ja": "読み取り <path>{{action.payload.args.path}}</path>",
"no": "Leser <path>{{action.payload.args.path}}</path>",
"ar": "قراءة <path>{{action.payload.args.path}}</path>",
"de": "Liest <path>{{action.payload.args.path}}</path>",
"fr": "Lecture de <path>{{action.payload.args.path}}</path>",
"it": "Lettura di <path>{{action.payload.args.path}}</path>",
"pt": "Lendo <path>{{action.payload.args.path}}</path>",
"es": "Leyendo <path>{{action.payload.args.path}}</path>",
"tr": "<path>{{action.payload.args.path}}</path> okunuyor",
"uk": "Читаю <path>{{action.payload.args.path}}</path>"
"en": "Reading <path>{{path}}</path>",
"zh-CN": "读取 <path>{{path}}</path>",
"zh-TW": "讀取 <path>{{path}}</path>",
"ko-KR": "읽기 <path>{{path}}</path>",
"ja": "読み取り <path>{{path}}</path>",
"no": "Leser <path>{{path}}</path>",
"ar": "قراءة <path>{{path}}</path>",
"de": "Liest <path>{{path}}</path>",
"fr": "Lecture de <path>{{path}}</path>",
"it": "Lettura di <path>{{path}}</path>",
"pt": "Lendo <path>{{path}}</path>",
"es": "Leyendo <path>{{path}}</path>",
"tr": "<path>{{path}}</path> okunuyor",
"uk": "Читаю <path>{{path}}</path>"
},
"ACTION_MESSAGE$EDIT": {
"en": "Editing <path>{{action.payload.args.path}}</path>",
"zh-CN": "编辑 <path>{{action.payload.args.path}}</path>",
"zh-TW": "編輯 <path>{{action.payload.args.path}}</path>",
"ko-KR": "편집 <path>{{action.payload.args.path}}</path>",
"ja": "編集 <path>{{action.payload.args.path}}</path>",
"no": "Redigerer <path>{{action.payload.args.path}}</path>",
"ar": "تحرير <path>{{action.payload.args.path}}</path>",
"de": "Bearbeitet <path>{{action.payload.args.path}}</path>",
"fr": "Modification de <path>{{action.payload.args.path}}</path>",
"it": "Modifica di <path>{{action.payload.args.path}}</path>",
"pt": "Editando <path>{{action.payload.args.path}}</path>",
"es": "Editando <path>{{action.payload.args.path}}</path>",
"tr": "<path>{{action.payload.args.path}}</path> düzenleniyor",
"uk": "Редагую <path>{{action.payload.args.path}}</path>"
"en": "Editing <path>{{path}}</path>",
"zh-CN": "编辑 <path>{{path}}</path>",
"zh-TW": "編輯 <path>{{path}}</path>",
"ko-KR": "편집 <path>{{path}}</path>",
"ja": "編集 <path>{{path}}</path>",
"no": "Redigerer <path>{{path}}</path>",
"ar": "تحرير <path>{{path}}</path>",
"de": "Bearbeitet <path>{{path}}</path>",
"fr": "Modification de <path>{{path}}</path>",
"it": "Modifica di <path>{{path}}</path>",
"pt": "Editando <path>{{path}}</path>",
"es": "Editando <path>{{path}}</path>",
"tr": "<path>{{path}}</path> düzenleniyor",
"uk": "Редагую <path>{{path}}</path>"
},
"ACTION_MESSAGE$WRITE": {
"en": "Writing to <path>{{action.payload.args.path}}</path>",
"zh-CN": "写入 <path>{{action.payload.args.path}}</path>",
"zh-TW": "寫入 <path>{{action.payload.args.path}}</path>",
"ko-KR": "쓰기 <path>{{action.payload.args.path}}</path>",
"ja": "書き込み <path>{{action.payload.args.path}}</path>",
"no": "Skriver til <path>{{action.payload.args.path}}</path>",
"ar": "الكتابة إلى <path>{{action.payload.args.path}}</path>",
"de": "Schreibt in <path>{{action.payload.args.path}}</path>",
"fr": "Écriture dans <path>{{action.payload.args.path}}</path>",
"it": "Scrittura su <path>{{action.payload.args.path}}</path>",
"pt": "Escrevendo em <path>{{action.payload.args.path}}</path>",
"es": "Escribiendo en <path>{{action.payload.args.path}}</path>",
"tr": "<path>{{action.payload.args.path}}</path> dosyasına yazılıyor",
"uk": "Записую в <path>{{action.payload.args.path}}</path>"
"en": "Writing to <path>{{path}}</path>",
"zh-CN": "写入 <path>{{path}}</path>",
"zh-TW": "寫入 <path>{{path}}</path>",
"ko-KR": "쓰기 <path>{{path}}</path>",
"ja": "書き込み <path>{{path}}</path>",
"no": "Skriver til <path>{{path}}</path>",
"ar": "الكتابة إلى <path>{{path}}</path>",
"de": "Schreibt in <path>{{path}}</path>",
"fr": "Écriture dans <path>{{path}}</path>",
"it": "Scrittura su <path>{{path}}</path>",
"pt": "Escrevendo em <path>{{path}}</path>",
"es": "Escribiendo en <path>{{path}}</path>",
"tr": "<path>{{path}}</path> dosyasına yazılıyor",
"uk": "Записую в <path>{{path}}</path>"
},
"ACTION_MESSAGE$BROWSE": {
"en": "Browsing the web",
@@ -6544,20 +6544,20 @@
"uk": "Системне повідомлення"
},
"OBSERVATION_MESSAGE$RUN": {
"en": "Ran <cmd>{{observation.payload.extras.command}}</cmd>",
"zh-CN": "运行 <cmd>{{observation.payload.extras.command}}</cmd>",
"zh-TW": "執行 <cmd>{{observation.payload.extras.command}}</cmd>",
"ko-KR": "실행 <cmd>{{observation.payload.extras.command}}</cmd>",
"ja": "実行 <cmd>{{observation.payload.extras.command}}</cmd>",
"no": "Kjørte <cmd>{{observation.payload.extras.command}}</cmd>",
"ar": "تم تشغيل <cmd>{{observation.payload.extras.command}}</cmd>",
"de": "Führte <cmd>{{observation.payload.extras.command}}</cmd> aus",
"fr": "A exécuté <cmd>{{observation.payload.extras.command}}</cmd>",
"it": "Ha eseguito <cmd>{{observation.payload.extras.command}}</cmd>",
"pt": "Executou <cmd>{{observation.payload.extras.command}}</cmd>",
"es": "Ejecutó <cmd>{{observation.payload.extras.command}}</cmd>",
"tr": "<cmd>{{observation.payload.extras.command}}</cmd> çalıştırıldı",
"uk": "Запустив <cmd>{{observation.payload.extras.command}}</cmd>"
"en": "Ran <cmd>{{command}}</cmd>",
"zh-CN": "运行 <cmd>{{command}}</cmd>",
"zh-TW": "執行 <cmd>{{command}}</cmd>",
"ko-KR": "실행 <cmd>{{command}}</cmd>",
"ja": "実行 <cmd>{{command}}</cmd>",
"no": "Kjørte <cmd>{{command}}</cmd>",
"ar": "تم تشغيل <cmd>{{command}}</cmd>",
"de": "Führte <cmd>{{command}}</cmd> aus",
"fr": "A exécuté <cmd>{{command}}</cmd>",
"it": "Ha eseguito <cmd>{{command}}</cmd>",
"pt": "Executou <cmd>{{command}}</cmd>",
"es": "Ejecutó <cmd>{{command}}</cmd>",
"tr": "<cmd>{{command}}</cmd> çalıştırıldı",
"uk": "Запустив <cmd>{{command}}</cmd>"
},
"OBSERVATION_MESSAGE$RUN_IPYTHON": {
"en": "Ran a Python command",
@@ -6576,52 +6576,52 @@
"uk": "Виконав команду Python"
},
"OBSERVATION_MESSAGE$READ": {
"en": "Read <path>{{observation.payload.extras.path}}</path>",
"zh-CN": "读取 <path>{{observation.payload.extras.path}}</path>",
"zh-TW": "讀取 <path>{{observation.payload.extras.path}}</path>",
"ko-KR": "읽기 <path>{{observation.payload.extras.path}}</path>",
"ja": "読み取り <path>{{observation.payload.extras.path}}</path>",
"no": "Leste <path>{{observation.payload.extras.path}}</path>",
"ar": "تمت قراءة <path>{{observation.payload.extras.path}}</path>",
"de": "Las <path>{{observation.payload.extras.path}}</path>",
"fr": "A lu <path>{{observation.payload.extras.path}}</path>",
"it": "Ha letto <path>{{observation.payload.extras.path}}</path>",
"pt": "Leu <path>{{observation.payload.extras.path}}</path>",
"es": "Leyó <path>{{observation.payload.extras.path}}</path>",
"tr": "<path>{{observation.payload.extras.path}}</path> okundu",
"uk": "Прочитав <path>{{observation.payload.extras.path}}</path>"
"en": "Read <path>{{path}}</path>",
"zh-CN": "读取 <path>{{path}}</path>",
"zh-TW": "讀取 <path>{{path}}</path>",
"ko-KR": "읽기 <path>{{path}}</path>",
"ja": "読み取り <path>{{path}}</path>",
"no": "Leste <path>{{path}}</path>",
"ar": "تمت قراءة <path>{{path}}</path>",
"de": "Las <path>{{path}}</path>",
"fr": "A lu <path>{{path}}</path>",
"it": "Ha letto <path>{{path}}</path>",
"pt": "Leu <path>{{path}}</path>",
"es": "Leyó <path>{{path}}</path>",
"tr": "<path>{{path}}</path> okundu",
"uk": "Прочитав <path>{{path}}</path>"
},
"OBSERVATION_MESSAGE$EDIT": {
"en": "Edited <path>{{observation.payload.extras.path}}</path>",
"zh-CN": "编辑 <path>{{observation.payload.extras.path}}</path>",
"zh-TW": "編輯 <path>{{observation.payload.extras.path}}</path>",
"ko-KR": "편집 <path>{{observation.payload.extras.path}}</path>",
"ja": "編集 <path>{{observation.payload.extras.path}}</path>",
"no": "Redigerte <path>{{observation.payload.extras.path}}</path>",
"ar": "تم تحرير <path>{{observation.payload.extras.path}}</path>",
"de": "Hat <path>{{observation.payload.extras.path}}</path> bearbeitet",
"fr": "A modifié <path>{{observation.payload.extras.path}}</path>",
"it": "Ha modificato <path>{{observation.payload.extras.path}}</path>",
"pt": "Editou <path>{{observation.payload.extras.path}}</path>",
"es": "Editó <path>{{observation.payload.extras.path}}</path>",
"tr": "<path>{{observation.payload.extras.path}}</path> düzenlendi",
"uk": "Відредагував <path>{{observation.payload.extras.path}}</path>"
"en": "Edited <path>{{path}}</path>",
"zh-CN": "编辑 <path>{{path}}</path>",
"zh-TW": "編輯 <path>{{path}}</path>",
"ko-KR": "편집 <path>{{path}}</path>",
"ja": "編集 <path>{{path}}</path>",
"no": "Redigerte <path>{{path}}</path>",
"ar": "تم تحرير <path>{{path}}</path>",
"de": "Hat <path>{{path}}</path> bearbeitet",
"fr": "A modifié <path>{{path}}</path>",
"it": "Ha modificato <path>{{path}}</path>",
"pt": "Editou <path>{{path}}</path>",
"es": "Editó <path>{{path}}</path>",
"tr": "<path>{{path}}</path> düzenlendi",
"uk": "Відредагував <path>{{path}}</path>"
},
"OBSERVATION_MESSAGE$WRITE": {
"en": "Wrote to <path>{{observation.payload.extras.path}}</path>",
"zh-CN": "写入 <path>{{observation.payload.extras.path}}</path>",
"zh-TW": "寫入 <path>{{observation.payload.extras.path}}</path>",
"ko-KR": "쓰기 <path>{{observation.payload.extras.path}}</path>",
"ja": "書き込み <path>{{observation.payload.extras.path}}</path>",
"no": "Skrev til <path>{{observation.payload.extras.path}}</path>",
"ar": "تمت الكتابة إلى <path>{{observation.payload.extras.path}}</path>",
"de": "Hat in <path>{{observation.payload.extras.path}}</path> geschrieben",
"fr": "A écrit dans <path>{{observation.payload.extras.path}}</path>",
"it": "Ha scritto su <path>{{observation.payload.extras.path}}</path>",
"pt": "Escreveu em <path>{{observation.payload.extras.path}}</path>",
"es": "Escribió en <path>{{observation.payload.extras.path}}</path>",
"tr": "<path>{{observation.payload.extras.path}}</path> dosyasına yazıldı",
"uk": "Записав на <path>{{observation.payload.extras.path}}</path>"
"en": "Wrote to <path>{{path}}</path>",
"zh-CN": "写入 <path>{{path}}</path>",
"zh-TW": "寫入 <path>{{path}}</path>",
"ko-KR": "쓰기 <path>{{path}}</path>",
"ja": "書き込み <path>{{path}}</path>",
"no": "Skrev til <path>{{path}}</path>",
"ar": "تمت الكتابة إلى <path>{{path}}</path>",
"de": "Hat in <path>{{path}}</path> geschrieben",
"fr": "A écrit dans <path>{{path}}</path>",
"it": "Ha scritto su <path>{{path}}</path>",
"pt": "Escreveu em <path>{{path}}</path>",
"es": "Escribió en <path>{{path}}</path>",
"tr": "<path>{{path}}</path> dosyasına yazıldı",
"uk": "Записав на <path>{{path}}</path>"
},
"OBSERVATION_MESSAGE$BROWSE": {
"en": "Browsing completed",

20
frontend/src/routes/conversation.tsx
View File

@@ -13,7 +13,6 @@ import {
useConversation,
} from "#/context/conversation-context";
import { Controls } from "#/components/features/controls/controls";
import { clearMessages, addUserMessage } from "#/state/chat-slice";
import { clearTerminal } from "#/state/command-slice";
import { useEffectOnce } from "#/hooks/use-effect-once";
import GlobeIcon from "#/icons/globe.svg?react";
@@ -34,7 +33,6 @@ import Security from "#/components/shared/modals/security/security";
import { useUserConversation } from "#/hooks/query/use-user-conversation";
import { ServedAppLabel } from "#/components/layout/served-app-label";
import { useSettings } from "#/hooks/query/use-settings";
import { clearFiles, clearInitialPrompt } from "#/state/initial-query-slice";
import { RootState } from "#/store";
import { displayErrorToast } from "#/utils/custom-toast-handlers";
import { useDocumentTitleFromState } from "#/hooks/use-document-title-from-state";
@@ -49,9 +47,7 @@ function AppContent() {
const { data: conversation, isFetched } = useUserConversation(
conversationId || null,
);
const { initialPrompt, files } = useSelector(
(state: RootState) => state.initialQuery,
);
const { curAgentState } = useSelector((state: RootState) => state.agent);
const dispatch = useDispatch();
const navigate = useNavigate();
@@ -71,25 +67,11 @@ function AppContent() {
}, [conversation, isFetched]);
React.useEffect(() => {
dispatch(clearMessages());
dispatch(clearTerminal());
dispatch(clearJupyter());
if (conversationId && (initialPrompt || files.length > 0)) {
dispatch(
addUserMessage({
content: initialPrompt || "",
imageUrls: files || [],
timestamp: new Date().toISOString(),
pending: true,
}),
);
dispatch(clearInitialPrompt());
dispatch(clearFiles());
}
}, [conversationId]);
useEffectOnce(() => {
dispatch(clearMessages());
dispatch(clearTerminal());
dispatch(clearJupyter());
});

4
frontend/src/services/__tests__/actions.test.ts
View File

@@ -4,7 +4,6 @@ import { StatusMessage } from "#/types/message";
import { queryClient } from "#/query-client-config";
import store from "#/store";
import { setCurStatusMessage } from "#/state/status-slice";
import { addErrorMessage } from "#/state/chat-slice";
import { trackError } from "#/utils/error-handler";
// Mock dependencies
@@ -101,9 +100,6 @@ describe("handleStatusMessage", () => {
metadata: { msgId: "ERROR_ID" },
});
// Verify that store.dispatch was called with addErrorMessage
expect(store.dispatch).toHaveBeenCalledWith(addErrorMessage(statusMessage));
// Verify that queryClient.invalidateQueries was not called
expect(queryClient.invalidateQueries).not.toHaveBeenCalled();
});

122
frontend/src/services/actions.ts
View File

@@ -1,13 +1,5 @@
import {
addAssistantMessage,
addAssistantAction,
addUserMessage,
addErrorMessage,
} from "#/state/chat-slice";
import { trackError } from "#/utils/error-handler";
import { appendSecurityAnalyzerInput } from "#/state/security-analyzer-slice";
import { setCode, setActiveFilepath } from "#/state/code-slice";
import { appendJupyterInput } from "#/state/jupyter-slice";
import { setCurStatusMessage } from "#/state/status-slice";
import { setMetrics } from "#/state/metrics-slice";
import store from "#/store";
@@ -21,67 +13,6 @@ import { handleObservationMessage } from "./observations";
import { appendInput } from "#/state/command-slice";
import { queryClient } from "#/query-client-config";
const messageActions = {
[ActionType.BROWSE]: (message: ActionMessage) => {
if (!message.args.thought && message.message) {
store.dispatch(addAssistantMessage(message.message));
}
},
[ActionType.BROWSE_INTERACTIVE]: (message: ActionMessage) => {
if (!message.args.thought && message.message) {
store.dispatch(addAssistantMessage(message.message));
}
},
[ActionType.WRITE]: (message: ActionMessage) => {
const { path, content } = message.args;
store.dispatch(setActiveFilepath(path));
store.dispatch(setCode(content));
},
[ActionType.MESSAGE]: (message: ActionMessage) => {
if (message.source === "user") {
store.dispatch(
addUserMessage({
content: message.args.content,
imageUrls:
typeof message.args.image_urls === "string"
? [message.args.image_urls]
: message.args.image_urls,
timestamp: message.timestamp,
pending: false,
}),
);
} else {
store.dispatch(addAssistantMessage(message.args.content));
}
},
[ActionType.RUN_IPYTHON]: (message: ActionMessage) => {
if (message.args.confirmation_state !== "rejected") {
store.dispatch(appendJupyterInput(message.args.code));
}
},
[ActionType.FINISH]: (message: ActionMessage) => {
store.dispatch(addAssistantMessage(message.args.final_thought));
let successPrediction = "";
if (message.args.task_completed === "partial") {
successPrediction =
"I believe that the task was **completed partially**.";
} else if (message.args.task_completed === "false") {
successPrediction = "I believe that the task was **not completed**.";
} else if (message.args.task_completed === "true") {
successPrediction =
"I believe that the task was **completed successfully**.";
}
if (successPrediction) {
// if final_thought is not empty, add a new line before the success prediction
if (message.args.final_thought) {
store.dispatch(addAssistantMessage(`\n${successPrediction}`));
} else {
store.dispatch(addAssistantMessage(successPrediction));
}
}
},
};
export function handleActionMessage(message: ActionMessage) {
if (message.args?.hidden) {
return;
@@ -103,26 +34,6 @@ export function handleActionMessage(message: ActionMessage) {
if ("args" in message && "security_risk" in message.args) {
store.dispatch(appendSecurityAnalyzerInput(message));
}
if (message.source === "agent") {
// Only add thought as a message if it's not a "think" action
if (
message.args &&
message.args.thought &&
message.action !== ActionType.THINK
) {
store.dispatch(addAssistantMessage(message.args.thought));
}
// Need to convert ActionMessage to RejectAction
// @ts-expect-error TODO: fix
store.dispatch(addAssistantAction(message));
}
if (message.action in messageActions) {
const actionFn =
messageActions[message.action as keyof typeof messageActions];
actionFn(message);
}
}
export function handleStatusMessage(message: StatusMessage) {
@@ -146,11 +57,6 @@ export function handleStatusMessage(message: StatusMessage) {
source: "chat",
metadata: { msgId: message.id },
});
store.dispatch(
addErrorMessage({
...message,
}),
);
}
}
@@ -161,33 +67,5 @@ export function handleAssistantMessage(message: Record<string, unknown>) {
handleObservationMessage(message as unknown as ObservationMessage);
} else if (message.status_update) {
handleStatusMessage(message as unknown as StatusMessage);
} else if (message.error) {
// Handle error messages from the server
const errorMessage =
typeof message.message === "string"
? message.message
: String(message.message || "Unknown error");
trackError({
message: errorMessage,
source: "websocket",
metadata: { raw_message: message },
});
store.dispatch(
addErrorMessage({
message: errorMessage,
}),
);
} else {
const errorMsg = "Unknown message type received";
trackError({
message: errorMsg,
source: "chat",
metadata: { raw_message: message },
});
store.dispatch(
addErrorMessage({
message: errorMsg,
}),
);
}
}

205
frontend/src/services/observations.ts
View File

@@ -2,14 +2,9 @@ import { setCurrentAgentState } from "#/state/agent-slice";
import { setUrl, setScreenshotSrc } from "#/state/browser-slice";
import store from "#/store";
import { ObservationMessage } from "#/types/message";
import { AgentState } from "#/types/agent-state";
import { appendOutput } from "#/state/command-slice";
import { appendJupyterOutput } from "#/state/jupyter-slice";
import ObservationType from "#/types/observation-type";
import {
addAssistantMessage,
addAssistantObservation,
} from "#/state/chat-slice";
export function handleObservationMessage(message: ObservationMessage) {
switch (message.observation) {
@@ -48,11 +43,6 @@ export function handleObservationMessage(message: ObservationMessage) {
store.dispatch(setCurrentAgentState(message.extras.agent_state));
break;
case ObservationType.DELEGATE:
// TODO: better UI for delegation result (#2309)
if (message.content) {
store.dispatch(addAssistantMessage(message.content));
}
break;
case ObservationType.READ:
case ObservationType.EDIT:
case ObservationType.THINK:
@@ -62,110 +52,13 @@ export function handleObservationMessage(message: ObservationMessage) {
case ObservationType.MCP:
break; // We don't display the default message for these observations
default:
store.dispatch(addAssistantMessage(message.message));
break;
}
if (!message.extras?.hidden) {
// Convert the message to the appropriate observation type
const { observation } = message;
const baseObservation = {
...message,
source: "agent" as const,
};
switch (observation) {
case "agent_state_changed":
store.dispatch(
addAssistantObservation({
...baseObservation,
observation: "agent_state_changed" as const,
extras: {
agent_state: (message.extras.agent_state as AgentState) || "idle",
},
}),
);
break;
case "recall":
store.dispatch(
addAssistantObservation({
...baseObservation,
observation: "recall" as const,
extras: {
...(message.extras || {}),
recall_type:
(message.extras?.recall_type as
| "workspace_context"
| "knowledge") || "knowledge",
},
}),
);
break;
case "run":
store.dispatch(
addAssistantObservation({
...baseObservation,
observation: "run" as const,
extras: {
command: String(message.extras.command || ""),
metadata: message.extras.metadata,
hidden: Boolean(message.extras.hidden),
},
}),
);
break;
case "read":
store.dispatch(
addAssistantObservation({
...baseObservation,
observation,
extras: {
path: String(message.extras.path || ""),
impl_source: String(message.extras.impl_source || ""),
},
}),
);
break;
case "edit":
store.dispatch(
addAssistantObservation({
...baseObservation,
observation,
extras: {
path: String(message.extras.path || ""),
diff: String(message.extras.diff || ""),
impl_source: String(message.extras.impl_source || ""),
},
}),
);
break;
case "run_ipython":
store.dispatch(
addAssistantObservation({
...baseObservation,
observation: "run_ipython" as const,
extras: {
code: String(message.extras.code || ""),
image_urls: Array.isArray(message.extras.image_urls)
? message.extras.image_urls
: [],
},
}),
);
break;
case "delegate":
store.dispatch(
addAssistantObservation({
...baseObservation,
observation: "delegate" as const,
extras: {
outputs:
typeof message.extras.outputs === "object"
? (message.extras.outputs as Record<string, unknown>)
: {},
},
}),
);
break;
case "browse":
if (message.extras?.screenshot) {
store.dispatch(setScreenshotSrc(message.extras.screenshot));
@@ -173,45 +66,6 @@ export function handleObservationMessage(message: ObservationMessage) {
if (message.extras?.url) {
store.dispatch(setUrl(message.extras.url));
}
store.dispatch(
addAssistantObservation({
...baseObservation,
observation: "browse" as const,
extras: {
url: String(message.extras.url || ""),
screenshot: String(message.extras.screenshot || ""),
error: Boolean(message.extras.error),
open_page_urls: Array.isArray(message.extras.open_page_urls)
? message.extras.open_page_urls
: [],
active_page_index: Number(message.extras.active_page_index || 0),
dom_object:
typeof message.extras.dom_object === "object"
? (message.extras.dom_object as Record<string, unknown>)
: {},
axtree_object:
typeof message.extras.axtree_object === "object"
? (message.extras.axtree_object as Record<string, unknown>)
: {},
extra_element_properties:
typeof message.extras.extra_element_properties === "object"
? (message.extras.extra_element_properties as Record<
string,
unknown
>)
: {},
last_browser_action: String(
message.extras.last_browser_action || "",
),
last_browser_action_error:
message.extras.last_browser_action_error,
focused_element_bid: String(
message.extras.focused_element_bid || "",
),
},
}),
);
break;
case "browse_interactive":
if (message.extras?.screenshot) {
@@ -220,65 +74,6 @@ export function handleObservationMessage(message: ObservationMessage) {
if (message.extras?.url) {
store.dispatch(setUrl(message.extras.url));
}
store.dispatch(
addAssistantObservation({
...baseObservation,
observation: "browse_interactive" as const,
extras: {
url: String(message.extras.url || ""),
screenshot: String(message.extras.screenshot || ""),
error: Boolean(message.extras.error),
open_page_urls: Array.isArray(message.extras.open_page_urls)
? message.extras.open_page_urls
: [],
active_page_index: Number(message.extras.active_page_index || 0),
dom_object:
typeof message.extras.dom_object === "object"
? (message.extras.dom_object as Record<string, unknown>)
: {},
axtree_object:
typeof message.extras.axtree_object === "object"
? (message.extras.axtree_object as Record<string, unknown>)
: {},
extra_element_properties:
typeof message.extras.extra_element_properties === "object"
? (message.extras.extra_element_properties as Record<
string,
unknown
>)
: {},
last_browser_action: String(
message.extras.last_browser_action || "",
),
last_browser_action_error:
message.extras.last_browser_action_error,
focused_element_bid: String(
message.extras.focused_element_bid || "",
),
},
}),
);
break;
case "error":
store.dispatch(
addAssistantObservation({
...baseObservation,
observation: "error" as const,
source: "user" as const,
extras: {
error_id: message.extras.error_id,
},
}),
);
break;
case "mcp":
store.dispatch(
addAssistantObservation({
...baseObservation,
observation: "mcp" as const,
}),
);
break;
default:
// For any unhandled observation types, just ignore them

380
frontend/src/state/chat-slice.ts
View File

@@ -1,380 +0,0 @@
import { createSlice, PayloadAction } from "@reduxjs/toolkit";
import type { Message } from "#/message";
import { ActionSecurityRisk } from "#/state/security-analyzer-slice";
import { OpenHandsAction } from "#/types/core/actions";
import { OpenHandsEventType } from "#/types/core/base";
import {
CommandObservation,
IPythonObservation,
OpenHandsObservation,
RecallObservation,
} from "#/types/core/observations";
type SliceState = {
messages: Message[];
systemMessage: {
content: string;
tools: Array<Record<string, unknown>> | null;
openhands_version: string | null;
agent_class: string | null;
} | null;
};
const MAX_CONTENT_LENGTH = 1000;
const HANDLED_ACTIONS: OpenHandsEventType[] = [
"run",
"run_ipython",
"write",
"read",
"browse",
"browse_interactive",
"edit",
"recall",
"think",
"system",
"call_tool_mcp",
"mcp",
];
function getRiskText(risk: ActionSecurityRisk) {
switch (risk) {
case ActionSecurityRisk.LOW:
return "Low Risk";
case ActionSecurityRisk.MEDIUM:
return "Medium Risk";
case ActionSecurityRisk.HIGH:
return "High Risk";
case ActionSecurityRisk.UNKNOWN:
default:
return "Unknown Risk";
}
}
const initialState: SliceState = {
messages: [],
systemMessage: null,
};
export const chatSlice = createSlice({
name: "chat",
initialState,
reducers: {
addUserMessage(
state,
action: PayloadAction<{
content: string;
imageUrls: string[];
timestamp: string;
pending?: boolean;
}>,
) {
const message: Message = {
type: "thought",
sender: "user",
content: action.payload.content,
imageUrls: action.payload.imageUrls,
timestamp: action.payload.timestamp || new Date().toISOString(),
pending: !!action.payload.pending,
};
// Remove any pending messages
let i = state.messages.length;
while (i) {
i -= 1;
const m = state.messages[i] as Message;
if (m.pending) {
state.messages.splice(i, 1);
}
}
state.messages.push(message);
},
addAssistantMessage(state: SliceState, action: PayloadAction<string>) {
const message: Message = {
type: "thought",
sender: "assistant",
content: action.payload,
imageUrls: [],
timestamp: new Date().toISOString(),
pending: false,
};
state.messages.push(message);
},
addAssistantAction(
state: SliceState,
action: PayloadAction<OpenHandsAction>,
) {
const actionID = action.payload.action;
if (!HANDLED_ACTIONS.includes(actionID)) {
return;
}
const translationID = `ACTION_MESSAGE$${actionID.toUpperCase()}`;
let text = "";
if (actionID === "system") {
// Store the system message in the state
state.systemMessage = {
content: action.payload.args.content,
tools: action.payload.args.tools,
openhands_version: action.payload.args.openhands_version,
agent_class: action.payload.args.agent_class,
};
// Don't add a message for system actions
return;
}
if (actionID === "run") {
text = `Command:\n\`${action.payload.args.command}\``;
} else if (actionID === "run_ipython") {
text = `\`\`\`\n${action.payload.args.code}\n\`\`\``;
} else if (actionID === "write") {
let { content } = action.payload.args;
if (content.length > MAX_CONTENT_LENGTH) {
content = `${content.slice(0, MAX_CONTENT_LENGTH)}...`;
}
text = `${action.payload.args.path}\n${content}`;
} else if (actionID === "browse") {
text = `Browsing ${action.payload.args.url}`;
} else if (actionID === "browse_interactive") {
// Include the browser_actions in the content
text = `**Action:**\n\n\`\`\`python\n${action.payload.args.browser_actions}\n\`\`\``;
} else if (actionID === "recall") {
// skip recall actions
return;
} else if (actionID === "call_tool_mcp") {
// Format MCP action with name and arguments
const name = action.payload.args.name || "";
const args = action.payload.args.arguments || {};
text = `**MCP Tool Call:** ${name}\n\n`;
// Include thought if available
if (action.payload.args.thought) {
text += `\n\n**Thought:**\n${action.payload.args.thought}`;
}
text += `\n\n**Arguments:**\n\`\`\`json\n${JSON.stringify(args, null, 2)}\n\`\`\``;
}
if (actionID === "run" || actionID === "run_ipython") {
if (
action.payload.args.confirmation_state === "awaiting_confirmation"
) {
text += `\n\n${getRiskText(action.payload.args.security_risk as unknown as ActionSecurityRisk)}`;
}
} else if (actionID === "think") {
text = action.payload.args.thought;
}
const message: Message = {
type: "action",
sender: "assistant",
translationID,
eventID: action.payload.id,
content: text,
imageUrls: [],
timestamp: new Date().toISOString(),
action,
};
state.messages.push(message);
},
addAssistantObservation(
state: SliceState,
observation: PayloadAction<OpenHandsObservation>,
) {
const observationID = observation.payload.observation;
if (!HANDLED_ACTIONS.includes(observationID)) {
return;
}
// Special handling for RecallObservation - create a new message instead of updating an existing one
if (observationID === "recall") {
const recallObs = observation.payload as RecallObservation;
let content = ``;
// Handle workspace context
if (recallObs.extras.recall_type === "workspace_context") {
if (recallObs.extras.repo_name) {
content += `\n\n**Repository:** ${recallObs.extras.repo_name}`;
}
if (recallObs.extras.repo_directory) {
content += `\n\n**Directory:** ${recallObs.extras.repo_directory}`;
}
if (recallObs.extras.date) {
content += `\n\n**Date:** ${recallObs.extras.date}`;
}
if (
recallObs.extras.runtime_hosts &&
Object.keys(recallObs.extras.runtime_hosts).length > 0
) {
content += `\n\n**Available Hosts**`;
for (const [host, port] of Object.entries(
recallObs.extras.runtime_hosts,
)) {
content += `\n\n- ${host} (port ${port})`;
}
}
if (
recallObs.extras.custom_secrets_descriptions &&
Object.keys(recallObs.extras.custom_secrets_descriptions).length > 0
) {
content += `\n\n**Custom Secrets**`;
for (const [name, description] of Object.entries(
recallObs.extras.custom_secrets_descriptions,
)) {
content += `\n\n- $${name}: ${description}`;
}
}
if (recallObs.extras.repo_instructions) {
content += `\n\n**Repository Instructions:**\n\n${recallObs.extras.repo_instructions}`;
}
if (recallObs.extras.additional_agent_instructions) {
content += `\n\n**Additional Instructions:**\n\n${recallObs.extras.additional_agent_instructions}`;
}
}
// Create a new message for the observation
// Use the correct translation ID format that matches what's in the i18n file
const translationID = `OBSERVATION_MESSAGE$${observationID.toUpperCase()}`;
// Handle microagent knowledge
if (
recallObs.extras.microagent_knowledge &&
recallObs.extras.microagent_knowledge.length > 0
) {
content += `\n\n**Triggered Microagent Knowledge:**`;
for (const knowledge of recallObs.extras.microagent_knowledge) {
content += `\n\n- **${knowledge.name}** (triggered by keyword: ${knowledge.trigger})\n\n\`\`\`\n${knowledge.content}\n\`\`\``;
}
}
const message: Message = {
type: "action",
sender: "assistant",
translationID,
eventID: observation.payload.id,
content,
imageUrls: [],
timestamp: new Date().toISOString(),
success: true,
};
state.messages.push(message);
return; // Skip the normal observation handling below
}
// Normal handling for other observation types
const translationID = `OBSERVATION_MESSAGE$${observationID.toUpperCase()}`;
const causeID = observation.payload.cause;
const causeMessage = state.messages.find(
(message) => message.eventID === causeID,
);
if (!causeMessage) {
return;
}
causeMessage.translationID = translationID;
causeMessage.observation = observation;
// Set success property based on observation type
if (observationID === "run") {
const commandObs = observation.payload as CommandObservation;
// If exit_code is -1, it means the command timed out, so we set success to undefined
// to not show any status indicator
if (commandObs.extras.metadata.exit_code === -1) {
causeMessage.success = undefined;
} else {
causeMessage.success = commandObs.extras.metadata.exit_code === 0;
}
} else if (observationID === "run_ipython") {
// For IPython, we consider it successful if there's no error message
const ipythonObs = observation.payload as IPythonObservation;
causeMessage.success = !ipythonObs.content
.toLowerCase()
.includes("error:");
} else if (observationID === "read" || observationID === "edit") {
// For read/edit operations, we consider it successful if there's content and no error
if (observation.payload.extras.impl_source === "oh_aci") {
causeMessage.success =
observation.payload.content.length > 0 &&
!observation.payload.content.startsWith("ERROR:\n");
} else {
causeMessage.success =
observation.payload.content.length > 0 &&
!observation.payload.content.toLowerCase().includes("error:");
}
}
if (observationID === "run" || observationID === "run_ipython") {
let { content } = observation.payload;
if (content.length > MAX_CONTENT_LENGTH) {
content = `${content.slice(0, MAX_CONTENT_LENGTH)}...`;
}
content = `${causeMessage.content}\n\nOutput:\n\`\`\`\n${content.trim() || "[Command finished execution with no output]"}\n\`\`\``;
causeMessage.content = content; // Observation content includes the action
} else if (observationID === "read") {
causeMessage.content = `\`\`\`\n${observation.payload.content}\n\`\`\``; // Content is already truncated by the ACI
} else if (observationID === "edit") {
if (causeMessage.success) {
causeMessage.content = `\`\`\`diff\n${observation.payload.extras.diff}\n\`\`\``; // Content is already truncated by the ACI
} else {
causeMessage.content = observation.payload.content;
}
} else if (observationID === "browse") {
let content = `**URL:** ${observation.payload.extras.url}\n`;
if (observation.payload.extras.error) {
content += `\n\n**Error:**\n${observation.payload.extras.error}\n`;
}
content += `\n\n**Output:**\n${observation.payload.content}`;
if (content.length > MAX_CONTENT_LENGTH) {
content = `${content.slice(0, MAX_CONTENT_LENGTH)}...(truncated)`;
}
causeMessage.content = content;
} else if (observationID === "mcp") {
// For MCP observations, we want to show the content as formatted output
// similar to how run/run_ipython actions are handled
let { content } = observation.payload;
if (content.length > MAX_CONTENT_LENGTH) {
content = `${content.slice(0, MAX_CONTENT_LENGTH)}...`;
}
content = `${causeMessage.content}\n\n**Output:**\n\`\`\`\n${content.trim() || "[MCP Tool finished execution with no output]"}\n\`\`\``;
causeMessage.content = content; // Observation content includes the action
// Set success based on whether there's an error message
causeMessage.success = !observation.payload.content
.toLowerCase()
.includes("error:");
}
},
addErrorMessage(
state: SliceState,
action: PayloadAction<{ id?: string; message: string }>,
) {
const { id, message } = action.payload;
state.messages.push({
translationID: id,
content: message,
type: "error",
sender: "assistant",
timestamp: new Date().toISOString(),
});
},
clearMessages(state: SliceState) {
state.messages = [];
state.systemMessage = null;
},
},
});
export const {
addUserMessage,
addAssistantMessage,
addAssistantAction,
addAssistantObservation,
addErrorMessage,
clearMessages,
} = chatSlice.actions;
// Selectors
export const selectSystemMessage = (state: { chat: SliceState }) =>
state.chat.systemMessage;
export default chatSlice.reducer;

2
frontend/src/store.ts
View File

@@ -1,7 +1,6 @@
import { combineReducers, configureStore } from "@reduxjs/toolkit";
import agentReducer from "./state/agent-slice";
import browserReducer from "./state/browser-slice";
import chatReducer from "./state/chat-slice";
import codeReducer from "./state/code-slice";
import fileStateReducer from "./state/file-state-slice";
import initialQueryReducer from "./state/initial-query-slice";
@@ -15,7 +14,6 @@ export const rootReducer = combineReducers({
fileState: fileStateReducer,
initialQuery: initialQueryReducer,
browser: browserReducer,
chat: chatReducer,
code: codeReducer,
cmd: commandReducer,
agent: agentReducer,

8
frontend/src/types/core/base.ts
View File

@@ -2,6 +2,7 @@ export type OpenHandsEventType =
| "message"
| "system"
| "agent_state_changed"
| "change_agent_state"
| "run"
| "read"
| "write"
@@ -16,11 +17,14 @@ export type OpenHandsEventType =
| "error"
| "recall"
| "mcp"
| "call_tool_mcp";
| "call_tool_mcp"
| "user_rejected";
export type OpenHandsSourceType = "agent" | "user" | "environment";
interface OpenHandsBaseEvent {
id: number;
source: "agent" | "user";
source: OpenHandsSourceType;
message: string;
timestamp: string; // ISO 8601
}

59
frontend/src/types/core/guards.ts Normal file
View File

@@ -0,0 +1,59 @@
import { OpenHandsParsedEvent } from ".";
import {
UserMessageAction,
AssistantMessageAction,
OpenHandsAction,
SystemMessageAction,
} from "./actions";
import {
CommandObservation,
ErrorObservation,
OpenHandsObservation,
} from "./observations";
export const isOpenHandsAction = (
event: OpenHandsParsedEvent,
): event is OpenHandsAction => "action" in event;
export const isOpenHandsObservation = (
event: OpenHandsParsedEvent,
): event is OpenHandsObservation => "observation" in event;
export const isUserMessage = (
event: OpenHandsParsedEvent,
): event is UserMessageAction =>
isOpenHandsAction(event) &&
event.source === "user" &&
event.action === "message";
export const isAssistantMessage = (
event: OpenHandsParsedEvent,
): event is AssistantMessageAction =>
isOpenHandsAction(event) &&
event.source === "agent" &&
(event.action === "message" || event.action === "finish");
export const isErrorObservation = (
event: OpenHandsParsedEvent,
): event is ErrorObservation =>
isOpenHandsObservation(event) && event.observation === "error";
export const isCommandObservation = (
event: OpenHandsParsedEvent,
): event is CommandObservation =>
isOpenHandsObservation(event) && event.observation === "run";
export const isFinishAction = (
event: OpenHandsParsedEvent,
): event is AssistantMessageAction =>
isOpenHandsAction(event) && event.action === "finish";
export const isSystemMessage = (
event: OpenHandsParsedEvent,
): event is SystemMessageAction =>
isOpenHandsAction(event) && event.action === "system";
export const isRejectObservation = (
event: OpenHandsParsedEvent,
): event is OpenHandsObservation =>
isOpenHandsObservation(event) && event.observation === "user_rejected";

11
frontend/src/types/core/observations.ts
View File

@@ -138,6 +138,14 @@ export interface MCPObservation extends OpenHandsObservationEvent<"mcp"> {
};
}
export interface UserRejectedObservation
extends OpenHandsObservationEvent<"user_rejected"> {
source: "agent";
extras: {
// Add any specific fields for MCP observations
};
}
export type OpenHandsObservation =
| AgentStateChangeObservation
| AgentThinkObservation
@@ -151,4 +159,5 @@ export type OpenHandsObservation =
| EditObservation
| ErrorObservation
| RecallObservation
| MCPObservation;
| MCPObservation
| UserRejectedObservation;