diff --git a/frontend/src/i18n/declaration.ts b/frontend/src/i18n/declaration.ts index 8fbfbcbaa9..ace7cfd4df 100644 --- a/frontend/src/i18n/declaration.ts +++ b/frontend/src/i18n/declaration.ts @@ -281,6 +281,7 @@ export enum I18nKey { ACTION_MESSAGE$EDIT = "ACTION_MESSAGE$EDIT", ACTION_MESSAGE$WRITE = "ACTION_MESSAGE$WRITE", ACTION_MESSAGE$BROWSE = "ACTION_MESSAGE$BROWSE", + ACTION_MESSAGE$BROWSE_INTERACTIVE = "ACTION_MESSAGE$BROWSE_INTERACTIVE", ACTION_MESSAGE$THINK = "ACTION_MESSAGE$THINK", OBSERVATION_MESSAGE$RUN = "OBSERVATION_MESSAGE$RUN", OBSERVATION_MESSAGE$RUN_IPYTHON = "OBSERVATION_MESSAGE$RUN_IPYTHON", diff --git a/frontend/src/i18n/translation.json b/frontend/src/i18n/translation.json index 3163f85f2d..dce5472842 100644 --- a/frontend/src/i18n/translation.json +++ b/frontend/src/i18n/translation.json @@ -4193,6 +4193,21 @@ "es": "Navegando en la web", "tr": "Web'de geziniyor" }, + "ACTION_MESSAGE$BROWSE_INTERACTIVE": { + "en": "Interactive browsing in progress...", + "zh-CN": "交互式浏览进行中...", + "zh-TW": "互動式瀏覽進行中...", + "ko-KR": "인터랙티브 브라우징 진행 중...", + "ja": "インタラクティブブラウジング進行中...", + "no": "Interaktiv surfing pågår...", + "ar": "التصفح التفاعلي قيد التقدم...", + "de": "Interaktives Browsen läuft...", + "fr": "Navigation interactive en cours...", + "it": "Navigazione interattiva in corso...", + "pt": "Navegação interativa em andamento...", + "es": "Navegación interactiva en progreso...", + "tr": "Etkileşimli tarama devam ediyor..." + }, "ACTION_MESSAGE$THINK": { "en": "Thinking", "zh-CN": "思考", diff --git a/frontend/src/services/observations.ts b/frontend/src/services/observations.ts index 9760e0c804..d867ff971e 100644 --- a/frontend/src/services/observations.ts +++ b/frontend/src/services/observations.ts @@ -30,6 +30,7 @@ export function handleObservationMessage(message: ObservationMessage) { store.dispatch(appendJupyterOutput(message.content)); break; case ObservationType.BROWSE: + case ObservationType.BROWSE_INTERACTIVE: if (message.extras?.screenshot) { store.dispatch(setScreenshotSrc(message.extras?.screenshot)); } @@ -178,6 +179,46 @@ export function handleObservationMessage(message: ObservationMessage) { }), ); break; + case "browse_interactive": + store.dispatch( + addAssistantObservation({ + ...baseObservation, + observation: "browse_interactive" as const, + extras: { + url: String(message.extras.url || ""), + screenshot: String(message.extras.screenshot || ""), + error: Boolean(message.extras.error), + open_page_urls: Array.isArray(message.extras.open_page_urls) + ? message.extras.open_page_urls + : [], + active_page_index: Number(message.extras.active_page_index || 0), + dom_object: + typeof message.extras.dom_object === "object" + ? (message.extras.dom_object as Record) + : {}, + axtree_object: + typeof message.extras.axtree_object === "object" + ? (message.extras.axtree_object as Record) + : {}, + extra_element_properties: + typeof message.extras.extra_element_properties === "object" + ? (message.extras.extra_element_properties as Record< + string, + unknown + >) + : {}, + last_browser_action: String( + message.extras.last_browser_action || "", + ), + last_browser_action_error: + message.extras.last_browser_action_error, + focused_element_bid: String( + message.extras.focused_element_bid || "", + ), + }, + }), + ); + break; case "error": store.dispatch( addAssistantObservation({ diff --git a/frontend/src/state/chat-slice.ts b/frontend/src/state/chat-slice.ts index 8a0bc20d39..97e19e65e3 100644 --- a/frontend/src/state/chat-slice.ts +++ b/frontend/src/state/chat-slice.ts @@ -20,6 +20,7 @@ const HANDLED_ACTIONS: OpenHandsEventType[] = [ "write", "read", "browse", + "browse_interactive", "edit", ]; @@ -108,6 +109,9 @@ export const chatSlice = createSlice({ text = `${action.payload.args.path}\n${content}`; } else if (actionID === "browse") { text = `Browsing ${action.payload.args.url}`; + } else if (actionID === "browse_interactive") { + // Include the browser_actions in the content + text = `**Action:**\n\n\`\`\`python\n${action.payload.args.browser_actions}\n\`\`\``; } if (actionID === "run" || actionID === "run_ipython") { if ( @@ -127,6 +131,7 @@ export const chatSlice = createSlice({ imageUrls: [], timestamp: new Date().toISOString(), }; + state.messages.push(message); }, @@ -191,11 +196,11 @@ export const chatSlice = createSlice({ } else if (observationID === "browse") { let content = `**URL:** ${observation.payload.extras.url}\n`; if (observation.payload.extras.error) { - content += `**Error:**\n${observation.payload.extras.error}\n`; + content += `\n\n**Error:**\n${observation.payload.extras.error}\n`; } - content += `**Output:**\n${observation.payload.content}`; + content += `\n\n**Output:**\n${observation.payload.content}`; if (content.length > MAX_CONTENT_LENGTH) { - content = `${content.slice(0, MAX_CONTENT_LENGTH)}...`; + content = `${content.slice(0, MAX_CONTENT_LENGTH)}...(truncated)`; } causeMessage.content = content; } diff --git a/frontend/src/types/core/observations.ts b/frontend/src/types/core/observations.ts index 30ecbde9af..4224aa5e4b 100644 --- a/frontend/src/types/core/observations.ts +++ b/frontend/src/types/core/observations.ts @@ -51,6 +51,24 @@ export interface BrowseObservation extends OpenHandsObservationEvent<"browse"> { }; } +export interface BrowseInteractiveObservation + extends OpenHandsObservationEvent<"browse_interactive"> { + source: "agent"; + extras: { + url: string; + screenshot: string; + error: boolean; + open_page_urls: string[]; + active_page_index: number; + dom_object: Record; + axtree_object: Record; + extra_element_properties: Record; + last_browser_action: string; + last_browser_action_error: unknown; + focused_element_bid: string; + }; +} + export interface WriteObservation extends OpenHandsObservationEvent<"write"> { source: "agent"; extras: { @@ -98,6 +116,7 @@ export type OpenHandsObservation = | IPythonObservation | DelegateObservation | BrowseObservation + | BrowseInteractiveObservation | WriteObservation | ReadObservation | EditObservation diff --git a/frontend/src/types/observation-type.tsx b/frontend/src/types/observation-type.tsx index 386d509ece..08e7d3bffa 100644 --- a/frontend/src/types/observation-type.tsx +++ b/frontend/src/types/observation-type.tsx @@ -8,6 +8,9 @@ enum ObservationType { // The HTML contents of a URL BROWSE = "browse", + // Interactive browsing + BROWSE_INTERACTIVE = "browse_interactive", + // The output of a command RUN = "run",