diff --git a/autogpt_platform/frontend/src/app/api/proxy/[...path]/route.ts b/autogpt_platform/frontend/src/app/api/proxy/[...path]/route.ts index 293c406373..2b84797e57 100644 --- a/autogpt_platform/frontend/src/app/api/proxy/[...path]/route.ts +++ b/autogpt_platform/frontend/src/app/api/proxy/[...path]/route.ts @@ -1,5 +1,6 @@ import { ApiError, + getServerAuthToken, makeAuthenticatedFileUpload, makeAuthenticatedRequest, } from "@/lib/autogpt-server-api/helpers"; @@ -15,6 +16,69 @@ function buildBackendUrl(path: string[], queryString: string): string { return `${environment.getAGPTServerBaseUrl()}/${backendPath}${queryString}`; } +/** + * Check if this is a workspace file download request that needs binary response handling. + */ +function isWorkspaceDownloadRequest(path: string[]): boolean { + // Match pattern: api/workspace/files/{id}/download + return ( + path.length >= 4 && + path[0] === "api" && + path[1] === "workspace" && + path[2] === "files" && + path[path.length - 1] === "download" + ); +} + +/** + * Handle workspace file download requests with proper binary response streaming. + */ +async function handleWorkspaceDownload( + req: NextRequest, + backendUrl: string, +): Promise { + const token = await getServerAuthToken(); + + const headers: Record = {}; + if (token && token !== "no-token-found") { + headers["Authorization"] = `Bearer ${token}`; + } + + const response = await fetch(backendUrl, { + method: "GET", + headers, + redirect: "follow", // Follow redirects to signed URLs + }); + + if (!response.ok) { + return NextResponse.json( + { error: `Failed to download file: ${response.statusText}` }, + { status: response.status }, + ); + } + + // Get the content type from the backend response + const contentType = + response.headers.get("Content-Type") || "application/octet-stream"; + const contentDisposition = response.headers.get("Content-Disposition"); + + // Stream the response body + const responseHeaders: Record = { + "Content-Type": contentType, + }; + + if (contentDisposition) { + responseHeaders["Content-Disposition"] = contentDisposition; + } + + // Return the binary content + const arrayBuffer = await response.arrayBuffer(); + return new NextResponse(arrayBuffer, { + status: 200, + headers: responseHeaders, + }); +} + async function handleJsonRequest( req: NextRequest, method: string, @@ -180,6 +244,11 @@ async function handler( }; try { + // Handle workspace file downloads separately (binary response) + if (method === "GET" && isWorkspaceDownloadRequest(path)) { + return await handleWorkspaceDownload(req, backendUrl); + } + if (method === "GET" || method === "DELETE") { responseBody = await handleGetDeleteRequest(method, backendUrl, req); } else if (contentType?.includes("application/json")) { diff --git a/autogpt_platform/frontend/src/components/contextual/Chat/components/MarkdownContent/MarkdownContent.tsx b/autogpt_platform/frontend/src/components/contextual/Chat/components/MarkdownContent/MarkdownContent.tsx index 51a0794090..24c128c474 100644 --- a/autogpt_platform/frontend/src/components/contextual/Chat/components/MarkdownContent/MarkdownContent.tsx +++ b/autogpt_platform/frontend/src/components/contextual/Chat/components/MarkdownContent/MarkdownContent.tsx @@ -1,6 +1,8 @@ "use client"; +import { getGetV2DownloadFileByIdUrl } from "@/app/api/__generated__/endpoints/workspace/workspace"; import { cn } from "@/lib/utils"; +import { EyeSlash } from "@phosphor-icons/react"; import React from "react"; import ReactMarkdown from "react-markdown"; import remarkGfm from "remark-gfm"; @@ -29,12 +31,90 @@ interface InputProps extends React.InputHTMLAttributes { type?: string; } +/** + * Converts a workspace:// URL to a proxy URL that routes through Next.js to the backend. + * workspace://abc123 -> /api/proxy/api/workspace/files/abc123/download + * + * Uses the generated API URL helper and routes through the Next.js proxy + * which handles authentication and proper backend routing. + */ +function resolveWorkspaceUrl(src: string): string { + if (src.startsWith("workspace://")) { + const fileId = src.replace("workspace://", ""); + // Use the generated API URL helper to get the correct path + const apiPath = getGetV2DownloadFileByIdUrl(fileId); + // Route through the Next.js proxy (same pattern as customMutator for client-side) + return `/api/proxy${apiPath}`; + } + return src; +} + +/** + * URL transformer for ReactMarkdown. + * Transforms workspace:// URLs to backend API download URLs before rendering. + * This is needed because ReactMarkdown sanitizes URLs and only allows + * http, https, mailto, and tel protocols by default. + */ +function transformUrl(url: string): string { + return resolveWorkspaceUrl(url); +} + +/** + * Check if the image URL is a workspace file (AI cannot see these yet). + * After URL transformation, workspace files have URLs like /api/proxy/api/workspace/files/... + */ +function isWorkspaceImage(src: string | undefined): boolean { + return src?.includes("/workspace/files/") ?? false; +} + +/** + * Custom image component that shows an indicator when the AI cannot see the image. + * Note: src is already transformed by urlTransform, so workspace:// is now /api/workspace/... + */ +function MarkdownImage(props: Record) { + const src = props.src as string | undefined; + const alt = props.alt as string | undefined; + + const aiCannotSee = isWorkspaceImage(src); + + // If no src, show a placeholder + if (!src) { + return ( + + [Image: {alt || "missing src"}] + + ); + } + + return ( + + {/* eslint-disable-next-line @next/next/no-img-element */} + {alt + {aiCannotSee && ( + + + AI cannot see this image + + )} + + ); +} + export function MarkdownContent({ content, className }: MarkdownContentProps) { return (
{ const isInline = !className?.includes("language-"); @@ -206,6 +286,9 @@ export function MarkdownContent({ content, className }: MarkdownContentProps) { {children} ), + img: ({ src, alt, ...props }) => ( + + ), }} > {content} diff --git a/autogpt_platform/frontend/src/components/contextual/Chat/components/ToolResponseMessage/helpers.ts b/autogpt_platform/frontend/src/components/contextual/Chat/components/ToolResponseMessage/helpers.ts index 400f32936e..d6667c96de 100644 --- a/autogpt_platform/frontend/src/components/contextual/Chat/components/ToolResponseMessage/helpers.ts +++ b/autogpt_platform/frontend/src/components/contextual/Chat/components/ToolResponseMessage/helpers.ts @@ -37,6 +37,77 @@ export function getErrorMessage(result: unknown): string { return "An error occurred"; } +/** + * Check if a value is a workspace file reference. + */ +function isWorkspaceRef(value: unknown): value is string { + return typeof value === "string" && value.startsWith("workspace://"); +} + +/** + * Check if a workspace reference appears to be an image based on common patterns. + * Since workspace refs don't have extensions, we check the context or assume image + * for certain block types. + */ +function isLikelyImageRef(value: string, outputKey?: string): boolean { + if (!isWorkspaceRef(value)) return false; + + // Check output key name for image-related hints + const imageKeywords = [ + "image", + "img", + "photo", + "picture", + "thumbnail", + "avatar", + "icon", + "screenshot", + "output", + "result", + "generated", + ]; + if (outputKey) { + const lowerKey = outputKey.toLowerCase(); + if (imageKeywords.some((kw) => lowerKey.includes(kw))) { + return true; + } + } + + // Default to treating workspace refs as potential images + // since that's the most common case for generated content + return true; +} + +/** + * Format a single output value, converting workspace refs to markdown images. + */ +function formatOutputValue(value: unknown, outputKey?: string): string { + if (isWorkspaceRef(value) && isLikelyImageRef(value, outputKey)) { + // Format as markdown image + return `![${outputKey || "Generated image"}](${value})`; + } + + if (typeof value === "string") { + // Check for data URIs (images) + if (value.startsWith("data:image/")) { + return `![${outputKey || "Generated image"}](${value})`; + } + return value; + } + + if (Array.isArray(value)) { + return value + .map((item, idx) => formatOutputValue(item, `${outputKey}_${idx}`)) + .join("\n\n"); + } + + if (typeof value === "object" && value !== null) { + return JSON.stringify(value, null, 2); + } + + return String(value); +} + function getToolCompletionPhrase(toolName: string): string { const toolCompletionPhrases: Record = { add_understanding: "Updated your business information", @@ -127,10 +198,26 @@ export function formatToolResponse(result: unknown, toolName: string): string { case "block_output": const blockName = (response.block_name as string) || "Block"; - const outputs = response.outputs as Record | undefined; + const outputs = response.outputs as Record | undefined; if (outputs && Object.keys(outputs).length > 0) { - const outputKeys = Object.keys(outputs); - return `${blockName} executed successfully. Outputs: ${outputKeys.join(", ")}`; + const formattedOutputs: string[] = []; + + for (const [key, values] of Object.entries(outputs)) { + if (!Array.isArray(values) || values.length === 0) continue; + + // Format each value in the output array + for (const value of values) { + const formatted = formatOutputValue(value, key); + if (formatted) { + formattedOutputs.push(formatted); + } + } + } + + if (formattedOutputs.length > 0) { + return `${blockName} executed successfully.\n\n${formattedOutputs.join("\n\n")}`; + } + return `${blockName} executed successfully.`; } return `${blockName} executed successfully.`;