From 599ffb77e646a0bd5c8245dd9cfd01686d0377eb Mon Sep 17 00:00:00 2001 From: Siddharth Ganesan Date: Thu, 29 Jan 2026 17:19:29 -0800 Subject: [PATCH] v1 --- .../messages-input/messages-input.tsx | 196 ++++++++++++++-- .../w/[workflowId]/hooks/use-wand.ts | 6 + apps/sim/blocks/blocks/agent.ts | 20 +- .../executor/handlers/agent/agent-handler.ts | 209 ++++++++++++++++-- apps/sim/executor/handlers/agent/types.ts | 8 +- apps/sim/providers/bedrock/index.ts | 5 +- apps/sim/providers/bedrock/utils.ts | 188 +++++++++++++++- apps/sim/providers/models.ts | 106 ++++++++- apps/sim/providers/types.ts | 4 +- apps/sim/providers/utils.ts | 5 + 10 files changed, 698 insertions(+), 49 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/messages-input/messages-input.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/messages-input/messages-input.tsx index e69f04b0b..838025395 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/messages-input/messages-input.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/messages-input/messages-input.tsx @@ -9,6 +9,8 @@ import { } from 'react' import { isEqual } from 'lodash' import { ArrowLeftRight, ChevronDown, ChevronsUpDown, ChevronUp, Plus } from 'lucide-react' +import { useParams } from 'next/navigation' +import { createLogger } from '@sim/logger' import { Button, Popover, @@ -22,15 +24,28 @@ import { cn } from '@/lib/core/utils/cn' import { EnvVarDropdown } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/env-var-dropdown' import { FileUpload } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/file-upload/file-upload' import { formatDisplayText } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/formatted-text' +import { ShortInput } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/short-input/short-input' import { TagDropdown } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/tag-dropdown/tag-dropdown' import { useSubBlockInput } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-sub-block-input' import { useSubBlockValue } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-sub-block-value' +import { useWorkflowRegistry } from '@/stores/workflows/registry/store' +import { useSubBlockStore } from '@/stores/workflows/subblock/store' import type { WandControlHandlers } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/sub-block' import { useAccessibleReferencePrefixes } from '@/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-accessible-reference-prefixes' import { useWand } from '@/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-wand' import type { SubBlockConfig } from '@/blocks/types' +const logger = createLogger('MessagesInput') + const MIN_TEXTAREA_HEIGHT_PX = 80 + +/** Workspace file record from API */ +interface WorkspaceFile { + id: string + name: string + path: string + type: string +} const MAX_TEXTAREA_HEIGHT_PX = 320 /** Pattern to match complete message objects in JSON */ @@ -49,14 +64,20 @@ const ROLE_BEFORE_CONTENT_PATTERN = /"role"\s*:\s*"(system|user|assistant|media) const unescapeContent = (str: string): string => str.replace(/\\n/g, '\n').replace(/\\"/g, '"').replace(/\\\\/g, '\\') - /** * Media content for multimodal messages */ interface MediaContent { + /** Source type: how the data was provided */ + sourceType: 'url' | 'base64' | 'file' + /** The URL or base64 data */ data: string + /** MIME type (e.g., 'image/png', 'application/pdf', 'audio/mp3') */ mimeType?: string + /** Optional filename for file uploads */ fileName?: string + /** Optional workspace file ID (used by wand to select existing files) */ + fileId?: string } /** @@ -93,14 +114,109 @@ export function MessagesInput({ disabled = false, wandControlRef, }: MessagesInputProps) { + const params = useParams() + const workspaceId = params?.workspaceId as string const [messages, setMessages] = useSubBlockValue(blockId, subBlockId, false) const [localMessages, setLocalMessages] = useState([{ role: 'user', content: '' }]) const accessiblePrefixes = useAccessibleReferencePrefixes(blockId) const [openPopoverIndex, setOpenPopoverIndex] = useState(null) + const { activeWorkflowId } = useWorkflowRegistry() // Local media mode state - basic = FileUpload, advanced = URL/base64 textarea const [mediaMode, setMediaMode] = useState<'basic' | 'advanced'>('basic') + // Workspace files for wand context + const [workspaceFiles, setWorkspaceFiles] = useState([]) + + // Fetch workspace files for wand context + const loadWorkspaceFiles = useCallback(async () => { + if (!workspaceId || isPreview) return + + try { + const response = await fetch(`/api/workspaces/${workspaceId}/files`) + const data = await response.json() + if (data.success) { + setWorkspaceFiles(data.files || []) + } + } catch (error) { + logger.error('Error loading workspace files:', error) + } + }, [workspaceId, isPreview]) + + // Load workspace files on mount + useEffect(() => { + void loadWorkspaceFiles() + }, [loadWorkspaceFiles]) + + // Build sources string for wand - available workspace files + const sourcesInfo = useMemo(() => { + if (workspaceFiles.length === 0) { + return 'No workspace files available. The user can upload files manually after generation.' + } + + const filesList = workspaceFiles + .filter((f) => f.type.startsWith('image/') || f.type.startsWith('audio/') || f.type.startsWith('video/') || f.type === 'application/pdf') + .map((f) => ` - id: "${f.id}", name: "${f.name}", type: "${f.type}"`) + .join('\n') + + if (!filesList) { + return 'No media files in workspace. The user can upload files manually after generation.' + } + + return `AVAILABLE WORKSPACE FILES (optional - you don't have to select one):\n${filesList}\n\nTo use a file, include "fileId": "" in the media object. If not selecting a file, omit the fileId field.` + }, [workspaceFiles]) + + + // Effect to sync FileUpload values to message media objects + useEffect(() => { + if (!activeWorkflowId || isPreview) return + + // Get all subblock values for this workflow + const workflowValues = useSubBlockStore.getState().workflowValues[activeWorkflowId] + if (!workflowValues?.[blockId]) return + + let hasChanges = false + const updatedMessages = localMessages.map((msg, index) => { + if (msg.role !== 'media') return msg + + // Check if there's a FileUpload value for this media message + const fileUploadKey = `${subBlockId}-media-${index}` + const fileValue = workflowValues[blockId][fileUploadKey] + + if (fileValue && typeof fileValue === 'object' && 'path' in fileValue) { + const uploadedFile = fileValue as { name: string; path: string; type: string; size: number } + const newMedia: MediaContent = { + sourceType: 'file', + data: uploadedFile.path, + mimeType: uploadedFile.type, + fileName: uploadedFile.name, + } + + // Only update if different + if ( + msg.media?.data !== newMedia.data || + msg.media?.sourceType !== newMedia.sourceType || + msg.media?.mimeType !== newMedia.mimeType || + msg.media?.fileName !== newMedia.fileName + ) { + hasChanges = true + return { + ...msg, + content: uploadedFile.name || msg.content, + media: newMedia, + } + } + } + + return msg + }) + + if (hasChanges) { + setLocalMessages(updatedMessages) + setMessages(updatedMessages) + } + }, [activeWorkflowId, blockId, subBlockId, localMessages, isPreview, setMessages]) + const subBlockInput = useSubBlockInput({ blockId, subBlockId, @@ -186,6 +302,7 @@ export function MessagesInput({ const wandHook = useWand({ wandConfig: config.wandConfig, currentValue: getMessagesJson(), + sources: sourcesInfo, onStreamStart: () => { streamBufferRef.current = '' setLocalMessages([{ role: 'system', content: '' }]) @@ -200,6 +317,46 @@ export function MessagesInput({ onGeneratedContent: (content) => { const validMessages = parseMessages(content) if (validMessages) { + // Process media messages - only allow fileId to set files, sanitize other attempts + validMessages.forEach((msg, index) => { + if (msg.role === 'media') { + // Check if this is an existing file with valid data (preserve it) + const hasExistingFile = msg.media?.sourceType === 'file' && + msg.media?.data?.startsWith('/api/') && + msg.media?.fileName + + if (hasExistingFile) { + // Preserve existing file data as-is + return + } + + // Check if wand provided a fileId to select a workspace file + if (msg.media?.fileId) { + const file = workspaceFiles.find((f) => f.id === msg.media?.fileId) + if (file) { + // Set the file value in SubBlockStore so FileUpload picks it up + const fileUploadKey = `${subBlockId}-media-${index}` + const uploadedFile = { + name: file.name, + path: file.path, + type: file.type, + size: 0, // Size not available from workspace files list + } + useSubBlockStore.getState().setValue(blockId, fileUploadKey, uploadedFile) + + // Clear the media object - the FileUpload will sync the file data via useEffect + // DON'T set media.data here as it would appear in the ShortInput (advanced mode) + msg.media = undefined + return + } + } + + // Sanitize: clear any media object that isn't a valid existing file or fileId match + // This prevents the LLM from setting arbitrary data/variable references + msg.media = undefined + } + }) + setLocalMessages(validMessages) setMessages(validMessages) } else { @@ -283,6 +440,7 @@ export function MessagesInput({ role, content: updatedMessages[index].content || '', media: updatedMessages[index].media || { + sourceType: 'file', data: '', }, } @@ -700,29 +858,41 @@ export function MessagesInput({ disabled={disabled} /> ) : ( -