mirror of
https://github.com/simstudioai/sim.git
synced 2026-01-29 16:58:11 -05:00
Add toggle, haven't tested
This commit is contained in:
Siddharth Ganesan
@@ -8,11 +8,19 @@ import {
|
||||
useState,
|
||||
} from 'react'
|
||||
import { isEqual } from 'lodash'
|
||||
import { ChevronDown, ChevronsUpDown, ChevronUp, Plus } from 'lucide-react'
|
||||
import { Button, Popover, PopoverContent, PopoverItem, PopoverTrigger } from '@/components/emcn'
|
||||
import { ArrowLeftRight, ChevronDown, ChevronsUpDown, ChevronUp, Plus } from 'lucide-react'
|
||||
import {
|
||||
Button,
|
||||
Popover,
|
||||
PopoverContent,
|
||||
PopoverItem,
|
||||
PopoverTrigger,
|
||||
Tooltip,
|
||||
} from '@/components/emcn'
|
||||
import { Trash } from '@/components/emcn/icons/trash'
|
||||
import { cn } from '@/lib/core/utils/cn'
|
||||
import { EnvVarDropdown } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/env-var-dropdown'
|
||||
import { FileUpload } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/file-upload/file-upload'
|
||||
import { formatDisplayText } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/formatted-text'
|
||||
import { TagDropdown } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/tag-dropdown/tag-dropdown'
|
||||
import { useSubBlockInput } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-sub-block-input'
|
||||
@@ -27,13 +35,13 @@ const MAX_TEXTAREA_HEIGHT_PX = 320
|
||||
|
||||
/** Pattern to match complete message objects in JSON */
|
||||
const COMPLETE_MESSAGE_PATTERN =
|
||||
/"role"\s*:\s*"(system|user|assistant)"[^}]*"content"\s*:\s*"((?:[^"\\]|\\.)*)"/g
|
||||
/"role"\s*:\s*"(system|user|assistant|media)"[^}]*"content"\s*:\s*"((?:[^"\\]|\\.)*)"/g
|
||||
|
||||
/** Pattern to match incomplete content at end of buffer */
|
||||
const INCOMPLETE_CONTENT_PATTERN = /"content"\s*:\s*"((?:[^"\\]|\\.)*)$/
|
||||
|
||||
/** Pattern to match role before content */
|
||||
const ROLE_BEFORE_CONTENT_PATTERN = /"role"\s*:\s*"(system|user|assistant)"[^{]*$/
|
||||
const ROLE_BEFORE_CONTENT_PATTERN = /"role"\s*:\s*"(system|user|assistant|media)"[^{]*$/
|
||||
|
||||
/**
|
||||
* Unescapes JSON string content
|
||||
@@ -41,41 +49,40 @@ const ROLE_BEFORE_CONTENT_PATTERN = /"role"\s*:\s*"(system|user|assistant)"[^{]*
|
||||
const unescapeContent = (str: string): string =>
|
||||
str.replace(/\\n/g, '\n').replace(/\\"/g, '"').replace(/\\\\/g, '\\')
|
||||
|
||||
|
||||
/**
|
||||
* Media content for multimodal messages
|
||||
*/
|
||||
interface MediaContent {
|
||||
data: string
|
||||
mimeType?: string
|
||||
fileName?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Interface for individual message in the messages array
|
||||
*/
|
||||
interface Message {
|
||||
role: 'system' | 'user' | 'assistant'
|
||||
role: 'system' | 'user' | 'assistant' | 'media'
|
||||
content: string
|
||||
media?: MediaContent
|
||||
}
|
||||
|
||||
/**
|
||||
* Props for the MessagesInput component
|
||||
*/
|
||||
interface MessagesInputProps {
|
||||
/** Unique identifier for the block */
|
||||
blockId: string
|
||||
/** Unique identifier for the sub-block */
|
||||
subBlockId: string
|
||||
/** Configuration object for the sub-block */
|
||||
config: SubBlockConfig
|
||||
/** Whether component is in preview mode */
|
||||
isPreview?: boolean
|
||||
/** Value to display in preview mode */
|
||||
previewValue?: Message[] | null
|
||||
/** Whether the input is disabled */
|
||||
disabled?: boolean
|
||||
/** Ref to expose wand control handlers to parent */
|
||||
wandControlRef?: React.MutableRefObject<WandControlHandlers | null>
|
||||
}
|
||||
|
||||
/**
|
||||
* MessagesInput component for managing LLM message history
|
||||
*
|
||||
* @remarks
|
||||
* - Manages an array of messages with role and content
|
||||
* - Each message can be edited, removed, or reordered
|
||||
* - Stores data in LLM-compatible format: [{ role, content }]
|
||||
*/
|
||||
export function MessagesInput({
|
||||
blockId,
|
||||
@@ -90,6 +97,10 @@ export function MessagesInput({
|
||||
const [localMessages, setLocalMessages] = useState<Message[]>([{ role: 'user', content: '' }])
|
||||
const accessiblePrefixes = useAccessibleReferencePrefixes(blockId)
|
||||
const [openPopoverIndex, setOpenPopoverIndex] = useState<number | null>(null)
|
||||
|
||||
// Local media mode state - basic = FileUpload, advanced = URL/base64 textarea
|
||||
const [mediaMode, setMediaMode] = useState<'basic' | 'advanced'>('basic')
|
||||
|
||||
const subBlockInput = useSubBlockInput({
|
||||
blockId,
|
||||
subBlockId,
|
||||
@@ -98,43 +109,38 @@ export function MessagesInput({
|
||||
disabled,
|
||||
})
|
||||
|
||||
/**
|
||||
* Gets the current messages as JSON string for wand context
|
||||
*/
|
||||
const getMessagesJson = useCallback((): string => {
|
||||
if (localMessages.length === 0) return ''
|
||||
// Filter out empty messages for cleaner context
|
||||
const nonEmptyMessages = localMessages.filter((m) => m.content.trim() !== '')
|
||||
if (nonEmptyMessages.length === 0) return ''
|
||||
return JSON.stringify(nonEmptyMessages, null, 2)
|
||||
}, [localMessages])
|
||||
|
||||
/**
|
||||
* Streaming buffer for accumulating JSON content
|
||||
*/
|
||||
const streamBufferRef = useRef<string>('')
|
||||
|
||||
/**
|
||||
* Parses and validates messages from JSON content
|
||||
*/
|
||||
const parseMessages = useCallback((content: string): Message[] | null => {
|
||||
try {
|
||||
const parsed = JSON.parse(content)
|
||||
if (Array.isArray(parsed)) {
|
||||
const validMessages: Message[] = parsed
|
||||
.filter(
|
||||
(m): m is { role: string; content: string } =>
|
||||
(m): m is { role: string; content: string; media?: MediaContent } =>
|
||||
typeof m === 'object' &&
|
||||
m !== null &&
|
||||
typeof m.role === 'string' &&
|
||||
typeof m.content === 'string'
|
||||
)
|
||||
.map((m) => ({
|
||||
role: (['system', 'user', 'assistant'].includes(m.role)
|
||||
? m.role
|
||||
: 'user') as Message['role'],
|
||||
content: m.content,
|
||||
}))
|
||||
.map((m) => {
|
||||
const role = ['system', 'user', 'assistant', 'media'].includes(m.role) ? m.role : 'user'
|
||||
const message: Message = {
|
||||
role: role as Message['role'],
|
||||
content: m.content,
|
||||
}
|
||||
if (m.media) {
|
||||
message.media = m.media
|
||||
}
|
||||
return message
|
||||
})
|
||||
return validMessages.length > 0 ? validMessages : null
|
||||
}
|
||||
} catch {
|
||||
@@ -143,26 +149,19 @@ export function MessagesInput({
|
||||
return null
|
||||
}, [])
|
||||
|
||||
/**
|
||||
* Extracts messages from streaming JSON buffer
|
||||
* Uses simple pattern matching for efficiency
|
||||
*/
|
||||
const extractStreamingMessages = useCallback(
|
||||
(buffer: string): Message[] => {
|
||||
// Try complete JSON parse first
|
||||
const complete = parseMessages(buffer)
|
||||
if (complete) return complete
|
||||
|
||||
const result: Message[] = []
|
||||
|
||||
// Reset regex lastIndex for global pattern
|
||||
COMPLETE_MESSAGE_PATTERN.lastIndex = 0
|
||||
let match
|
||||
while ((match = COMPLETE_MESSAGE_PATTERN.exec(buffer)) !== null) {
|
||||
result.push({ role: match[1] as Message['role'], content: unescapeContent(match[2]) })
|
||||
}
|
||||
|
||||
// Check for incomplete message at end (content still streaming)
|
||||
const lastContentIdx = buffer.lastIndexOf('"content"')
|
||||
if (lastContentIdx !== -1) {
|
||||
const tail = buffer.slice(lastContentIdx)
|
||||
@@ -172,7 +171,6 @@ export function MessagesInput({
|
||||
const roleMatch = head.match(ROLE_BEFORE_CONTENT_PATTERN)
|
||||
if (roleMatch) {
|
||||
const content = unescapeContent(incomplete[1])
|
||||
// Only add if not duplicate of last complete message
|
||||
if (result.length === 0 || result[result.length - 1].content !== content) {
|
||||
result.push({ role: roleMatch[1] as Message['role'], content })
|
||||
}
|
||||
@@ -185,9 +183,6 @@ export function MessagesInput({
|
||||
[parseMessages]
|
||||
)
|
||||
|
||||
/**
|
||||
* Wand hook for AI-assisted content generation
|
||||
*/
|
||||
const wandHook = useWand({
|
||||
wandConfig: config.wandConfig,
|
||||
currentValue: getMessagesJson(),
|
||||
@@ -208,7 +203,6 @@ export function MessagesInput({
|
||||
setLocalMessages(validMessages)
|
||||
setMessages(validMessages)
|
||||
} else {
|
||||
// Fallback: treat as raw system prompt
|
||||
const trimmed = content.trim()
|
||||
if (trimmed) {
|
||||
const fallback: Message[] = [{ role: 'system', content: trimmed }]
|
||||
@@ -219,9 +213,6 @@ export function MessagesInput({
|
||||
},
|
||||
})
|
||||
|
||||
/**
|
||||
* Expose wand control handlers to parent via ref
|
||||
*/
|
||||
useImperativeHandle(
|
||||
wandControlRef,
|
||||
() => ({
|
||||
@@ -249,9 +240,6 @@ export function MessagesInput({
|
||||
}
|
||||
}, [isPreview, previewValue, messages])
|
||||
|
||||
/**
|
||||
* Gets the current messages array
|
||||
*/
|
||||
const currentMessages = useMemo<Message[]>(() => {
|
||||
if (isPreview && previewValue && Array.isArray(previewValue)) {
|
||||
return previewValue
|
||||
@@ -269,9 +257,6 @@ export function MessagesInput({
|
||||
startHeight: number
|
||||
} | null>(null)
|
||||
|
||||
/**
|
||||
* Updates a specific message's content
|
||||
*/
|
||||
const updateMessageContent = useCallback(
|
||||
(index: number, content: string) => {
|
||||
if (isPreview || disabled) return
|
||||
@@ -287,17 +272,26 @@ export function MessagesInput({
|
||||
[localMessages, setMessages, isPreview, disabled]
|
||||
)
|
||||
|
||||
/**
|
||||
* Updates a specific message's role
|
||||
*/
|
||||
const updateMessageRole = useCallback(
|
||||
(index: number, role: 'system' | 'user' | 'assistant') => {
|
||||
(index: number, role: 'system' | 'user' | 'assistant' | 'media') => {
|
||||
if (isPreview || disabled) return
|
||||
|
||||
const updatedMessages = [...localMessages]
|
||||
updatedMessages[index] = {
|
||||
...updatedMessages[index],
|
||||
role,
|
||||
if (role === 'media') {
|
||||
updatedMessages[index] = {
|
||||
...updatedMessages[index],
|
||||
role,
|
||||
content: updatedMessages[index].content || '',
|
||||
media: updatedMessages[index].media || {
|
||||
data: '',
|
||||
},
|
||||
}
|
||||
} else {
|
||||
const { media: _, ...rest } = updatedMessages[index]
|
||||
updatedMessages[index] = {
|
||||
...rest,
|
||||
role,
|
||||
}
|
||||
}
|
||||
setLocalMessages(updatedMessages)
|
||||
setMessages(updatedMessages)
|
||||
@@ -305,9 +299,6 @@ export function MessagesInput({
|
||||
[localMessages, setMessages, isPreview, disabled]
|
||||
)
|
||||
|
||||
/**
|
||||
* Adds a message after the specified index
|
||||
*/
|
||||
const addMessageAfter = useCallback(
|
||||
(index: number) => {
|
||||
if (isPreview || disabled) return
|
||||
@@ -320,9 +311,6 @@ export function MessagesInput({
|
||||
[localMessages, setMessages, isPreview, disabled]
|
||||
)
|
||||
|
||||
/**
|
||||
* Deletes a message at the specified index
|
||||
*/
|
||||
const deleteMessage = useCallback(
|
||||
(index: number) => {
|
||||
if (isPreview || disabled) return
|
||||
@@ -335,9 +323,6 @@ export function MessagesInput({
|
||||
[localMessages, setMessages, isPreview, disabled]
|
||||
)
|
||||
|
||||
/**
|
||||
* Moves a message up in the list
|
||||
*/
|
||||
const moveMessageUp = useCallback(
|
||||
(index: number) => {
|
||||
if (isPreview || disabled || index === 0) return
|
||||
@@ -352,9 +337,6 @@ export function MessagesInput({
|
||||
[localMessages, setMessages, isPreview, disabled]
|
||||
)
|
||||
|
||||
/**
|
||||
* Moves a message down in the list
|
||||
*/
|
||||
const moveMessageDown = useCallback(
|
||||
(index: number) => {
|
||||
if (isPreview || disabled || index === localMessages.length - 1) return
|
||||
@@ -369,18 +351,11 @@ export function MessagesInput({
|
||||
[localMessages, setMessages, isPreview, disabled]
|
||||
)
|
||||
|
||||
/**
|
||||
* Capitalizes the first letter of the role
|
||||
*/
|
||||
const formatRole = (role: string): string => {
|
||||
return role.charAt(0).toUpperCase() + role.slice(1)
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles header click to focus the textarea
|
||||
*/
|
||||
const handleHeaderClick = useCallback((index: number, e: React.MouseEvent) => {
|
||||
// Don't focus if clicking on interactive elements
|
||||
const target = e.target as HTMLElement
|
||||
if (target.closest('button') || target.closest('[data-radix-popper-content-wrapper]')) {
|
||||
return
|
||||
@@ -570,50 +545,52 @@ export function MessagesInput({
|
||||
className='flex cursor-pointer items-center justify-between px-[8px] pt-[6px]'
|
||||
onClick={(e) => handleHeaderClick(index, e)}
|
||||
>
|
||||
<Popover
|
||||
open={openPopoverIndex === index}
|
||||
onOpenChange={(open) => setOpenPopoverIndex(open ? index : null)}
|
||||
>
|
||||
<PopoverTrigger asChild>
|
||||
<button
|
||||
type='button'
|
||||
disabled={isPreview || disabled}
|
||||
className={cn(
|
||||
'group -ml-1.5 -my-1 flex items-center gap-1 rounded px-1.5 py-1 font-medium text-[13px] text-[var(--text-primary)] leading-none transition-colors hover:bg-[var(--surface-5)] hover:text-[var(--text-secondary)]',
|
||||
(isPreview || disabled) &&
|
||||
'cursor-default hover:bg-transparent hover:text-[var(--text-primary)]'
|
||||
)}
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
aria-label='Select message role'
|
||||
>
|
||||
{formatRole(message.role)}
|
||||
{!isPreview && !disabled && (
|
||||
<ChevronDown
|
||||
className={cn(
|
||||
'h-3 w-3 flex-shrink-0 transition-transform duration-100',
|
||||
openPopoverIndex === index && 'rotate-180'
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
</button>
|
||||
</PopoverTrigger>
|
||||
<PopoverContent minWidth={140} align='start'>
|
||||
<div className='flex flex-col gap-[2px]'>
|
||||
{(['system', 'user', 'assistant'] as const).map((role) => (
|
||||
<PopoverItem
|
||||
key={role}
|
||||
active={message.role === role}
|
||||
onClick={() => {
|
||||
updateMessageRole(index, role)
|
||||
setOpenPopoverIndex(null)
|
||||
}}
|
||||
>
|
||||
<span>{formatRole(role)}</span>
|
||||
</PopoverItem>
|
||||
))}
|
||||
</div>
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
<div className='flex items-center'>
|
||||
<Popover
|
||||
open={openPopoverIndex === index}
|
||||
onOpenChange={(open) => setOpenPopoverIndex(open ? index : null)}
|
||||
>
|
||||
<PopoverTrigger asChild>
|
||||
<button
|
||||
type='button'
|
||||
disabled={isPreview || disabled}
|
||||
className={cn(
|
||||
'group -ml-1.5 -my-1 flex items-center gap-1 rounded px-1.5 py-1 font-medium text-[13px] text-[var(--text-primary)] leading-none transition-colors hover:bg-[var(--surface-5)] hover:text-[var(--text-secondary)]',
|
||||
(isPreview || disabled) &&
|
||||
'cursor-default hover:bg-transparent hover:text-[var(--text-primary)]'
|
||||
)}
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
aria-label='Select message role'
|
||||
>
|
||||
{formatRole(message.role)}
|
||||
{!isPreview && !disabled && (
|
||||
<ChevronDown
|
||||
className={cn(
|
||||
'h-3 w-3 flex-shrink-0 transition-transform duration-100',
|
||||
openPopoverIndex === index && 'rotate-180'
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
</button>
|
||||
</PopoverTrigger>
|
||||
<PopoverContent minWidth={140} align='start'>
|
||||
<div className='flex flex-col gap-[2px]'>
|
||||
{(['system', 'user', 'assistant', 'media'] as const).map((role) => (
|
||||
<PopoverItem
|
||||
key={role}
|
||||
active={message.role === role}
|
||||
onClick={() => {
|
||||
updateMessageRole(index, role)
|
||||
setOpenPopoverIndex(null)
|
||||
}}
|
||||
>
|
||||
<span>{formatRole(role)}</span>
|
||||
</PopoverItem>
|
||||
))}
|
||||
</div>
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
</div>
|
||||
|
||||
{!isPreview && !disabled && (
|
||||
<div className='flex items-center'>
|
||||
@@ -657,6 +634,43 @@ export function MessagesInput({
|
||||
</Button>
|
||||
</>
|
||||
)}
|
||||
{/* Mode toggle for media messages */}
|
||||
{message.role === 'media' && (
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger asChild>
|
||||
<Button
|
||||
variant='ghost'
|
||||
onClick={(e: React.MouseEvent) => {
|
||||
e.stopPropagation()
|
||||
setMediaMode((m) => (m === 'basic' ? 'advanced' : 'basic'))
|
||||
}}
|
||||
disabled={disabled}
|
||||
className='-my-1 -mr-1 h-6 w-6 p-0'
|
||||
aria-label={
|
||||
mediaMode === 'advanced'
|
||||
? 'Switch to file upload'
|
||||
: 'Switch to URL/text input'
|
||||
}
|
||||
>
|
||||
<ArrowLeftRight
|
||||
className={cn(
|
||||
'h-3 w-3',
|
||||
mediaMode === 'advanced'
|
||||
? 'text-[var(--text-primary)]'
|
||||
: 'text-[var(--text-secondary)]'
|
||||
)}
|
||||
/>
|
||||
</Button>
|
||||
</Tooltip.Trigger>
|
||||
<Tooltip.Content side='top'>
|
||||
<p>
|
||||
{mediaMode === 'advanced'
|
||||
? 'Switch to file upload'
|
||||
: 'Switch to URL/text input'}
|
||||
</p>
|
||||
</Tooltip.Content>
|
||||
</Tooltip.Root>
|
||||
)}
|
||||
<Button
|
||||
variant='ghost'
|
||||
onClick={(e: React.MouseEvent) => {
|
||||
@@ -673,98 +687,138 @@ export function MessagesInput({
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Content Input with overlay for variable highlighting */}
|
||||
<div className='relative w-full overflow-hidden'>
|
||||
<textarea
|
||||
ref={(el) => {
|
||||
textareaRefs.current[fieldId] = el
|
||||
}}
|
||||
className='relative z-[2] m-0 box-border h-auto min-h-[80px] w-full resize-none overflow-y-auto overflow-x-hidden whitespace-pre-wrap break-words border-none bg-transparent px-[8px] py-[8px] font-medium font-sans text-sm text-transparent leading-[1.5] caret-[var(--text-primary)] outline-none [-ms-overflow-style:none] [scrollbar-width:none] placeholder:text-[var(--text-muted)] focus:outline-none focus-visible:outline-none disabled:cursor-not-allowed [&::-webkit-scrollbar]:hidden'
|
||||
placeholder='Enter message content...'
|
||||
value={message.content}
|
||||
onChange={fieldHandlers.onChange}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Tab' && !isPreview && !disabled) {
|
||||
e.preventDefault()
|
||||
const direction = e.shiftKey ? -1 : 1
|
||||
const nextIndex = index + direction
|
||||
|
||||
if (nextIndex >= 0 && nextIndex < currentMessages.length) {
|
||||
const nextFieldId = `message-${nextIndex}`
|
||||
const nextTextarea = textareaRefs.current[nextFieldId]
|
||||
if (nextTextarea) {
|
||||
nextTextarea.focus()
|
||||
nextTextarea.selectionStart = nextTextarea.value.length
|
||||
nextTextarea.selectionEnd = nextTextarea.value.length
|
||||
{/* Content Input - different for media vs text messages */}
|
||||
{message.role === 'media' ? (
|
||||
<div className='relative w-full px-[8px] py-[8px]'>
|
||||
{mediaMode === 'basic' ? (
|
||||
<FileUpload
|
||||
blockId={blockId}
|
||||
subBlockId={`${subBlockId}-media-${index}`}
|
||||
acceptedTypes='image/*,audio/*,video/*,application/pdf,.doc,.docx,.txt'
|
||||
multiple={false}
|
||||
isPreview={isPreview}
|
||||
disabled={disabled}
|
||||
/>
|
||||
) : (
|
||||
<textarea
|
||||
ref={(el) => {
|
||||
textareaRefs.current[fieldId] = el
|
||||
}}
|
||||
className='relative z-[2] m-0 box-border h-auto min-h-[60px] w-full resize-none overflow-y-auto overflow-x-hidden whitespace-pre-wrap break-words rounded-[4px] border border-[var(--border-1)] bg-transparent px-[8px] py-[6px] font-medium font-sans text-[var(--text-primary)] text-sm leading-[1.5] outline-none transition-colors [-ms-overflow-style:none] [scrollbar-width:none] placeholder:text-[var(--text-muted)] hover:border-[var(--border-2)] focus:border-[var(--border-2)] focus:outline-none focus-visible:outline-none disabled:cursor-not-allowed [&::-webkit-scrollbar]:hidden'
|
||||
placeholder='Enter URL or paste base64 data...'
|
||||
value={message.media?.data || ''}
|
||||
onChange={(e) => {
|
||||
const updatedMessages = [...localMessages]
|
||||
if (updatedMessages[index].role === 'media') {
|
||||
updatedMessages[index] = {
|
||||
...updatedMessages[index],
|
||||
content: e.target.value.substring(0, 50),
|
||||
media: {
|
||||
...updatedMessages[index].media,
|
||||
data: e.target.value,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
fieldHandlers.onKeyDown(e)
|
||||
}}
|
||||
onDrop={fieldHandlers.onDrop}
|
||||
onDragOver={fieldHandlers.onDragOver}
|
||||
onFocus={fieldHandlers.onFocus}
|
||||
onScroll={(e) => {
|
||||
const overlay = overlayRefs.current[fieldId]
|
||||
if (overlay) {
|
||||
overlay.scrollTop = e.currentTarget.scrollTop
|
||||
overlay.scrollLeft = e.currentTarget.scrollLeft
|
||||
}
|
||||
}}
|
||||
disabled={isPreview || disabled}
|
||||
/>
|
||||
<div
|
||||
ref={(el) => {
|
||||
overlayRefs.current[fieldId] = el
|
||||
}}
|
||||
className='pointer-events-none absolute top-0 left-0 z-[1] m-0 box-border w-full overflow-y-auto overflow-x-hidden whitespace-pre-wrap break-words border-none bg-transparent px-[8px] py-[8px] font-medium font-sans text-[var(--text-primary)] text-sm leading-[1.5] [-ms-overflow-style:none] [scrollbar-width:none] [&::-webkit-scrollbar]:hidden'
|
||||
>
|
||||
{formatDisplayText(message.content, {
|
||||
accessiblePrefixes,
|
||||
highlightAll: !accessiblePrefixes,
|
||||
})}
|
||||
{message.content.endsWith('\n') && '\u200B'}
|
||||
setLocalMessages(updatedMessages)
|
||||
setMessages(updatedMessages)
|
||||
}}
|
||||
disabled={isPreview || disabled}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Env var dropdown for this message */}
|
||||
<EnvVarDropdown
|
||||
visible={fieldState.showEnvVars && !isPreview && !disabled}
|
||||
onSelect={handleEnvSelect}
|
||||
searchTerm={fieldState.searchTerm}
|
||||
inputValue={message.content}
|
||||
cursorPosition={fieldState.cursorPosition}
|
||||
onClose={() => subBlockInput.fieldHelpers.hideFieldDropdowns(fieldId)}
|
||||
workspaceId={subBlockInput.workspaceId}
|
||||
maxHeight='192px'
|
||||
inputRef={textareaRefObject}
|
||||
/>
|
||||
|
||||
{/* Tag dropdown for this message */}
|
||||
<TagDropdown
|
||||
visible={fieldState.showTags && !isPreview && !disabled}
|
||||
onSelect={handleTagSelect}
|
||||
blockId={blockId}
|
||||
activeSourceBlockId={fieldState.activeSourceBlockId}
|
||||
inputValue={message.content}
|
||||
cursorPosition={fieldState.cursorPosition}
|
||||
onClose={() => subBlockInput.fieldHelpers.hideFieldDropdowns(fieldId)}
|
||||
inputRef={textareaRefObject}
|
||||
/>
|
||||
|
||||
{!isPreview && !disabled && (
|
||||
<div
|
||||
className='absolute right-1 bottom-1 z-[3] flex h-4 w-4 cursor-ns-resize items-center justify-center rounded-[4px] border border-[var(--border-1)] bg-[var(--surface-5)] dark:bg-[var(--surface-5)]'
|
||||
onMouseDown={(e) => handleResizeStart(fieldId, e)}
|
||||
onDragStart={(e) => {
|
||||
e.preventDefault()
|
||||
) : (
|
||||
<div className='relative w-full overflow-hidden'>
|
||||
<textarea
|
||||
ref={(el) => {
|
||||
textareaRefs.current[fieldId] = el
|
||||
}}
|
||||
className='relative z-[2] m-0 box-border h-auto min-h-[80px] w-full resize-none overflow-y-auto overflow-x-hidden whitespace-pre-wrap break-words border-none bg-transparent px-[8px] py-[8px] font-medium font-sans text-sm text-transparent leading-[1.5] caret-[var(--text-primary)] outline-none [-ms-overflow-style:none] [scrollbar-width:none] placeholder:text-[var(--text-muted)] focus:outline-none focus-visible:outline-none disabled:cursor-not-allowed [&::-webkit-scrollbar]:hidden'
|
||||
placeholder='Enter message content...'
|
||||
value={message.content}
|
||||
onChange={fieldHandlers.onChange}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Tab' && !isPreview && !disabled) {
|
||||
e.preventDefault()
|
||||
const direction = e.shiftKey ? -1 : 1
|
||||
const nextIndex = index + direction
|
||||
|
||||
if (nextIndex >= 0 && nextIndex < currentMessages.length) {
|
||||
const nextFieldId = `message-${nextIndex}`
|
||||
const nextTextarea = textareaRefs.current[nextFieldId]
|
||||
if (nextTextarea) {
|
||||
nextTextarea.focus()
|
||||
nextTextarea.selectionStart = nextTextarea.value.length
|
||||
nextTextarea.selectionEnd = nextTextarea.value.length
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
fieldHandlers.onKeyDown(e)
|
||||
}}
|
||||
onDrop={fieldHandlers.onDrop}
|
||||
onDragOver={fieldHandlers.onDragOver}
|
||||
onFocus={fieldHandlers.onFocus}
|
||||
onScroll={(e) => {
|
||||
const overlay = overlayRefs.current[fieldId]
|
||||
if (overlay) {
|
||||
overlay.scrollTop = e.currentTarget.scrollTop
|
||||
overlay.scrollLeft = e.currentTarget.scrollLeft
|
||||
}
|
||||
}}
|
||||
disabled={isPreview || disabled}
|
||||
/>
|
||||
<div
|
||||
ref={(el) => {
|
||||
overlayRefs.current[fieldId] = el
|
||||
}}
|
||||
className='pointer-events-none absolute top-0 left-0 z-[1] m-0 box-border w-full overflow-y-auto overflow-x-hidden whitespace-pre-wrap break-words border-none bg-transparent px-[8px] py-[8px] font-medium font-sans text-[var(--text-primary)] text-sm leading-[1.5] [-ms-overflow-style:none] [scrollbar-width:none] [&::-webkit-scrollbar]:hidden'
|
||||
>
|
||||
<ChevronsUpDown className='h-3 w-3 text-[var(--text-muted)]' />
|
||||
{formatDisplayText(message.content, {
|
||||
accessiblePrefixes,
|
||||
highlightAll: !accessiblePrefixes,
|
||||
})}
|
||||
{message.content.endsWith('\n') && '\u200B'}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Env var dropdown for this message */}
|
||||
<EnvVarDropdown
|
||||
visible={fieldState.showEnvVars && !isPreview && !disabled}
|
||||
onSelect={handleEnvSelect}
|
||||
searchTerm={fieldState.searchTerm}
|
||||
inputValue={message.content}
|
||||
cursorPosition={fieldState.cursorPosition}
|
||||
onClose={() => subBlockInput.fieldHelpers.hideFieldDropdowns(fieldId)}
|
||||
workspaceId={subBlockInput.workspaceId}
|
||||
maxHeight='192px'
|
||||
inputRef={textareaRefObject}
|
||||
/>
|
||||
|
||||
{/* Tag dropdown for this message */}
|
||||
<TagDropdown
|
||||
visible={fieldState.showTags && !isPreview && !disabled}
|
||||
onSelect={handleTagSelect}
|
||||
blockId={blockId}
|
||||
activeSourceBlockId={fieldState.activeSourceBlockId}
|
||||
inputValue={message.content}
|
||||
cursorPosition={fieldState.cursorPosition}
|
||||
onClose={() => subBlockInput.fieldHelpers.hideFieldDropdowns(fieldId)}
|
||||
inputRef={textareaRefObject}
|
||||
/>
|
||||
|
||||
{!isPreview && !disabled && (
|
||||
<div
|
||||
className='absolute right-1 bottom-1 z-[3] flex h-4 w-4 cursor-ns-resize items-center justify-center rounded-[4px] border border-[var(--border-1)] bg-[var(--surface-5)] dark:bg-[var(--surface-5)]'
|
||||
onMouseDown={(e) => handleResizeStart(fieldId, e)}
|
||||
onDragStart={(e) => {
|
||||
e.preventDefault()
|
||||
}}
|
||||
>
|
||||
<ChevronsUpDown className='h-3 w-3 text-[var(--text-muted)]' />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
)
|
||||
})()}
|
||||
|
||||
@@ -58,7 +58,12 @@ export class AgentBlockHandler implements BlockHandler {
|
||||
const providerId = getProviderFromModel(model)
|
||||
const formattedTools = await this.formatTools(ctx, filteredInputs.tools || [])
|
||||
const streamingConfig = this.getStreamingConfig(ctx, block)
|
||||
const messages = await this.buildMessages(ctx, filteredInputs)
|
||||
const rawMessages = await this.buildMessages(ctx, filteredInputs)
|
||||
|
||||
// Transform media messages to provider-specific format
|
||||
const messages = rawMessages
|
||||
? this.transformMediaMessages(rawMessages, providerId)
|
||||
: undefined
|
||||
|
||||
const providerRequest = this.buildProviderRequest({
|
||||
ctx,
|
||||
@@ -815,10 +820,248 @@ export class AgentBlockHandler implements BlockHandler {
|
||||
typeof msg === 'object' &&
|
||||
'role' in msg &&
|
||||
'content' in msg &&
|
||||
['system', 'user', 'assistant'].includes(msg.role)
|
||||
['system', 'user', 'assistant', 'media'].includes(msg.role)
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Transforms messages with 'media' role into provider-compatible format.
|
||||
* Media messages are merged with the preceding or following user message,
|
||||
* or converted to a user message with multimodal content.
|
||||
*/
|
||||
private transformMediaMessages(messages: Message[], providerId: string): Message[] {
|
||||
const result: Message[] = []
|
||||
|
||||
for (let i = 0; i < messages.length; i++) {
|
||||
const msg = messages[i]
|
||||
|
||||
if (msg.role !== 'media') {
|
||||
result.push(msg)
|
||||
continue
|
||||
}
|
||||
|
||||
// Media message - transform based on provider
|
||||
const mediaContent = this.createProviderMediaContent(msg, providerId)
|
||||
if (!mediaContent) {
|
||||
logger.warn('Could not create media content for message', { msg })
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if we should merge with the previous user message
|
||||
const lastMessage = result[result.length - 1]
|
||||
if (lastMessage && lastMessage.role === 'user') {
|
||||
// Merge media into the previous user message's content array
|
||||
const existingContent = this.ensureContentArray(lastMessage, providerId)
|
||||
existingContent.push(mediaContent)
|
||||
lastMessage.content = existingContent as any
|
||||
} else {
|
||||
// Create a new user message with the media content
|
||||
result.push({
|
||||
role: 'user',
|
||||
content: [mediaContent] as any,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Post-process: ensure all user messages have consistent content format
|
||||
return result.map((msg) => {
|
||||
if (msg.role === 'user' && typeof msg.content === 'string') {
|
||||
// Convert string content to provider-specific text format
|
||||
return {
|
||||
...msg,
|
||||
content: this.createTextContent(msg.content, providerId) as any,
|
||||
}
|
||||
}
|
||||
return msg
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures a user message has content as an array for multimodal support
|
||||
*/
|
||||
private ensureContentArray(msg: Message, providerId: string): any[] {
|
||||
if (Array.isArray(msg.content)) {
|
||||
return msg.content
|
||||
}
|
||||
if (typeof msg.content === 'string' && msg.content) {
|
||||
return [this.createTextContent(msg.content, providerId)]
|
||||
}
|
||||
return []
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates provider-specific text content block
|
||||
*/
|
||||
private createTextContent(text: string, providerId: string): any {
|
||||
switch (providerId) {
|
||||
case 'google':
|
||||
case 'vertex':
|
||||
return { text }
|
||||
case 'anthropic':
|
||||
return { type: 'text', text }
|
||||
default:
|
||||
// OpenAI format (used by most providers)
|
||||
return { type: 'text', text }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates provider-specific media content from a media message
|
||||
*/
|
||||
private createProviderMediaContent(msg: Message, providerId: string): any {
|
||||
const media = msg.media
|
||||
if (!media) return null
|
||||
|
||||
const { sourceType, data, mimeType } = media
|
||||
|
||||
switch (providerId) {
|
||||
case 'anthropic':
|
||||
return this.createAnthropicMediaContent(sourceType, data, mimeType)
|
||||
|
||||
case 'google':
|
||||
case 'vertex':
|
||||
return this.createGeminiMediaContent(sourceType, data, mimeType)
|
||||
|
||||
default:
|
||||
// OpenAI format (used by OpenAI, Azure, xAI, Mistral, etc.)
|
||||
return this.createOpenAIMediaContent(sourceType, data, mimeType)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates OpenAI-compatible media content
|
||||
*/
|
||||
private createOpenAIMediaContent(
|
||||
sourceType: string,
|
||||
data: string,
|
||||
mimeType?: string
|
||||
): any {
|
||||
const isImage = mimeType?.startsWith('image/')
|
||||
const isAudio = mimeType?.startsWith('audio/')
|
||||
|
||||
if (isImage) {
|
||||
if (sourceType === 'url') {
|
||||
return {
|
||||
type: 'image_url',
|
||||
image_url: { url: data, detail: 'auto' },
|
||||
}
|
||||
}
|
||||
// base64 or file (already converted to base64)
|
||||
return {
|
||||
type: 'image_url',
|
||||
image_url: { url: data, detail: 'auto' },
|
||||
}
|
||||
}
|
||||
|
||||
if (isAudio) {
|
||||
const base64Data = data.includes(',') ? data.split(',')[1] : data
|
||||
return {
|
||||
type: 'input_audio',
|
||||
input_audio: {
|
||||
data: base64Data,
|
||||
format: mimeType === 'audio/wav' ? 'wav' : 'mp3',
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// For documents/files, include as URL
|
||||
if (sourceType === 'url') {
|
||||
return {
|
||||
type: 'file',
|
||||
file: { url: data },
|
||||
}
|
||||
}
|
||||
|
||||
// Base64 file - some providers may not support this directly
|
||||
logger.warn('Base64 file content may not be supported by this provider')
|
||||
return {
|
||||
type: 'text',
|
||||
text: `[File: ${mimeType || 'unknown type'}]`,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates Anthropic-compatible media content
|
||||
*/
|
||||
private createAnthropicMediaContent(
|
||||
sourceType: string,
|
||||
data: string,
|
||||
mimeType?: string
|
||||
): any {
|
||||
const isImage = mimeType?.startsWith('image/')
|
||||
const isPdf = mimeType === 'application/pdf'
|
||||
|
||||
if (isImage) {
|
||||
if (sourceType === 'url') {
|
||||
return {
|
||||
type: 'image',
|
||||
source: { type: 'url', url: data },
|
||||
}
|
||||
}
|
||||
// base64
|
||||
const base64Data = data.includes(',') ? data.split(',')[1] : data
|
||||
return {
|
||||
type: 'image',
|
||||
source: {
|
||||
type: 'base64',
|
||||
media_type: mimeType || 'image/png',
|
||||
data: base64Data,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
if (isPdf) {
|
||||
if (sourceType === 'url') {
|
||||
return {
|
||||
type: 'document',
|
||||
source: { type: 'url', url: data },
|
||||
}
|
||||
}
|
||||
const base64Data = data.includes(',') ? data.split(',')[1] : data
|
||||
return {
|
||||
type: 'document',
|
||||
source: {
|
||||
type: 'base64',
|
||||
media_type: 'application/pdf',
|
||||
data: base64Data,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback for other types
|
||||
return {
|
||||
type: 'text',
|
||||
text: `[File: ${mimeType || 'unknown type'}]`,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates Google Gemini-compatible media content
|
||||
*/
|
||||
private createGeminiMediaContent(
|
||||
sourceType: string,
|
||||
data: string,
|
||||
mimeType?: string
|
||||
): any {
|
||||
if (sourceType === 'url') {
|
||||
return {
|
||||
fileData: {
|
||||
mimeType: mimeType || 'application/octet-stream',
|
||||
fileUri: data,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// base64
|
||||
const base64Data = data.includes(',') ? data.split(',')[1] : data
|
||||
return {
|
||||
inlineData: {
|
||||
mimeType: mimeType || 'application/octet-stream',
|
||||
data: base64Data,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
private processMemories(memories: any): Message[] {
|
||||
if (!memories) return []
|
||||
|
||||
|
||||
@@ -42,9 +42,25 @@ export interface ToolInput {
|
||||
customToolId?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Media content for multimodal messages
|
||||
*/
|
||||
export interface MediaContent {
|
||||
/** Mode: basic (file upload) or advanced (URL/base64 text input) */
|
||||
mode: 'basic' | 'advanced'
|
||||
/** The URL or base64 data */
|
||||
data: string
|
||||
/** MIME type (e.g., 'image/png', 'application/pdf', 'audio/mp3') */
|
||||
mimeType?: string
|
||||
/** Optional filename for file uploads */
|
||||
fileName?: string
|
||||
}
|
||||
|
||||
export interface Message {
|
||||
role: 'system' | 'user' | 'assistant'
|
||||
role: 'system' | 'user' | 'assistant' | 'media'
|
||||
content: string
|
||||
/** Media content for 'media' role messages */
|
||||
media?: MediaContent
|
||||
executionId?: string
|
||||
function_call?: any
|
||||
tool_calls?: any[]
|
||||
|
||||
@@ -111,9 +111,25 @@ export interface ProviderToolConfig {
|
||||
usageControl?: ToolUsageControl
|
||||
}
|
||||
|
||||
/**
|
||||
* Media content for multimodal messages
|
||||
*/
|
||||
export interface MediaContent {
|
||||
/** Mode: basic (file upload) or advanced (URL/base64 text input) */
|
||||
mode: 'basic' | 'advanced'
|
||||
/** The URL or base64 data */
|
||||
data: string
|
||||
/** MIME type (e.g., 'image/png', 'application/pdf', 'audio/mp3') */
|
||||
mimeType?: string
|
||||
/** Optional filename for file uploads */
|
||||
fileName?: string
|
||||
}
|
||||
|
||||
export interface Message {
|
||||
role: 'system' | 'user' | 'assistant' | 'function' | 'tool'
|
||||
role: 'system' | 'user' | 'assistant' | 'function' | 'tool' | 'media'
|
||||
content: string | null
|
||||
/** Media content for 'media' role messages */
|
||||
media?: MediaContent
|
||||
name?: string
|
||||
function_call?: {
|
||||
name: string
|
||||
|
||||
Reference in New Issue
Block a user