github/sim
1
0
Fork 0
mirror of https://github.com/simstudioai/sim.git synced 2026-01-30 01:07:59 -05:00
Code Issues Packages Projects Releases Wiki Activity

Fix media

Browse Source
This commit is contained in:
Siddharth Ganesan
2026-01-29 17:20:38 -08:00
parent 599ffb77e6
commit aa893d56d8
4 changed files with 48 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/messages-input/messages-input.tsx
View File

@@ -7,10 +7,10 @@ import {
useRef,
useState,
} from 'react'
import { createLogger } from '@sim/logger'
import { isEqual } from 'lodash'
import { ArrowLeftRight, ChevronDown, ChevronsUpDown, ChevronUp, Plus } from 'lucide-react'
import { useParams } from 'next/navigation'
import { createLogger } from '@sim/logger'
import {
Button,
Popover,
@@ -28,12 +28,12 @@ import { ShortInput } from '@/app/workspace/[workspaceId]/w/[workflowId]/compone
import { TagDropdown } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/tag-dropdown/tag-dropdown'
import { useSubBlockInput } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-sub-block-input'
import { useSubBlockValue } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-sub-block-value'
import { useWorkflowRegistry } from '@/stores/workflows/registry/store'
import { useSubBlockStore } from '@/stores/workflows/subblock/store'
import type { WandControlHandlers } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/sub-block'
import { useAccessibleReferencePrefixes } from '@/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-accessible-reference-prefixes'
import { useWand } from '@/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-wand'
import type { SubBlockConfig } from '@/blocks/types'
import { useWorkflowRegistry } from '@/stores/workflows/registry/store'
import { useSubBlockStore } from '@/stores/workflows/subblock/store'
const logger = createLogger('MessagesInput')
@@ -155,7 +155,13 @@ export function MessagesInput({
}
const filesList = workspaceFiles
.filter((f) => f.type.startsWith('image/') || f.type.startsWith('audio/') || f.type.startsWith('video/') || f.type === 'application/pdf')
.filter(
(f) =>
f.type.startsWith('image/') ||
f.type.startsWith('audio/') ||
f.type.startsWith('video/') ||
f.type === 'application/pdf'
)
.map((f) => ` - id: "${f.id}", name: "${f.name}", type: "${f.type}"`)
.join('\n')
@@ -166,7 +172,6 @@ export function MessagesInput({
return `AVAILABLE WORKSPACE FILES (optional - you don't have to select one):\n${filesList}\n\nTo use a file, include "fileId": "<id>" in the media object. If not selecting a file, omit the fileId field.`
}, [workspaceFiles])
// Effect to sync FileUpload values to message media objects
useEffect(() => {
if (!activeWorkflowId || isPreview) return
@@ -321,8 +326,9 @@ export function MessagesInput({
validMessages.forEach((msg, index) => {
if (msg.role === 'media') {
// Check if this is an existing file with valid data (preserve it)
const hasExistingFile = msg.media?.sourceType === 'file' &&
msg.media?.data?.startsWith('/api/') &&
const hasExistingFile =
msg.media?.sourceType === 'file' &&
msg.media?.data?.startsWith('/api/') &&
msg.media?.fileName
if (hasExistingFile) {
@@ -868,7 +874,7 @@ export function MessagesInput({
}}
value={
// Only show value for variable references, not file uploads
message.media?.sourceType === 'file' ? '' : (message.media?.data || '')
message.media?.sourceType === 'file' ? '' : message.media?.data || ''
}
onChange={(newValue: string) => {
const updatedMessages = [...localMessages]

5
apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-wand.ts
View File

@@ -158,7 +158,10 @@ export function useWand({
systemPrompt = systemPrompt.replace('{context}', contextInfo)
}
if (systemPrompt.includes('{sources}')) {
systemPrompt = systemPrompt.replace('{sources}', sources || 'No upstream sources available')
systemPrompt = systemPrompt.replace(
'{sources}',
sources || 'No upstream sources available'
)
}
const userMessage = prompt

60
apps/sim/executor/handlers/agent/agent-handler.ts
View File

@@ -61,9 +61,7 @@ export class AgentBlockHandler implements BlockHandler {
const rawMessages = await this.buildMessages(ctx, filteredInputs)
// Transform media messages to provider-specific format
const messages = rawMessages
? this.transformMediaMessages(rawMessages, providerId)
: undefined
const messages = rawMessages ? this.transformMediaMessages(rawMessages, providerId) : undefined
const providerRequest = this.buildProviderRequest({
ctx,
@@ -827,7 +825,10 @@ export class AgentBlockHandler implements BlockHandler {
}
messageArray = parsed
} catch (error) {
logger.warn('Failed to parse messages JSON string', { error, messages: trimmed.substring(0, 100) })
logger.warn('Failed to parse messages JSON string', {
error,
messages: trimmed.substring(0, 100),
})
return []
}
} else if (Array.isArray(messages)) {
@@ -948,9 +949,11 @@ export class AgentBlockHandler implements BlockHandler {
if (sourceType === 'url' || sourceType === 'file') {
const trimmedData = data.trim()
// Must start with http://, https://, or / (relative path for workspace files)
if (!trimmedData.startsWith('http://') &&
!trimmedData.startsWith('https://') &&
!trimmedData.startsWith('/')) {
if (
!trimmedData.startsWith('http://') &&
!trimmedData.startsWith('https://') &&
!trimmedData.startsWith('/')
) {
logger.warn('Invalid URL format for media content', { data: trimmedData.substring(0, 50) })
// Try to salvage by treating as text
return { type: 'text', text: `[Invalid media URL: ${trimmedData.substring(0, 30)}...]` }
@@ -961,8 +964,13 @@ export class AgentBlockHandler implements BlockHandler {
if (sourceType === 'base64') {
const trimmedData = data.trim()
// Should be a data URL or raw base64
if (!trimmedData.startsWith('data:') && !/^[A-Za-z0-9+/]+=*$/.test(trimmedData.replace(/\s/g, ''))) {
logger.warn('Invalid base64 format for media content', { data: trimmedData.substring(0, 50) })
if (
!trimmedData.startsWith('data:') &&
!/^[A-Za-z0-9+/]+=*$/.test(trimmedData.replace(/\s/g, ''))
) {
logger.warn('Invalid base64 format for media content', {
data: trimmedData.substring(0, 50),
})
return { type: 'text', text: `[Invalid base64 data]` }
}
}
@@ -990,11 +998,7 @@ export class AgentBlockHandler implements BlockHandler {
/**
* Creates OpenAI-compatible media content
*/
private createOpenAIMediaContent(
sourceType: string,
data: string,
mimeType?: string
): any {
private createOpenAIMediaContent(sourceType: string, data: string, mimeType?: string): any {
const isImage = mimeType?.startsWith('image/')
const isAudio = mimeType?.startsWith('audio/')
// Treat 'file' as 'url' since workspace files are served via URL
@@ -1037,11 +1041,7 @@ export class AgentBlockHandler implements BlockHandler {
/**
* Creates Anthropic-compatible media content
*/
private createAnthropicMediaContent(
sourceType: string,
data: string,
mimeType?: string
): any {
private createAnthropicMediaContent(sourceType: string, data: string, mimeType?: string): any {
const isImage = mimeType?.startsWith('image/')
const isPdf = mimeType === 'application/pdf'
// Treat 'file' as 'url' since workspace files are served via URL
@@ -1094,11 +1094,7 @@ export class AgentBlockHandler implements BlockHandler {
/**
* Creates Google Gemini-compatible media content
*/
private createGeminiMediaContent(
sourceType: string,
data: string,
mimeType?: string
): any {
private createGeminiMediaContent(sourceType: string, data: string, mimeType?: string): any {
// Treat 'file' as 'url' since workspace files are served via URL
const isUrl = sourceType === 'url' || sourceType === 'file'
@@ -1126,11 +1122,7 @@ export class AgentBlockHandler implements BlockHandler {
* Note: Mistral uses a simplified format where image_url is a direct string,
* NOT a nested object like OpenAI
*/
private createMistralMediaContent(
sourceType: string,
data: string,
mimeType?: string
): any {
private createMistralMediaContent(sourceType: string, data: string, mimeType?: string): any {
const isImage = mimeType?.startsWith('image/')
// Treat 'file' as 'url' since workspace files are served via URL
const isUrl = sourceType === 'url' || sourceType === 'file'
@@ -1144,7 +1136,9 @@ export class AgentBlockHandler implements BlockHandler {
}
}
// Base64 - Mistral accepts data URLs directly
const base64Data = data.includes(',') ? data : `data:${mimeType || 'image/png'};base64,${data}`
const base64Data = data.includes(',')
? data
: `data:${mimeType || 'image/png'};base64,${data}`
return {
type: 'image_url',
image_url: base64Data,
@@ -1163,11 +1157,7 @@ export class AgentBlockHandler implements BlockHandler {
* Bedrock uses a different structure: { image: { format, source: { bytes } } }
* Note: The actual bytes conversion happens in the provider layer
*/
private createBedrockMediaContent(
sourceType: string,
data: string,
mimeType?: string
): any {
private createBedrockMediaContent(sourceType: string, data: string, mimeType?: string): any {
const isImage = mimeType?.startsWith('image/')
// Treat 'file' as 'url' since workspace files are served via URL
const isUrl = sourceType === 'url' || sourceType === 'file'

6
apps/sim/providers/bedrock/utils.ts
View File

@@ -1,4 +1,8 @@
import type { ContentBlock, ConverseStreamOutput, ImageFormat } from '@aws-sdk/client-bedrock-runtime'
import type {
ContentBlock,
ConverseStreamOutput,
ImageFormat,
} from '@aws-sdk/client-bedrock-runtime'
import { createLogger } from '@sim/logger'
import { trackForcedToolUsage } from '@/providers/utils'