Merge pull request #652 from simstudioai/fix/resp-format-json-extraction

fix(resp-format): add UI warning for invalid json in response format

apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/components/code.tsx

@@ -29,6 +29,7 @@ interface CodeProps {
   isPreview?: boolean
   previewValue?: string | null
   disabled?: boolean
+  onValidationChange?: (isValid: boolean) => void
 }
 
 if (typeof document !== 'undefined') {
@@ -60,6 +61,7 @@ export function Code({
   isPreview = false,
   previewValue,
   disabled = false,
+  onValidationChange,
 }: CodeProps) {
   // Determine the AI prompt placeholder based on language
   const aiPromptPlaceholder = useMemo(() => {
@@ -90,6 +92,27 @@ export function Code({
   const showCollapseButton =
     (subBlockId === 'responseFormat' || subBlockId === 'code') && code.split('\n').length > 5
 
+  const isValidJson = useMemo(() => {
+    if (subBlockId !== 'responseFormat' || !code.trim()) {
+      return true
+    }
+    try {
+      JSON.parse(code)
+      return true
+    } catch {
+      return false
+    }
+  }, [subBlockId, code])
+
+  useEffect(() => {
+    if (onValidationChange && subBlockId === 'responseFormat') {
+      const timeoutId = setTimeout(() => {
+        onValidationChange(isValidJson)
+      }, 150) // Match debounce time from setStoreValue
+      return () => clearTimeout(timeoutId)
+    }
+  }, [isValidJson, onValidationChange, subBlockId])
+
   const editorRef = useRef<HTMLDivElement>(null)
 
   // Function to toggle collapsed state
@@ -343,9 +366,11 @@ export function Code({
 
       <div
         className={cn(
-          'group relative min-h-[100px] rounded-md border bg-background font-mono text-sm',
-          isConnecting && 'ring-2 ring-blue-500 ring-offset-2'
+          'group relative min-h-[100px] rounded-md border bg-background font-mono text-sm transition-colors',
+          isConnecting && 'ring-2 ring-blue-500 ring-offset-2',
+          !isValidJson && 'border-2 border-destructive bg-destructive/10'
         )}
+        title={!isValidJson ? 'Invalid JSON' : undefined}
         onDragOver={(e) => e.preventDefault()}
         onDrop={handleDrop}
       >

apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/sub-block.tsx

@@ -1,4 +1,5 @@
-import { Info } from 'lucide-react'
+import { useState } from 'react'
+import { AlertTriangle, Info } from 'lucide-react'
 import { Label } from '@/components/ui/label'
 import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip'
 import { getBlock } from '@/blocks/index'
@@ -48,10 +49,16 @@ export function SubBlock({
   subBlockValues,
   disabled = false,
 }: SubBlockProps) {
+  const [isValidJson, setIsValidJson] = useState(true)
+
   const handleMouseDown = (e: React.MouseEvent) => {
     e.stopPropagation()
   }
 
+  const handleValidationChange = (isValid: boolean) => {
+    setIsValidJson(isValid)
+  }
+
   const isFieldRequired = () => {
     const blockType = useWorkflowStore.getState().blocks[blockId]?.type
     if (!blockType) return false
@@ -169,6 +176,7 @@ export function SubBlock({
             isPreview={isPreview}
             previewValue={previewValue}
             disabled={isDisabled}
+            onValidationChange={handleValidationChange}
           />
         )
       case 'switch':
@@ -406,6 +414,16 @@ export function SubBlock({
               </TooltipContent>
             </Tooltip>
           )}
+          {config.id === 'responseFormat' && !isValidJson && (
+            <Tooltip>
+              <TooltipTrigger asChild>
+                <AlertTriangle className='h-4 w-4 cursor-pointer text-destructive' />
+              </TooltipTrigger>
+              <TooltipContent side='top'>
+                <p>Invalid JSON</p>
+              </TooltipContent>
+            </Tooltip>
+          )}
           {config.description && (
             <Tooltip>
               <TooltipTrigger asChild>

apps/sim/executor/handlers/agent/agent-handler.ts

@@ -487,6 +487,7 @@ export class AgentBlockHandler implements BlockHandler {
     const contentType = response.headers.get('Content-Type')
     if (contentType?.includes('text/event-stream')) {
       // Handle streaming response
+      logger.info('Received streaming response')
       return this.handleStreamingResponse(response, block)
     }
 
@@ -703,15 +704,31 @@ export class AgentBlockHandler implements BlockHandler {
   }
 
   private processStructuredResponse(result: any, responseFormat: any): BlockOutput {
+    const content = result.content
+
     try {
-      const parsedContent = JSON.parse(result.content)
+      const extractedJson = JSON.parse(content.trim())
+      logger.info('Successfully parsed structured response content')
       return {
-        ...parsedContent,
+        ...extractedJson,
         ...this.createResponseMetadata(result),
       }
     } catch (error) {
-      logger.error('Failed to parse response content:', { error })
-      return this.processStandardResponse(result)
+      logger.info('JSON parsing failed', {
+        error: error instanceof Error ? error.message : 'Unknown error',
+      })
+
+      // LLM did not adhere to structured response format
+      logger.error('LLM did not adhere to structured response format:', {
+        content: content.substring(0, 200) + (content.length > 200 ? '...' : ''),
+        responseFormat: responseFormat,
+      })
+
+      const standardResponse = this.processStandardResponse(result)
+      return Object.assign(standardResponse, {
+        _responseFormatWarning:
+          'LLM did not adhere to the specified structured response format. Expected valid JSON but received malformed content. Falling back to standard format.',
+      })
     }
   }