From 209d822ce9abb83f7b674cda2fbf6097e3cb3741 Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Thu, 10 Jul 2025 11:47:29 -0700 Subject: [PATCH 1/5] fix response format json extraction issues + add warning for invalid json --- .../components/sub-block/components/code.tsx | 28 ++++- .../components/sub-block/sub-block.tsx | 20 ++- .../executor/handlers/agent/agent-handler.ts | 116 +++++++++++++++++- 3 files changed, 154 insertions(+), 10 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/components/code.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/components/code.tsx index 8306e1e8f8..35f65b4a2b 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/components/code.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/components/code.tsx @@ -1,6 +1,6 @@ import type { ReactElement } from 'react' import { useEffect, useMemo, useRef, useState } from 'react' -import { Wand2 } from 'lucide-react' +import { Wand2, AlertTriangle } from 'lucide-react' import { highlight, languages } from 'prismjs' import 'prismjs/components/prism-javascript' import 'prismjs/themes/prism.css' @@ -29,6 +29,7 @@ interface CodeProps { isPreview?: boolean previewValue?: string | null disabled?: boolean + onValidationChange?: (isValid: boolean) => void } if (typeof document !== 'undefined') { @@ -60,6 +61,7 @@ export function Code({ isPreview = false, previewValue, disabled = false, + onValidationChange, }: CodeProps) { // Determine the AI prompt placeholder based on language const aiPromptPlaceholder = useMemo(() => { @@ -90,6 +92,24 @@ export function Code({ const showCollapseButton = (subBlockId === 'responseFormat' || subBlockId === 'code') && code.split('\n').length > 5 + const isValidJson = useMemo(() => { + if (subBlockId !== 'responseFormat' || !code.trim()) { + return true + } + try { + JSON.parse(code) + return true + } catch { + return false + } + }, [subBlockId, code]) + + useEffect(() => { + if (onValidationChange && subBlockId === 'responseFormat') { + onValidationChange(isValidJson) + } + }, [isValidJson, onValidationChange, subBlockId]) + const editorRef = useRef(null) // Function to toggle collapsed state @@ -343,9 +363,11 @@ export function Code({
e.preventDefault()} onDrop={handleDrop} > diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/sub-block.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/sub-block.tsx index e821aedd96..c3bfd9ed87 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/sub-block.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/sub-block.tsx @@ -1,4 +1,5 @@ -import { Info } from 'lucide-react' +import { useState } from 'react' +import { Info, AlertTriangle } from 'lucide-react' import { Label } from '@/components/ui/label' import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip' import { getBlock } from '@/blocks/index' @@ -48,10 +49,16 @@ export function SubBlock({ subBlockValues, disabled = false, }: SubBlockProps) { + const [isValidJson, setIsValidJson] = useState(true) + const handleMouseDown = (e: React.MouseEvent) => { e.stopPropagation() } + const handleValidationChange = (isValid: boolean) => { + setIsValidJson(isValid) + } + const isFieldRequired = () => { const blockType = useWorkflowStore.getState().blocks[blockId]?.type if (!blockType) return false @@ -169,6 +176,7 @@ export function SubBlock({ isPreview={isPreview} previewValue={previewValue} disabled={isDisabled} + onValidationChange={handleValidationChange} /> ) case 'switch': @@ -406,6 +414,16 @@ export function SubBlock({ )} + {config.id === 'responseFormat' && !isValidJson && ( + + + + + +

Invalid JSON

+ + + )} {config.description && ( diff --git a/apps/sim/executor/handlers/agent/agent-handler.ts b/apps/sim/executor/handlers/agent/agent-handler.ts index 80382f5fff..e76d2ba33e 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.ts @@ -703,16 +703,120 @@ export class AgentBlockHandler implements BlockHandler { } private processStructuredResponse(result: any, responseFormat: any): BlockOutput { - try { - const parsedContent = JSON.parse(result.content) + const content = result.content + + const extractedJson = this.extractJsonFromContent(content) + + if (extractedJson !== null) { + logger.info('Successfully parsed structured response content') return { - ...parsedContent, + ...extractedJson, ...this.createResponseMetadata(result), } - } catch (error) { - logger.error('Failed to parse response content:', { error }) - return this.processStandardResponse(result) } + + // All parsing attempts failed + logger.error('Failed to parse response content as JSON:', { + content: content.substring(0, 200) + (content.length > 200 ? '...' : ''), + responseFormat: responseFormat + }) + + // Return standard response but include a warning + const standardResponse = this.processStandardResponse(result) + return Object.assign(standardResponse, { + _responseFormatWarning: 'Response format was specified but content could not be parsed as JSON. Falling back to standard format.', + }) + } + + private extractJsonFromContent(content: string): any | null { + // Strategy 1: Direct JSON parsing + try { + return JSON.parse(content.trim()) + } catch { + // Continue to next strategy + } + + // Strategy 2: Extract from markdown code blocks (most common case) + // Matches ```json ... ``` or ``` ... ``` + const codeBlockRegex = /```(?:json)?\s*([\s\S]*?)\s*```/ + const codeBlockMatch = content.match(codeBlockRegex) + if (codeBlockMatch?.[1]) { + try { + return JSON.parse(codeBlockMatch[1].trim()) + } catch { + // Continue to next strategy + } + } + + // Strategy 3: Find first complete JSON object/array in the text + // Look for { ... } or [ ... ] with proper bracket matching + const jsonObjectMatch = this.findCompleteJsonInText(content) + if (jsonObjectMatch) { + try { + return JSON.parse(jsonObjectMatch) + } catch { + // Continue to next strategy + } + } + + return null + } + + private findCompleteJsonInText(text: string): string | null { + const trimmed = text.trim() + + // Find first { or [ + let startIndex = -1 + let startChar = '' + + for (let i = 0; i < trimmed.length; i++) { + if (trimmed[i] === '{' || trimmed[i] === '[') { + startIndex = i + startChar = trimmed[i] + break + } + } + + if (startIndex === -1) { + return null + } + + const endChar = startChar === '{' ? '}' : ']' + let depth = 0 + let inString = false + let escaped = false + + for (let i = startIndex; i < trimmed.length; i++) { + const char = trimmed[i] + + if (escaped) { + escaped = false + continue + } + + if (char === '\\') { + escaped = true + continue + } + + if (char === '"' && !escaped) { + inString = !inString + continue + } + + if (!inString) { + if (char === startChar) { + depth++ + } else if (char === endChar) { + depth-- + if (depth === 0) { + return trimmed.substring(startIndex, i + 1) + } + } + } + } + + return null } private processStandardResponse(result: any): BlockOutput { From 9ede0012024b1db7847048b298340f6cdd2fd3e9 Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Thu, 10 Jul 2025 11:47:36 -0700 Subject: [PATCH 2/5] lint --- .../workflow-block/components/sub-block/components/code.tsx | 4 ++-- .../workflow-block/components/sub-block/sub-block.tsx | 2 +- apps/sim/executor/handlers/agent/agent-handler.ts | 5 +++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/components/code.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/components/code.tsx index 35f65b4a2b..a5ca9bb3c1 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/components/code.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/components/code.tsx @@ -1,6 +1,6 @@ import type { ReactElement } from 'react' import { useEffect, useMemo, useRef, useState } from 'react' -import { Wand2, AlertTriangle } from 'lucide-react' +import { Wand2 } from 'lucide-react' import { highlight, languages } from 'prismjs' import 'prismjs/components/prism-javascript' import 'prismjs/themes/prism.css' @@ -365,7 +365,7 @@ export function Code({ className={cn( 'group relative min-h-[100px] rounded-md border bg-background font-mono text-sm transition-colors', isConnecting && 'ring-2 ring-blue-500 ring-offset-2', - !isValidJson && 'border-destructive border-2 bg-destructive/10' + !isValidJson && 'border-2 border-destructive bg-destructive/10' )} title={!isValidJson ? 'Invalid JSON' : undefined} onDragOver={(e) => e.preventDefault()} diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/sub-block.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/sub-block.tsx index c3bfd9ed87..54efcb75b8 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/sub-block.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/sub-block.tsx @@ -1,5 +1,5 @@ import { useState } from 'react' -import { Info, AlertTriangle } from 'lucide-react' +import { AlertTriangle, Info } from 'lucide-react' import { Label } from '@/components/ui/label' import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip' import { getBlock } from '@/blocks/index' diff --git a/apps/sim/executor/handlers/agent/agent-handler.ts b/apps/sim/executor/handlers/agent/agent-handler.ts index e76d2ba33e..20a4e50d9a 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.ts @@ -718,13 +718,14 @@ export class AgentBlockHandler implements BlockHandler { // All parsing attempts failed logger.error('Failed to parse response content as JSON:', { content: content.substring(0, 200) + (content.length > 200 ? '...' : ''), - responseFormat: responseFormat + responseFormat: responseFormat, }) // Return standard response but include a warning const standardResponse = this.processStandardResponse(result) return Object.assign(standardResponse, { - _responseFormatWarning: 'Response format was specified but content could not be parsed as JSON. Falling back to standard format.', + _responseFormatWarning: + 'Response format was specified but content could not be parsed as JSON. Falling back to standard format.', }) } From 614948948332c4d4332cc32bd57a3f6100b00851 Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Thu, 10 Jul 2025 11:56:25 -0700 Subject: [PATCH 3/5] Update apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/components/code.tsx Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../workflow-block/components/sub-block/components/code.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/components/code.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/components/code.tsx index a5ca9bb3c1..fb70307ea4 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/components/code.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/components/code.tsx @@ -106,7 +106,10 @@ export function Code({ useEffect(() => { if (onValidationChange && subBlockId === 'responseFormat') { - onValidationChange(isValidJson) + const timeoutId = setTimeout(() => { + onValidationChange(isValidJson) + }, 150) // Match debounce time from setStoreValue + return () => clearTimeout(timeoutId) } }, [isValidJson, onValidationChange, subBlockId]) From 1f6dcd8465136990a087f423ecd96c456815bb06 Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Thu, 10 Jul 2025 13:17:35 -0700 Subject: [PATCH 4/5] remove regex handling never hit --- .../executor/handlers/agent/agent-handler.ts | 124 +++--------------- 1 file changed, 17 insertions(+), 107 deletions(-) diff --git a/apps/sim/executor/handlers/agent/agent-handler.ts b/apps/sim/executor/handlers/agent/agent-handler.ts index 20a4e50d9a..00b8a4aae2 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.ts @@ -487,6 +487,7 @@ export class AgentBlockHandler implements BlockHandler { const contentType = response.headers.get('Content-Type') if (contentType?.includes('text/event-stream')) { // Handle streaming response + logger.info('Received streaming response') return this.handleStreamingResponse(response, block) } @@ -705,119 +706,28 @@ export class AgentBlockHandler implements BlockHandler { private processStructuredResponse(result: any, responseFormat: any): BlockOutput { const content = result.content - const extractedJson = this.extractJsonFromContent(content) - - if (extractedJson !== null) { + try { + const extractedJson = JSON.parse(content.trim()) logger.info('Successfully parsed structured response content') return { ...extractedJson, ...this.createResponseMetadata(result), } + } catch (error) { + logger.info('JSON parsing failed', { error: error instanceof Error ? error.message : 'Unknown error' }) + + // LLM did not adhere to structured response format + logger.error('LLM did not adhere to structured response format:', { + content: content.substring(0, 200) + (content.length > 200 ? '...' : ''), + responseFormat: responseFormat, + }) + + const standardResponse = this.processStandardResponse(result) + return Object.assign(standardResponse, { + _responseFormatWarning: + 'LLM did not adhere to the specified structured response format. Expected valid JSON but received malformed content. Falling back to standard format.', + }) } - - // All parsing attempts failed - logger.error('Failed to parse response content as JSON:', { - content: content.substring(0, 200) + (content.length > 200 ? '...' : ''), - responseFormat: responseFormat, - }) - - // Return standard response but include a warning - const standardResponse = this.processStandardResponse(result) - return Object.assign(standardResponse, { - _responseFormatWarning: - 'Response format was specified but content could not be parsed as JSON. Falling back to standard format.', - }) - } - - private extractJsonFromContent(content: string): any | null { - // Strategy 1: Direct JSON parsing - try { - return JSON.parse(content.trim()) - } catch { - // Continue to next strategy - } - - // Strategy 2: Extract from markdown code blocks (most common case) - // Matches ```json ... ``` or ``` ... ``` - const codeBlockRegex = /```(?:json)?\s*([\s\S]*?)\s*```/ - const codeBlockMatch = content.match(codeBlockRegex) - if (codeBlockMatch?.[1]) { - try { - return JSON.parse(codeBlockMatch[1].trim()) - } catch { - // Continue to next strategy - } - } - - // Strategy 3: Find first complete JSON object/array in the text - // Look for { ... } or [ ... ] with proper bracket matching - const jsonObjectMatch = this.findCompleteJsonInText(content) - if (jsonObjectMatch) { - try { - return JSON.parse(jsonObjectMatch) - } catch { - // Continue to next strategy - } - } - - return null - } - - private findCompleteJsonInText(text: string): string | null { - const trimmed = text.trim() - - // Find first { or [ - let startIndex = -1 - let startChar = '' - - for (let i = 0; i < trimmed.length; i++) { - if (trimmed[i] === '{' || trimmed[i] === '[') { - startIndex = i - startChar = trimmed[i] - break - } - } - - if (startIndex === -1) { - return null - } - - const endChar = startChar === '{' ? '}' : ']' - let depth = 0 - let inString = false - let escaped = false - - for (let i = startIndex; i < trimmed.length; i++) { - const char = trimmed[i] - - if (escaped) { - escaped = false - continue - } - - if (char === '\\') { - escaped = true - continue - } - - if (char === '"' && !escaped) { - inString = !inString - continue - } - - if (!inString) { - if (char === startChar) { - depth++ - } else if (char === endChar) { - depth-- - if (depth === 0) { - return trimmed.substring(startIndex, i + 1) - } - } - } - } - - return null } private processStandardResponse(result: any): BlockOutput { From a0a4b2100005f33e06e19d387b2256f0c1d930bd Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Thu, 10 Jul 2025 13:17:45 -0700 Subject: [PATCH 5/5] remove useless paths --- apps/sim/executor/handlers/agent/agent-handler.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/sim/executor/handlers/agent/agent-handler.ts b/apps/sim/executor/handlers/agent/agent-handler.ts index 00b8a4aae2..880b96419a 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.ts @@ -714,7 +714,9 @@ export class AgentBlockHandler implements BlockHandler { ...this.createResponseMetadata(result), } } catch (error) { - logger.info('JSON parsing failed', { error: error instanceof Error ? error.message : 'Unknown error' }) + logger.info('JSON parsing failed', { + error: error instanceof Error ? error.message : 'Unknown error', + }) // LLM did not adhere to structured response format logger.error('LLM did not adhere to structured response format:', {