From a5b809c8e93812a0eb124ccb969a649096f6f2e0 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 20 May 2025 23:48:18 -0700 Subject: [PATCH] fix(docs): fixed naming semantics for memory tools, updated doc generator, & included docs for memory block (#389) * added docs for memories * updated doc generator script to better find outputs * include google drive new tools in docs --- apps/docs/content/docs/tools/github.mdx | 4 + apps/docs/content/docs/tools/google_docs.mdx | 8 +- apps/docs/content/docs/tools/google_drive.mdx | 2 +- apps/docs/content/docs/tools/jina.mdx | 4 +- apps/docs/content/docs/tools/mem0.mdx | 7 +- apps/docs/content/docs/tools/memory.mdx | 130 ++++++++++++ apps/docs/content/docs/tools/meta.json | 1 + apps/docs/content/docs/tools/pinecone.mdx | 1 + apps/docs/content/docs/tools/reddit.mdx | 2 + apps/docs/content/docs/tools/s3.mdx | 1 + apps/docs/content/docs/tools/serper.mdx | 6 +- apps/docs/content/docs/tools/typeform.mdx | 8 +- apps/docs/content/docs/tools/x.mdx | 1 + apps/docs/content/docs/tools/youtube.mdx | 1 + .../tools/memory/{add_memory.ts => add.ts} | 69 +++---- .../memory/{delete_memory.ts => delete.ts} | 31 ++- .../tools/memory/{get_memory.ts => get.ts} | 33 ++- .../{get_all_memories.ts => get_all.ts} | 39 ++-- apps/sim/tools/memory/index.ts | 10 +- apps/sim/tools/openai/image.ts | 190 ++++++++++++++++++ scripts/generate-block-docs.ts | 31 ++- 21 files changed, 470 insertions(+), 109 deletions(-) create mode 100644 apps/docs/content/docs/tools/memory.mdx rename apps/sim/tools/memory/{add_memory.ts => add.ts} (82%) rename apps/sim/tools/memory/{delete_memory.ts => delete.ts} (87%) rename apps/sim/tools/memory/{get_memory.ts => get.ts} (88%) rename apps/sim/tools/memory/{get_all_memories.ts => get_all.ts} (84%) create mode 100644 apps/sim/tools/openai/image.ts diff --git a/apps/docs/content/docs/tools/github.mdx b/apps/docs/content/docs/tools/github.mdx index ac663ba5f..80d98759d 100644 --- a/apps/docs/content/docs/tools/github.mdx +++ b/apps/docs/content/docs/tools/github.mdx @@ -70,6 +70,7 @@ Fetch PR details including diff and files changed | `blob_url` | string | | `raw_url` | string | | `status` | string | +| `content` | string | ### `github_comment` @@ -103,6 +104,7 @@ Create comments on GitHub PRs | `line` | string | | `side` | string | | `commit_id` | string | +| `content` | string | ### `github_repo_info` @@ -126,6 +128,7 @@ Retrieve comprehensive GitHub repository metadata including stars, forks, issues | `forks` | string | | `openIssues` | string | | `language` | string | +| `content` | string | ### `github_latest_commit` @@ -150,6 +153,7 @@ Retrieve the latest commit from a GitHub repository | `author` | string | | `login` | string | | `avatar_url` | string | +| `content` | string | ## Block Configuration diff --git a/apps/docs/content/docs/tools/google_docs.mdx b/apps/docs/content/docs/tools/google_docs.mdx index 21928b0d7..bca2ee3f6 100644 --- a/apps/docs/content/docs/tools/google_docs.mdx +++ b/apps/docs/content/docs/tools/google_docs.mdx @@ -97,9 +97,10 @@ Read content from a Google Docs document #### Output -| Parameter | Type | -| --------- | ------ | -| `content` | string | +| Parameter | Type | +| ---------- | ------ | +| `content` | string | +| `metadata` | string | ### `google_docs_write` @@ -118,6 +119,7 @@ Write or update content in a Google Docs document | Parameter | Type | | ---------------- | ------ | | `updatedContent` | string | +| `metadata` | string | ### `google_docs_create` diff --git a/apps/docs/content/docs/tools/google_drive.mdx b/apps/docs/content/docs/tools/google_drive.mdx index 7350a9d30..e8aaa7f0f 100644 --- a/apps/docs/content/docs/tools/google_drive.mdx +++ b/apps/docs/content/docs/tools/google_drive.mdx @@ -1,6 +1,6 @@ --- title: Google Drive -description: Create and list files +description: Create, upload, and list files --- import { BlockInfoCard } from '@/components/ui/block-info-card' diff --git a/apps/docs/content/docs/tools/jina.mdx b/apps/docs/content/docs/tools/jina.mdx index 51ff42676..3de301b69 100644 --- a/apps/docs/content/docs/tools/jina.mdx +++ b/apps/docs/content/docs/tools/jina.mdx @@ -78,7 +78,9 @@ Extract and process web content into clean, LLM-friendly text using Jina AI Read #### Output -This tool does not produce any outputs. +| Parameter | Type | +| --------- | ------ | +| `content` | string | ## Block Configuration diff --git a/apps/docs/content/docs/tools/mem0.mdx b/apps/docs/content/docs/tools/mem0.mdx index 2d350743c..06938ecdf 100644 --- a/apps/docs/content/docs/tools/mem0.mdx +++ b/apps/docs/content/docs/tools/mem0.mdx @@ -102,9 +102,10 @@ Retrieve memories from Mem0 by ID or filter criteria #### Output -| Parameter | Type | -| --------- | ------ | -| `data` | string | +| Parameter | Type | +| ---------- | ------ | +| `memories` | string | +| `ids` | string | ## Block Configuration diff --git a/apps/docs/content/docs/tools/memory.mdx b/apps/docs/content/docs/tools/memory.mdx new file mode 100644 index 000000000..07c60a42f --- /dev/null +++ b/apps/docs/content/docs/tools/memory.mdx @@ -0,0 +1,130 @@ +--- +title: Memory +description: Add memory store +--- + +import { BlockInfoCard } from '@/components/ui/block-info-card' + + + + + + + + + + + + `} +/> + +## Usage Instructions + +Create persistent storage for data that needs to be accessed across multiple workflow steps. Store and retrieve information throughout your workflow execution to maintain context and state. + +## Tools + +### `memory_add` + +Add a new memory to the database or append to existing memory with the same ID. When appending to existing memory, the memory types must match. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ------ | -------- | -------------------------------------------------------------------------------------------------------- | +| `id` | string | Yes | Identifier for the memory. If a memory with this ID already exists, the new data will be appended to it. | +| `type` | string | Yes | Type of memory \(agent or raw\) | +| `role` | string | No | Role for agent memory \(user, assistant, or system\) | +| `content` | string | No | Content for agent memory | +| `rawData` | json | No | Raw data to store \(JSON format\) | + +#### Output + +| Parameter | Type | +| ---------- | ------ | +| `memories` | string | + +### `memory_get` + +Retrieve a specific memory by its ID + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ------ | -------- | ------------------------------------- | +| `id` | string | Yes | Identifier for the memory to retrieve | + +#### Output + +| Parameter | Type | +| ---------- | ------ | +| `memories` | string | +| `message` | string | + +### `memory_get_all` + +Retrieve all memories from the database + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | + +#### Output + +| Parameter | Type | +| ---------- | ------ | +| `message` | string | +| `memories` | string | + +### `memory_delete` + +Delete a specific memory by its ID + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ------ | -------- | ----------------------------------- | +| `id` | string | Yes | Identifier for the memory to delete | + +#### Output + +| Parameter | Type | +| --------- | ------ | +| `message` | string | + +## Block Configuration + +### Input + +| Parameter | Type | Required | Description | +| ----------- | ------ | -------- | ----------- | +| `operation` | string | Yes | Operation | + +### Outputs + +| Output | Type | Description | +| ------------ | ------ | ------------------------ | +| `response` | object | Output from response | +| ↳ `memories` | any | memories of the response | +| ↳ `id` | string | id of the response | + +## Notes + +- Category: `blocks` +- Type: `memory` diff --git a/apps/docs/content/docs/tools/meta.json b/apps/docs/content/docs/tools/meta.json index f331ca1a8..0a52f6e25 100644 --- a/apps/docs/content/docs/tools/meta.json +++ b/apps/docs/content/docs/tools/meta.json @@ -23,6 +23,7 @@ "jira", "linkup", "mem0", + "memory", "notion", "openai", "perplexity", diff --git a/apps/docs/content/docs/tools/pinecone.mdx b/apps/docs/content/docs/tools/pinecone.mdx index 5176ecd33..527fa91ed 100644 --- a/apps/docs/content/docs/tools/pinecone.mdx +++ b/apps/docs/content/docs/tools/pinecone.mdx @@ -160,6 +160,7 @@ Fetch vectors by ID from a Pinecone index | `values` | string | | `metadata` | string | | `score` | string | +| `id` | string | ## Block Configuration diff --git a/apps/docs/content/docs/tools/reddit.mdx b/apps/docs/content/docs/tools/reddit.mdx index cf8e4d81a..42e4408fb 100644 --- a/apps/docs/content/docs/tools/reddit.mdx +++ b/apps/docs/content/docs/tools/reddit.mdx @@ -63,6 +63,7 @@ Fetch the most popular (hot) posts from a specified subreddit. | Parameter | Type | | ----------- | ------ | | `subreddit` | string | +| `posts` | string | ### `reddit_get_posts` @@ -82,6 +83,7 @@ Fetch posts from a subreddit with different sorting options | Parameter | Type | | ----------- | ------ | | `subreddit` | string | +| `posts` | string | ### `reddit_get_comments` diff --git a/apps/docs/content/docs/tools/s3.mdx b/apps/docs/content/docs/tools/s3.mdx index 110526b7b..0268d0d7d 100644 --- a/apps/docs/content/docs/tools/s3.mdx +++ b/apps/docs/content/docs/tools/s3.mdx @@ -69,6 +69,7 @@ Retrieve an object from an AWS S3 bucket | `size` | string | | `name` | string | | `lastModified` | string | +| `url` | string | ## Block Configuration diff --git a/apps/docs/content/docs/tools/serper.mdx b/apps/docs/content/docs/tools/serper.mdx index 48836b3cd..2aa15e18b 100644 --- a/apps/docs/content/docs/tools/serper.mdx +++ b/apps/docs/content/docs/tools/serper.mdx @@ -98,9 +98,9 @@ A powerful web search tool that provides access to Google search results through #### Output -| Parameter | Type | -| --------- | ----- | -| `results` | array | +| Parameter | Type | +| --------------- | ------ | +| `searchResults` | string | ## Block Configuration diff --git a/apps/docs/content/docs/tools/typeform.mdx b/apps/docs/content/docs/tools/typeform.mdx index e64a52021..fa51b0025 100644 --- a/apps/docs/content/docs/tools/typeform.mdx +++ b/apps/docs/content/docs/tools/typeform.mdx @@ -86,9 +86,11 @@ Download files uploaded in Typeform responses #### Output -| Parameter | Type | -| --------- | ------ | -| `fileUrl` | string | +| Parameter | Type | +| ------------- | ------ | +| `fileUrl` | string | +| `contentType` | string | +| `filename` | string | ### `typeform_insights` diff --git a/apps/docs/content/docs/tools/x.mdx b/apps/docs/content/docs/tools/x.mdx index 8e302973f..f0608dcde 100644 --- a/apps/docs/content/docs/tools/x.mdx +++ b/apps/docs/content/docs/tools/x.mdx @@ -83,6 +83,7 @@ Read tweet details, including replies and conversation context | Parameter | Type | | --------- | ------ | | `tweet` | string | +| `context` | string | ### `x_search` diff --git a/apps/docs/content/docs/tools/youtube.mdx b/apps/docs/content/docs/tools/youtube.mdx index d5341b25f..314a51b07 100644 --- a/apps/docs/content/docs/tools/youtube.mdx +++ b/apps/docs/content/docs/tools/youtube.mdx @@ -61,6 +61,7 @@ Search for videos on YouTube using the YouTube Data API. | --------------- | ------ | | `totalResults` | string | | `nextPageToken` | string | +| `items` | string | ## Block Configuration diff --git a/apps/sim/tools/memory/add_memory.ts b/apps/sim/tools/memory/add.ts similarity index 82% rename from apps/sim/tools/memory/add_memory.ts rename to apps/sim/tools/memory/add.ts index 3b6d159e2..bd8ab6eab 100644 --- a/apps/sim/tools/memory/add_memory.ts +++ b/apps/sim/tools/memory/add.ts @@ -1,17 +1,18 @@ import { ToolConfig } from '../types' import { MemoryResponse } from './types' -// Add Memory Tool export const memoryAddTool: ToolConfig = { id: 'memory_add', name: 'Add Memory', - description: 'Add a new memory to the database or append to existing memory with the same ID. When appending to existing memory, the memory types must match.', + description: + 'Add a new memory to the database or append to existing memory with the same ID. When appending to existing memory, the memory types must match.', version: '1.0.0', params: { id: { type: 'string', required: true, - description: 'Identifier for the memory. If a memory with this ID already exists, the new data will be appended to it.', + description: + 'Identifier for the memory. If a memory with this ID already exists, the new data will be appended to it.', }, type: { type: 'string', @@ -32,7 +33,7 @@ export const memoryAddTool: ToolConfig = { type: 'json', required: false, description: 'Raw data to store (JSON format)', - } + }, }, request: { url: '/api/memory', @@ -43,7 +44,7 @@ export const memoryAddTool: ToolConfig = { body: (params) => { // Get workflowId from context (set by workflow execution) const workflowId = params._context?.workflowId - + // Prepare error response instead of throwing error if (!workflowId) { return { @@ -52,17 +53,17 @@ export const memoryAddTool: ToolConfig = { data: { success: false, error: { - message: 'workflowId is required and must be provided in execution context' - } - } - } + message: 'workflowId is required and must be provided in execution context', + }, + }, + }, } } - + const body: Record = { key: params.id, type: params.type, - workflowId + workflowId, } // Set data based on type @@ -74,10 +75,10 @@ export const memoryAddTool: ToolConfig = { data: { success: false, error: { - message: 'Role and content are required for agent memory' - } - } - } + message: 'Role and content are required for agent memory', + }, + }, + }, } } body.data = { @@ -92,13 +93,13 @@ export const memoryAddTool: ToolConfig = { data: { success: false, error: { - message: 'Raw data is required for raw memory' - } - } - } + message: 'Raw data is required for raw memory', + }, + }, + }, } } - + let parsedRawData if (typeof params.rawData === 'string') { try { @@ -110,16 +111,16 @@ export const memoryAddTool: ToolConfig = { data: { success: false, error: { - message: 'Invalid JSON for raw data' - } - } - } + message: 'Invalid JSON for raw data', + }, + }, + }, } } } else { parsedRawData = params.rawData } - + body.data = parsedRawData } @@ -131,9 +132,9 @@ export const memoryAddTool: ToolConfig = { try { const result = await response.json() let errorMessage = result.error?.message || 'Failed to add memory' - + const data = result.data || result - + // Extract the memories from the response based on memory type let memories if (data.type === 'agent') { @@ -143,13 +144,13 @@ export const memoryAddTool: ToolConfig = { // For raw memories, return the raw data object memories = data.data } - + return { success: true, output: { memories, }, - error: errorMessage + error: errorMessage, } } catch (error: any) { return { @@ -157,19 +158,19 @@ export const memoryAddTool: ToolConfig = { output: { memories: undefined, }, - error + error, } } }, transformError: async (error): Promise => { - const errorMessage = `Memory operation failed: ${error.message || 'Unknown error occurred'}`; + const errorMessage = `Memory operation failed: ${error.message || 'Unknown error occurred'}` return { success: false, output: { memories: undefined, - message: `Memory operation failed: ${error.message || 'Unknown error occurred'}` + message: `Memory operation failed: ${error.message || 'Unknown error occurred'}`, }, - error: errorMessage + error: errorMessage, } }, -} \ No newline at end of file +} diff --git a/apps/sim/tools/memory/delete_memory.ts b/apps/sim/tools/memory/delete.ts similarity index 87% rename from apps/sim/tools/memory/delete_memory.ts rename to apps/sim/tools/memory/delete.ts index b9e5be477..86f14fa7c 100644 --- a/apps/sim/tools/memory/delete_memory.ts +++ b/apps/sim/tools/memory/delete.ts @@ -1,7 +1,6 @@ import { ToolConfig } from '../types' import { MemoryResponse } from './types' -// Delete Memory Tool export const memoryDeleteTool: ToolConfig = { id: 'memory_delete', name: 'Delete Memory', @@ -12,13 +11,13 @@ export const memoryDeleteTool: ToolConfig = { type: 'string', required: true, description: 'Identifier for the memory to delete', - } + }, }, request: { url: (params): any => { // Get workflowId from context (set by workflow execution) const workflowId = params._context?.workflowId - + if (!workflowId) { return { _errorResponse: { @@ -26,13 +25,13 @@ export const memoryDeleteTool: ToolConfig = { data: { success: false, error: { - message: 'workflowId is required and must be provided in execution context' - } - } - } + message: 'workflowId is required and must be provided in execution context', + }, + }, + }, } } - + // Append workflowId as query parameter return `/api/memory/${encodeURIComponent(params.id)}?workflowId=${encodeURIComponent(workflowId)}` }, @@ -45,25 +44,25 @@ export const memoryDeleteTool: ToolConfig = { transformResponse: async (response): Promise => { try { const result = await response.json() - + if (!response.ok) { const errorMessage = result.error?.message || 'Failed to delete memory' throw new Error(errorMessage) } - + return { success: true, output: { - message: `Memory deleted successfully.` + message: `Memory deleted successfully.`, }, } } catch (error: any) { return { success: false, output: { - message: `Failed to delete memory: ${error.message || 'Unknown error'}` + message: `Failed to delete memory: ${error.message || 'Unknown error'}`, }, - error: `Failed to delete memory: ${error.message || 'Unknown error'}` + error: `Failed to delete memory: ${error.message || 'Unknown error'}`, } } }, @@ -72,9 +71,9 @@ export const memoryDeleteTool: ToolConfig = { return { success: false, output: { - message: errorMessage + message: errorMessage, }, - error: errorMessage + error: errorMessage, } }, -} \ No newline at end of file +} diff --git a/apps/sim/tools/memory/get_memory.ts b/apps/sim/tools/memory/get.ts similarity index 88% rename from apps/sim/tools/memory/get_memory.ts rename to apps/sim/tools/memory/get.ts index 9a3e18b65..21b183fca 100644 --- a/apps/sim/tools/memory/get_memory.ts +++ b/apps/sim/tools/memory/get.ts @@ -1,7 +1,6 @@ import { ToolConfig } from '../types' import { MemoryResponse } from './types' -// Get Memory Tool export const memoryGetTool: ToolConfig = { id: 'memory_get', name: 'Get Memory', @@ -12,13 +11,13 @@ export const memoryGetTool: ToolConfig = { type: 'string', required: true, description: 'Identifier for the memory to retrieve', - } + }, }, request: { url: (params): any => { // Get workflowId from context (set by workflow execution) const workflowId = params._context?.workflowId - + if (!workflowId) { return { _errorResponse: { @@ -26,13 +25,13 @@ export const memoryGetTool: ToolConfig = { data: { success: false, error: { - message: 'workflowId is required and must be provided in execution context' - } - } - } + message: 'workflowId is required and must be provided in execution context', + }, + }, + }, } } - + // Append workflowId as query parameter return `/api/memory/${encodeURIComponent(params.id)}?workflowId=${encodeURIComponent(workflowId)}` }, @@ -45,19 +44,19 @@ export const memoryGetTool: ToolConfig = { transformResponse: async (response): Promise => { try { const result = await response.json() - + if (!response.ok) { const errorMessage = result.error?.message || 'Failed to retrieve memory' throw new Error(errorMessage) } - + const data = result.data || result - + return { success: true, output: { memories: data.data, - message: 'Memory retrieved successfully' + message: 'Memory retrieved successfully', }, } } catch (error: any) { @@ -65,9 +64,9 @@ export const memoryGetTool: ToolConfig = { success: false, output: { memories: undefined, - message: `Failed to retrieve memory: ${error.message || 'Unknown error'}` + message: `Failed to retrieve memory: ${error.message || 'Unknown error'}`, }, - error: `Failed to retrieve memory: ${error.message || 'Unknown error'}` + error: `Failed to retrieve memory: ${error.message || 'Unknown error'}`, } } }, @@ -77,9 +76,9 @@ export const memoryGetTool: ToolConfig = { success: false, output: { memories: undefined, - message: errorMessage + message: errorMessage, }, - error: errorMessage + error: errorMessage, } }, -} \ No newline at end of file +} diff --git a/apps/sim/tools/memory/get_all_memories.ts b/apps/sim/tools/memory/get_all.ts similarity index 84% rename from apps/sim/tools/memory/get_all_memories.ts rename to apps/sim/tools/memory/get_all.ts index cd4de28f4..9e717465e 100644 --- a/apps/sim/tools/memory/get_all_memories.ts +++ b/apps/sim/tools/memory/get_all.ts @@ -1,7 +1,6 @@ import { ToolConfig } from '../types' import { MemoryResponse } from './types' -// Get All Memories Tool export const memoryGetAllTool: ToolConfig = { id: 'memory_get_all', name: 'Get All Memories', @@ -12,7 +11,7 @@ export const memoryGetAllTool: ToolConfig = { url: (params): any => { // Get workflowId from context (set by workflow execution) const workflowId = params._context?.workflowId - + if (!workflowId) { return { _errorResponse: { @@ -20,13 +19,13 @@ export const memoryGetAllTool: ToolConfig = { data: { success: false, error: { - message: 'workflowId is required and must be provided in execution context' - } - } - } + message: 'workflowId is required and must be provided in execution context', + }, + }, + }, } } - + // Append workflowId as query parameter return `/api/memory?workflowId=${encodeURIComponent(workflowId)}` }, @@ -39,28 +38,28 @@ export const memoryGetAllTool: ToolConfig = { transformResponse: async (response): Promise => { try { const result = await response.json() - + if (!response.ok) { const errorMessage = result.error?.message || 'Failed to retrieve memories' throw new Error(errorMessage) } - + // Extract memories from the response const data = result.data || result - let rawMemories = data.memories || data || []; - + let rawMemories = data.memories || data || [] + // Transform memories to return them with their keys and types for better context const memories = rawMemories.map((memory: any) => ({ key: memory.key, type: memory.type, - data: memory.data - })); - + data: memory.data, + })) + return { success: true, output: { memories, - message: 'Memories retrieved successfully' + message: 'Memories retrieved successfully', }, } } catch (error: any) { @@ -68,9 +67,9 @@ export const memoryGetAllTool: ToolConfig = { success: false, output: { memories: [], - message: `Failed to retrieve memories: ${error.message || 'Unknown error'}` + message: `Failed to retrieve memories: ${error.message || 'Unknown error'}`, }, - error: `Failed to retrieve memories: ${error.message || 'Unknown error'}` + error: `Failed to retrieve memories: ${error.message || 'Unknown error'}`, } } }, @@ -80,9 +79,9 @@ export const memoryGetAllTool: ToolConfig = { success: false, output: { memories: [], - message: errorMessage + message: errorMessage, }, - error: errorMessage + error: errorMessage, } }, -} \ No newline at end of file +} diff --git a/apps/sim/tools/memory/index.ts b/apps/sim/tools/memory/index.ts index 909d75afa..897e12027 100644 --- a/apps/sim/tools/memory/index.ts +++ b/apps/sim/tools/memory/index.ts @@ -1,6 +1,6 @@ -import { memoryAddTool } from './add_memory' -import { memoryGetTool } from './get_memory' -import { memoryGetAllTool } from './get_all_memories' -import { memoryDeleteTool } from './delete_memory' +import { memoryAddTool } from './add' +import { memoryDeleteTool } from './delete' +import { memoryGetTool } from './get' +import { memoryGetAllTool } from './get_all' -export { memoryAddTool, memoryGetTool, memoryGetAllTool, memoryDeleteTool } \ No newline at end of file +export { memoryAddTool, memoryGetTool, memoryGetAllTool, memoryDeleteTool } diff --git a/apps/sim/tools/openai/image.ts b/apps/sim/tools/openai/image.ts new file mode 100644 index 000000000..43d20e8d8 --- /dev/null +++ b/apps/sim/tools/openai/image.ts @@ -0,0 +1,190 @@ +import { createLogger } from '@/lib/logs/console-logger' +import { getBaseUrl } from '@/lib/urls/utils' +import { ToolConfig } from '../types' + +const logger = createLogger('DalleTool') + +export const imageTool: ToolConfig = { + id: 'openai_image', + name: 'Image Generate', + description: "Generate images using OpenAI's Image models", + version: '1.0.0', + params: { + prompt: { + type: 'string', + required: true, + description: 'A text description of the desired image(s)', + }, + model: { + type: 'string', + required: true, + description: 'The DALL-E model to use (gpt-image-1 or dall-e-3)', + }, + size: { + type: 'string', + required: false, + description: 'The size of the generated images (1024x1024, 1024x1792, or 1792x1024)', + }, + quality: { + type: 'string', + required: false, + description: 'The quality of the image (standard or hd)', + }, + style: { + type: 'string', + required: false, + description: 'The style of the image (vivid or natural)', + }, + n: { + type: 'number', + required: false, + description: 'The number of images to generate (1-10)', + }, + apiKey: { + type: 'string', + required: true, + description: 'Your OpenAI API key', + }, + }, + request: { + url: 'https://api.openai.com/v1/images/generations', + method: 'POST', + headers: (params) => ({ + 'Content-Type': 'application/json', + Authorization: `Bearer ${params.apiKey}`, + }), + body: (params) => ({ + model: params.model, + prompt: params.prompt, + size: params.size || '1024x1024', + quality: params.quality || 'standard', + style: params.style || 'vivid', + n: params.n || 1, + }), + }, + transformResponse: async (response, params) => { + try { + const data = await response.json() + + logger.info('Image API response:', JSON.stringify(data, null, 2)) + + if (!data.data?.[0]?.url) { + logger.error('No image URL in Image response:', data) + throw new Error('No image URL in response') + } + + const imageUrl = data.data[0].url + const modelName = data.model || params?.model || 'dall-e' + + try { + logger.info('Fetching image from URL via proxy...') + const baseUrl = getBaseUrl() + const proxyUrl = new URL(`/api/proxy/image`, baseUrl) + proxyUrl.searchParams.append('url', imageUrl) + + const imageResponse = await fetch(proxyUrl.toString(), { + headers: { + Accept: 'image/*, */*', + }, + cache: 'no-store', + }) + + if (!imageResponse.ok) { + logger.error('Failed to fetch image:', imageResponse.status, imageResponse.statusText) + throw new Error(`Failed to fetch image: ${imageResponse.statusText}`) + } + + const imageBlob = await imageResponse.blob() + + if (imageBlob.size === 0) { + logger.error('Empty image blob received') + throw new Error('Empty image received') + } + + const arrayBuffer = await imageBlob.arrayBuffer() + const buffer = Buffer.from(arrayBuffer) + const base64Image = buffer.toString('base64') + + return { + success: true, + output: { + content: imageUrl, // Now using image URL as content + image: base64Image, // Base64 image in separate field + metadata: { + model: modelName, // Only include model name in metadata + }, + }, + } + } catch (error) { + // Log the error but continue with returning the URL + logger.error('Error fetching or processing image:', error) + + // Try again with a direct browser fetch as fallback + try { + logger.info('Attempting fallback with direct browser fetch...') + const directImageResponse = await fetch(imageUrl, { + cache: 'no-store', + headers: { + Accept: 'image/*, */*', + 'User-Agent': 'Mozilla/5.0 (compatible DalleProxy/1.0)', + }, + }) + + if (!directImageResponse.ok) { + throw new Error(`Direct fetch failed: ${directImageResponse.status}`) + } + + const imageBlob = await directImageResponse.blob() + if (imageBlob.size === 0) { + throw new Error('Empty blob received from direct fetch') + } + + // Server-side safe way to convert blob to base64 + const arrayBuffer = await imageBlob.arrayBuffer() + const buffer = Buffer.from(arrayBuffer) + const base64Image = buffer.toString('base64') + + logger.info( + 'Successfully converted image to base64 via direct fetch, length:', + base64Image.length + ) + + return { + success: true, + output: { + content: imageUrl, + image: base64Image, + metadata: { + model: modelName, + }, + }, + } + } catch (fallbackError) { + logger.error('Fallback fetch also failed:', fallbackError) + + // Even if both attempts fail, still return the URL and metadata + return { + success: true, + output: { + content: imageUrl, // URL as content + image: '', // Empty image since we couldn't get it + metadata: { + model: modelName, + }, + }, + } + } + } + } catch (error) { + logger.error('Error in DALL-E response handling:', error) + throw error + } + }, + transformError: (error) => { + logger.error('DALL-E error:', error) + if (error.response?.data?.error?.message) { + return error.response.data.error.message + } + return error.message || 'Failed to generate image with DALL-E' + }, +} diff --git a/scripts/generate-block-docs.ts b/scripts/generate-block-docs.ts index 8bed56f1b..94d65e9c8 100644 --- a/scripts/generate-block-docs.ts +++ b/scripts/generate-block-docs.ts @@ -640,7 +640,7 @@ function parseOutputStructure( while ((fieldMatch = fieldRegex.exec(outputContent)) !== null) { const fieldName = fieldMatch[1].trim() - const fieldType = fieldMatch[2].trim().replace(/['"\[\]]/g, '') + const fieldType = fieldMatch[2].trim().replace(/["'\[\]]/g, '') // Determine a good description based on field name let description = 'Dynamic output field' @@ -658,6 +658,31 @@ function parseOutputStructure( outputs[fieldName] = description } + const shorthandRegex = /(?:^\s*|[,{]\s*)([A-Za-z_][\w]*)\s*(?=,|})/g + let shorthandMatch + + while ((shorthandMatch = shorthandRegex.exec(outputContent)) !== null) { + const fieldName = shorthandMatch[1].trim() + + // Ignore fields already captured or those that are part of key/value pairs + if (outputs[fieldName]) continue + + // Provide the same heuristic descriptions as above + let description = 'Dynamic output field' + + if (fieldName === 'results' || fieldName === 'memories' || fieldName === 'searchResults') { + description = `${fieldName.charAt(0).toUpperCase() + fieldName.slice(1)} from the operation` + } else if (fieldName === 'ids') { + description = 'IDs of created or retrieved resources' + } else if (fieldName === 'answer') { + description = 'Generated answer text' + } else if (fieldName === 'citations') { + description = 'References used to generate the answer' + } + + outputs[fieldName] = description + } + // Try to identify common patterns based on tool types if (Object.keys(outputs).length === 0) { if (toolName.includes('_search')) { @@ -884,9 +909,9 @@ async function generateBlockDoc(blockPath: string, icons: Record return } - // Skip blocks with category 'blocks', only process blocks with category 'tools', and skip specific blocks + // Skip blocks with category 'blocks' (except memory type), and skip specific blocks if ( - blockConfig.category === 'blocks' || + (blockConfig.category === 'blocks' && blockConfig.type !== 'memory') || blockConfig.type === 'evaluator' || blockConfig.type === 'number' ) {