diff --git a/.claude/commands/add-block.md b/.claude/commands/add-block.md index 94386bbee..d31edcf01 100644 --- a/.claude/commands/add-block.md +++ b/.claude/commands/add-block.md @@ -183,6 +183,109 @@ export const {ServiceName}Block: BlockConfig = { } ``` +## File Input Handling + +When your block accepts file uploads, use the basic/advanced mode pattern with `normalizeFileInput`. + +### Basic/Advanced File Pattern + +```typescript +// Basic mode: Visual file upload +{ + id: 'uploadFile', + title: 'File', + type: 'file-upload', + canonicalParamId: 'file', // Both map to 'file' param + placeholder: 'Upload file', + mode: 'basic', + multiple: false, + required: true, + condition: { field: 'operation', value: 'upload' }, +}, +// Advanced mode: Reference from other blocks +{ + id: 'fileRef', + title: 'File', + type: 'short-input', + canonicalParamId: 'file', // Both map to 'file' param + placeholder: 'Reference file (e.g., {{file_block.output}})', + mode: 'advanced', + required: true, + condition: { field: 'operation', value: 'upload' }, +}, +``` + +**Critical constraints:** +- `canonicalParamId` must NOT match any subblock's `id` in the same block +- Values are stored under subblock `id`, not `canonicalParamId` + +### Normalizing File Input in tools.config + +Use `normalizeFileInput` to handle all input variants: + +```typescript +import { normalizeFileInput } from '@/blocks/utils' + +tools: { + access: ['service_upload'], + config: { + tool: (params) => { + // Check all field IDs: uploadFile (basic), fileRef (advanced), fileContent (legacy) + const normalizedFile = normalizeFileInput( + params.uploadFile || params.fileRef || params.fileContent, + { single: true } + ) + if (normalizedFile) { + params.file = normalizedFile + } + return `service_${params.operation}` + }, + }, +} +``` + +**Why this pattern?** +- Values come through as `params.uploadFile` or `params.fileRef` (the subblock IDs) +- `canonicalParamId` only controls UI/schema mapping, not runtime values +- `normalizeFileInput` handles JSON strings from advanced mode template resolution + +### File Input Types in `inputs` + +Use `type: 'json'` for file inputs: + +```typescript +inputs: { + uploadFile: { type: 'json', description: 'Uploaded file (UserFile)' }, + fileRef: { type: 'json', description: 'File reference from previous block' }, + // Legacy field for backwards compatibility + fileContent: { type: 'string', description: 'Legacy: base64 encoded content' }, +} +``` + +### Multiple Files + +For multiple file uploads: + +```typescript +{ + id: 'attachments', + title: 'Attachments', + type: 'file-upload', + multiple: true, // Allow multiple files + maxSize: 25, // Max size in MB per file + acceptedTypes: 'image/*,application/pdf,.doc,.docx', +} + +// In tools.config: +const normalizedFiles = normalizeFileInput( + params.attachments || params.attachmentRefs, + // No { single: true } - returns array +) +if (normalizedFiles) { + params.files = normalizedFiles +} +``` + ## Condition Syntax Controls when a field is shown based on other field values. diff --git a/.claude/commands/add-integration.md b/.claude/commands/add-integration.md index a6d2af5cf..12350ccd5 100644 --- a/.claude/commands/add-integration.md +++ b/.claude/commands/add-integration.md @@ -457,7 +457,230 @@ You can usually find this in the service's brand/press kit page, or copy it from Paste the SVG code here and I'll convert it to a React component. ``` -## Common Gotchas +## File Handling + +When your integration handles file uploads or downloads, follow these patterns to work with `UserFile` objects consistently. + +### What is a UserFile? + +A `UserFile` is the standard file representation in Sim: + +```typescript +interface UserFile { + id: string // Unique identifier + name: string // Original filename + url: string // Presigned URL for download + size: number // File size in bytes + type: string // MIME type (e.g., 'application/pdf') + base64?: string // Optional base64 content (if small file) + key?: string // Internal storage key + context?: object // Storage context metadata +} +``` + +### File Input Pattern (Uploads) + +For tools that accept file uploads, **always route through an internal API endpoint** rather than calling external APIs directly. This ensures proper file content retrieval. + +#### 1. Block SubBlocks for File Input + +Use the basic/advanced mode pattern: + +```typescript +// Basic mode: File upload UI +{ + id: 'uploadFile', + title: 'File', + type: 'file-upload', + canonicalParamId: 'file', // Maps to 'file' param + placeholder: 'Upload file', + mode: 'basic', + multiple: false, + required: true, + condition: { field: 'operation', value: 'upload' }, +}, +// Advanced mode: Reference from previous block +{ + id: 'fileRef', + title: 'File', + type: 'short-input', + canonicalParamId: 'file', // Same canonical param + placeholder: 'Reference file (e.g., {{file_block.output}})', + mode: 'advanced', + required: true, + condition: { field: 'operation', value: 'upload' }, +}, +``` + +**Critical:** `canonicalParamId` must NOT match any subblock `id`. + +#### 2. Normalize File Input in Block Config + +In `tools.config.tool`, use `normalizeFileInput` to handle all input variants: + +```typescript +import { normalizeFileInput } from '@/blocks/utils' + +tools: { + config: { + tool: (params) => { + // Normalize file from basic (uploadFile), advanced (fileRef), or legacy (fileContent) + const normalizedFile = normalizeFileInput( + params.uploadFile || params.fileRef || params.fileContent, + { single: true } + ) + if (normalizedFile) { + params.file = normalizedFile + } + return `{service}_${params.operation}` + }, + }, +} +``` + +#### 3. Create Internal API Route + +Create `apps/sim/app/api/tools/{service}/{action}/route.ts`: + +```typescript +import { createLogger } from '@sim/logger' +import { NextResponse, type NextRequest } from 'next/server' +import { z } from 'zod' +import { checkInternalAuth } from '@/lib/auth/hybrid' +import { generateRequestId } from '@/lib/core/utils/request' +import { FileInputSchema, type RawFileInput } from '@/lib/uploads/utils/file-schemas' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' + +const logger = createLogger('{Service}UploadAPI') + +const RequestSchema = z.object({ + accessToken: z.string(), + file: FileInputSchema.optional().nullable(), + // Legacy field for backwards compatibility + fileContent: z.string().optional().nullable(), + // ... other params +}) + +export async function POST(request: NextRequest) { + const requestId = generateRequestId() + + const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) + if (!authResult.success) { + return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) + } + + const body = await request.json() + const data = RequestSchema.parse(body) + + let fileBuffer: Buffer + let fileName: string + + // Prefer UserFile input, fall back to legacy base64 + if (data.file) { + const userFiles = processFilesToUserFiles([data.file as RawFileInput], requestId, logger) + if (userFiles.length === 0) { + return NextResponse.json({ success: false, error: 'Invalid file' }, { status: 400 }) + } + const userFile = userFiles[0] + fileBuffer = await downloadFileFromStorage(userFile, requestId, logger) + fileName = userFile.name + } else if (data.fileContent) { + // Legacy: base64 string (backwards compatibility) + fileBuffer = Buffer.from(data.fileContent, 'base64') + fileName = 'file' + } else { + return NextResponse.json({ success: false, error: 'File required' }, { status: 400 }) + } + + // Now call external API with fileBuffer + const response = await fetch('https://api.{service}.com/upload', { + method: 'POST', + headers: { Authorization: `Bearer ${data.accessToken}` }, + body: new Uint8Array(fileBuffer), // Convert Buffer for fetch + }) + + // ... handle response +} +``` + +#### 4. Update Tool to Use Internal Route + +```typescript +export const {service}UploadTool: ToolConfig = { + id: '{service}_upload', + // ... + params: { + file: { type: 'file', required: false, visibility: 'user-or-llm' }, + fileContent: { type: 'string', required: false, visibility: 'hidden' }, // Legacy + }, + request: { + url: '/api/tools/{service}/upload', // Internal route + method: 'POST', + body: (params) => ({ + accessToken: params.accessToken, + file: params.file, + fileContent: params.fileContent, + }), + }, +} +``` + +### File Output Pattern (Downloads) + +For tools that return files, use `FileToolProcessor` to store files and return `UserFile` objects. + +#### In Tool transformResponse + +```typescript +import { FileToolProcessor } from '@/executor/utils/file-tool-processor' + +transformResponse: async (response, context) => { + const data = await response.json() + + // Process file outputs to UserFile objects + const fileProcessor = new FileToolProcessor(context) + const file = await fileProcessor.processFileData({ + data: data.content, // base64 or buffer + mimeType: data.mimeType, + filename: data.filename, + }) + + return { + success: true, + output: { file }, + } +} +``` + +#### In API Route (for complex file handling) + +```typescript +// Return file data that FileToolProcessor can handle +return NextResponse.json({ + success: true, + output: { + file: { + data: base64Content, + mimeType: 'application/pdf', + filename: 'document.pdf', + }, + }, +}) +``` + +### Key Helpers Reference + +| Helper | Location | Purpose | +|--------|----------|---------| +| `normalizeFileInput` | `@/blocks/utils` | Normalize file params in block config | +| `processFilesToUserFiles` | `@/lib/uploads/utils/file-utils` | Convert raw inputs to UserFile[] | +| `downloadFileFromStorage` | `@/lib/uploads/utils/file-utils.server` | Get file Buffer from UserFile | +| `FileToolProcessor` | `@/executor/utils/file-tool-processor` | Process tool output files | +| `isUserFile` | `@/lib/core/utils/user-file` | Type guard for UserFile objects | +| `FileInputSchema` | `@/lib/uploads/utils/file-schemas` | Zod schema for file validation | + +### Common Gotchas 1. **OAuth serviceId must match** - The `serviceId` in oauth-input must match the OAuth provider configuration 2. **Tool IDs are snake_case** - `stripe_create_payment`, not `stripeCreatePayment` @@ -465,3 +688,5 @@ Paste the SVG code here and I'll convert it to a React component. 4. **Alphabetical ordering** - Keep imports and registry entries alphabetically sorted 5. **Required can be conditional** - Use `required: { field: 'op', value: 'create' }` instead of always true 6. **DependsOn clears options** - When a dependency changes, selector options are refetched +7. **Never pass Buffer directly to fetch** - Convert to `new Uint8Array(buffer)` for TypeScript compatibility +8. **Always handle legacy file params** - Keep hidden `fileContent` params for backwards compatibility diff --git a/.claude/rules/sim-integrations.md b/.claude/rules/sim-integrations.md index cef0c895b..825acce5d 100644 --- a/.claude/rules/sim-integrations.md +++ b/.claude/rules/sim-integrations.md @@ -195,6 +195,52 @@ import { {service}WebhookTrigger } from '@/triggers/{service}' {service}_webhook: {service}WebhookTrigger, ``` +## File Handling + +When integrations handle file uploads/downloads, use `UserFile` objects consistently. + +### File Input (Uploads) + +1. **Block subBlocks:** Use basic/advanced mode pattern with `canonicalParamId` +2. **Normalize in block config:** Use `normalizeFileInput` from `@/blocks/utils` +3. **Internal API route:** Create route that uses `downloadFileFromStorage` to get file content +4. **Tool routes to internal API:** Don't call external APIs directly with files + +```typescript +// In block tools.config: +import { normalizeFileInput } from '@/blocks/utils' + +const normalizedFile = normalizeFileInput( + params.uploadFile || params.fileRef || params.fileContent, + { single: true } +) +if (normalizedFile) params.file = normalizedFile +``` + +### File Output (Downloads) + +Use `FileToolProcessor` in tool `transformResponse` to store files: + +```typescript +import { FileToolProcessor } from '@/executor/utils/file-tool-processor' + +const processor = new FileToolProcessor(context) +const file = await processor.processFileData({ + data: base64Content, + mimeType: 'application/pdf', + filename: 'doc.pdf', +}) +``` + +### Key Helpers + +| Helper | Location | Purpose | +|--------|----------|---------| +| `normalizeFileInput` | `@/blocks/utils` | Normalize file params in block config | +| `processFilesToUserFiles` | `@/lib/uploads/utils/file-utils` | Convert raw inputs to UserFile[] | +| `downloadFileFromStorage` | `@/lib/uploads/utils/file-utils.server` | Get Buffer from UserFile | +| `FileToolProcessor` | `@/executor/utils/file-tool-processor` | Process tool output files | + ## Checklist - [ ] Look up API docs for the service @@ -207,3 +253,5 @@ import { {service}WebhookTrigger } from '@/triggers/{service}' - [ ] Register block in `blocks/registry.ts` - [ ] (Optional) Create triggers in `triggers/{service}/` - [ ] (Optional) Register triggers in `triggers/registry.ts` +- [ ] (If file uploads) Create internal API route with `downloadFileFromStorage` +- [ ] (If file uploads) Use `normalizeFileInput` in block config diff --git a/.cursor/rules/sim-integrations.mdc b/.cursor/rules/sim-integrations.mdc index 9923ec009..20edc82e1 100644 --- a/.cursor/rules/sim-integrations.mdc +++ b/.cursor/rules/sim-integrations.mdc @@ -193,6 +193,52 @@ import { {service}WebhookTrigger } from '@/triggers/{service}' {service}_webhook: {service}WebhookTrigger, ``` +## File Handling + +When integrations handle file uploads/downloads, use `UserFile` objects consistently. + +### File Input (Uploads) + +1. **Block subBlocks:** Use basic/advanced mode pattern with `canonicalParamId` +2. **Normalize in block config:** Use `normalizeFileInput` from `@/blocks/utils` +3. **Internal API route:** Create route that uses `downloadFileFromStorage` to get file content +4. **Tool routes to internal API:** Don't call external APIs directly with files + +```typescript +// In block tools.config: +import { normalizeFileInput } from '@/blocks/utils' + +const normalizedFile = normalizeFileInput( + params.uploadFile || params.fileRef || params.fileContent, + { single: true } +) +if (normalizedFile) params.file = normalizedFile +``` + +### File Output (Downloads) + +Use `FileToolProcessor` in tool `transformResponse` to store files: + +```typescript +import { FileToolProcessor } from '@/executor/utils/file-tool-processor' + +const processor = new FileToolProcessor(context) +const file = await processor.processFileData({ + data: base64Content, + mimeType: 'application/pdf', + filename: 'doc.pdf', +}) +``` + +### Key Helpers + +| Helper | Location | Purpose | +|--------|----------|---------| +| `normalizeFileInput` | `@/blocks/utils` | Normalize file params in block config | +| `processFilesToUserFiles` | `@/lib/uploads/utils/file-utils` | Convert raw inputs to UserFile[] | +| `downloadFileFromStorage` | `@/lib/uploads/utils/file-utils.server` | Get Buffer from UserFile | +| `FileToolProcessor` | `@/executor/utils/file-tool-processor` | Process tool output files | + ## Checklist - [ ] Look up API docs for the service @@ -205,3 +251,5 @@ import { {service}WebhookTrigger } from '@/triggers/{service}' - [ ] Register block in `blocks/registry.ts` - [ ] (Optional) Create triggers in `triggers/{service}/` - [ ] (Optional) Register triggers in `triggers/registry.ts` +- [ ] (If file uploads) Create internal API route with `downloadFileFromStorage` +- [ ] (If file uploads) Use `normalizeFileInput` in block config diff --git a/CLAUDE.md b/CLAUDE.md index 7f18cee1c..71e8ef716 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -265,6 +265,23 @@ Register in `blocks/registry.ts` (alphabetically). **dependsOn:** `['field']` or `{ all: ['a'], any: ['b', 'c'] }` +**File Input Pattern (basic/advanced mode):** +```typescript +// Basic: file-upload UI +{ id: 'uploadFile', type: 'file-upload', canonicalParamId: 'file', mode: 'basic' }, +// Advanced: reference from other blocks +{ id: 'fileRef', type: 'short-input', canonicalParamId: 'file', mode: 'advanced' }, +``` + +In `tools.config.tool`, normalize with: +```typescript +import { normalizeFileInput } from '@/blocks/utils' +const file = normalizeFileInput(params.uploadFile || params.fileRef, { single: true }) +if (file) params.file = file +``` + +For file uploads, create an internal API route (`/api/tools/{service}/upload`) that uses `downloadFileFromStorage` to get file content from `UserFile` objects. + ### 3. Icon (`components/icons.tsx`) ```typescript @@ -293,3 +310,5 @@ Register in `triggers/registry.ts`. - [ ] Create block in `blocks/blocks/{service}.ts` - [ ] Register block in `blocks/registry.ts` - [ ] (Optional) Create and register triggers +- [ ] (If file uploads) Create internal API route with `downloadFileFromStorage` +- [ ] (If file uploads) Use `normalizeFileInput` in block config diff --git a/apps/docs/content/docs/en/execution/files.mdx b/apps/docs/content/docs/en/execution/files.mdx new file mode 100644 index 000000000..125093952 --- /dev/null +++ b/apps/docs/content/docs/en/execution/files.mdx @@ -0,0 +1,168 @@ +--- +title: Passing Files +--- + +import { Callout } from 'fumadocs-ui/components/callout' +import { Tab, Tabs } from 'fumadocs-ui/components/tabs' + +Sim makes it easy to work with files throughout your workflows. Blocks can receive files, process them, and pass them to other blocks seamlessly. + +## File Objects + +When blocks output files (like Gmail attachments, generated images, or parsed documents), they return a standardized file object: + +```json +{ + "name": "report.pdf", + "url": "https://...", + "base64": "JVBERi0xLjQK...", + "type": "application/pdf", + "size": 245678 +} +``` + +You can access any of these properties when referencing files from previous blocks. + +## The File Block + +The **File block** is the universal entry point for files in your workflows. It accepts files from any source and outputs standardized file objects that work with all integrations. + +**Inputs:** +- **Uploaded files** - Drag and drop or select files directly +- **External URLs** - Any publicly accessible file URL +- **Files from other blocks** - Pass files from Gmail attachments, Slack downloads, etc. + +**Outputs:** +- A list of `UserFile` objects with consistent structure (`name`, `url`, `base64`, `type`, `size`) +- `combinedContent` - Extracted text content from all files (for documents) + +**Example usage:** + +``` +// Get all files from the File block + + +// Get the first file + + +// Get combined text content from parsed documents + +``` + +The File block automatically: +- Detects file types from URLs and extensions +- Extracts text from PDFs, CSVs, and documents +- Generates base64 encoding for binary files +- Creates presigned URLs for secure access + +Use the File block when you need to normalize files from different sources before passing them to other blocks like Vision, STT, or email integrations. + +## Passing Files Between Blocks + +Reference files from previous blocks using the tag dropdown. Click in any file input field and type `<` to see available outputs. + +**Common patterns:** + +``` +// Single file from a block + + +// Pass the whole file object + + +// Access specific properties + + +``` + +Most blocks accept the full file object and extract what they need automatically. You don't need to manually extract `base64` or `url` in most cases. + +## Triggering Workflows with Files + +When calling a workflow via API that expects file input, include files in your request: + + + + ```bash + curl -X POST "https://sim.ai/api/workflows/YOUR_WORKFLOW_ID/execute" \ + -H "Content-Type: application/json" \ + -H "x-api-key: YOUR_API_KEY" \ + -d '{ + "document": { + "name": "report.pdf", + "base64": "JVBERi0xLjQK...", + "type": "application/pdf" + } + }' + ``` + + + ```bash + curl -X POST "https://sim.ai/api/workflows/YOUR_WORKFLOW_ID/execute" \ + -H "Content-Type: application/json" \ + -H "x-api-key: YOUR_API_KEY" \ + -d '{ + "document": { + "name": "report.pdf", + "url": "https://example.com/report.pdf", + "type": "application/pdf" + } + }' + ``` + + + +The workflow's Start block should have an input field configured to receive the file parameter. + +## Receiving Files in API Responses + +When a workflow outputs files, they're included in the response: + +```json +{ + "success": true, + "output": { + "generatedFile": { + "name": "output.png", + "url": "https://...", + "base64": "iVBORw0KGgo...", + "type": "image/png", + "size": 34567 + } + } +} +``` + +Use `url` for direct downloads or `base64` for inline processing. + +## Blocks That Work with Files + +**File inputs:** +- **File** - Parse documents, images, and text files +- **Vision** - Analyze images with AI models +- **Mistral Parser** - Extract text from PDFs + +**File outputs:** +- **Gmail** - Email attachments +- **Slack** - Downloaded files +- **TTS** - Generated audio files +- **Video Generator** - Generated videos +- **Image Generator** - Generated images + +**File storage:** +- **Supabase** - Upload/download from storage +- **S3** - AWS S3 operations +- **Google Drive** - Drive file operations +- **Dropbox** - Dropbox file operations + + + Files are automatically available to downstream blocks. The execution engine handles all file transfer and format conversion. + + +## Best Practices + +1. **Use file objects directly** - Pass the full file object rather than extracting individual properties. Blocks handle the conversion automatically. + +2. **Check file types** - Ensure the file type matches what the receiving block expects. The Vision block needs images, the File block handles documents. + +3. **Consider file size** - Large files increase execution time. For very large files, consider using storage blocks (S3, Supabase) for intermediate storage. diff --git a/apps/docs/content/docs/en/execution/meta.json b/apps/docs/content/docs/en/execution/meta.json index 37cac68f5..fd2124b9d 100644 --- a/apps/docs/content/docs/en/execution/meta.json +++ b/apps/docs/content/docs/en/execution/meta.json @@ -1,3 +1,3 @@ { - "pages": ["index", "basics", "api", "logging", "costs"] + "pages": ["index", "basics", "files", "api", "logging", "costs"] } diff --git a/apps/sim/app/api/a2a/serve/[agentId]/route.ts b/apps/sim/app/api/a2a/serve/[agentId]/route.ts index 45fe2906b..f8acda5a8 100644 --- a/apps/sim/app/api/a2a/serve/[agentId]/route.ts +++ b/apps/sim/app/api/a2a/serve/[agentId]/route.ts @@ -16,7 +16,7 @@ import { import { checkHybridAuth } from '@/lib/auth/hybrid' import { getBrandConfig } from '@/lib/branding/branding' import { acquireLock, getRedisClient, releaseLock } from '@/lib/core/config/redis' -import { validateExternalUrl } from '@/lib/core/security/input-validation' +import { validateUrlWithDNS } from '@/lib/core/security/input-validation.server' import { SSE_HEADERS } from '@/lib/core/utils/sse' import { getBaseUrl } from '@/lib/core/utils/urls' import { markExecutionCancelled } from '@/lib/execution/cancellation' @@ -1119,7 +1119,7 @@ async function handlePushNotificationSet( ) } - const urlValidation = validateExternalUrl( + const urlValidation = await validateUrlWithDNS( params.pushNotificationConfig.url, 'Push notification URL' ) diff --git a/apps/sim/app/api/files/parse/route.ts b/apps/sim/app/api/files/parse/route.ts index 50dc55572..25112133f 100644 --- a/apps/sim/app/api/files/parse/route.ts +++ b/apps/sim/app/api/files/parse/route.ts @@ -6,7 +6,11 @@ import { createLogger } from '@sim/logger' import binaryExtensionsList from 'binary-extensions' import { type NextRequest, NextResponse } from 'next/server' import { checkHybridAuth } from '@/lib/auth/hybrid' -import { secureFetchWithPinnedIP, validateUrlWithDNS } from '@/lib/core/security/input-validation' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { sanitizeUrlForLog } from '@/lib/core/utils/logging' import { isSupportedFileType, parseFile } from '@/lib/file-parsers' import { isUsingCloudStorage, type StorageContext, StorageService } from '@/lib/uploads' import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' @@ -19,6 +23,7 @@ import { getMimeTypeFromExtension, getViewerUrl, inferContextFromKey, + isInternalFileUrl, } from '@/lib/uploads/utils/file-utils' import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' import { verifyFileAccess } from '@/app/api/files/authorization' @@ -215,7 +220,7 @@ async function parseFileSingle( } } - if (filePath.includes('/api/files/serve/')) { + if (isInternalFileUrl(filePath)) { return handleCloudFile(filePath, fileType, undefined, userId, executionContext) } @@ -246,7 +251,7 @@ function validateFilePath(filePath: string): { isValid: boolean; error?: string return { isValid: false, error: 'Invalid path: tilde character not allowed' } } - if (filePath.startsWith('/') && !filePath.startsWith('/api/files/serve/')) { + if (filePath.startsWith('/') && !isInternalFileUrl(filePath)) { return { isValid: false, error: 'Path outside allowed directory' } } @@ -420,7 +425,7 @@ async function handleExternalUrl( return parseResult } catch (error) { - logger.error(`Error handling external URL ${url}:`, error) + logger.error(`Error handling external URL ${sanitizeUrlForLog(url)}:`, error) return { success: false, error: `Error fetching URL: ${(error as Error).message}`, diff --git a/apps/sim/app/api/tools/a2a/send-message/route.ts b/apps/sim/app/api/tools/a2a/send-message/route.ts index a66c2b3d3..4c98dc67a 100644 --- a/apps/sim/app/api/tools/a2a/send-message/route.ts +++ b/apps/sim/app/api/tools/a2a/send-message/route.ts @@ -4,6 +4,7 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { createA2AClient, extractTextContent, isTerminalState } from '@/lib/a2a/utils' import { checkHybridAuth } from '@/lib/auth/hybrid' +import { validateUrlWithDNS } from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' export const dynamic = 'force-dynamic' @@ -95,6 +96,14 @@ export async function POST(request: NextRequest) { if (validatedData.files && validatedData.files.length > 0) { for (const file of validatedData.files) { if (file.type === 'url') { + const urlValidation = await validateUrlWithDNS(file.data, 'fileUrl') + if (!urlValidation.isValid) { + return NextResponse.json( + { success: false, error: urlValidation.error }, + { status: 400 } + ) + } + const filePart: FilePart = { kind: 'file', file: { diff --git a/apps/sim/app/api/tools/a2a/set-push-notification/route.ts b/apps/sim/app/api/tools/a2a/set-push-notification/route.ts index 11dbf7684..132bb6be2 100644 --- a/apps/sim/app/api/tools/a2a/set-push-notification/route.ts +++ b/apps/sim/app/api/tools/a2a/set-push-notification/route.ts @@ -3,7 +3,7 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { createA2AClient } from '@/lib/a2a/utils' import { checkHybridAuth } from '@/lib/auth/hybrid' -import { validateExternalUrl } from '@/lib/core/security/input-validation' +import { validateUrlWithDNS } from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' export const dynamic = 'force-dynamic' @@ -40,7 +40,7 @@ export async function POST(request: NextRequest) { const body = await request.json() const validatedData = A2ASetPushNotificationSchema.parse(body) - const urlValidation = validateExternalUrl(validatedData.webhookUrl, 'Webhook URL') + const urlValidation = await validateUrlWithDNS(validatedData.webhookUrl, 'Webhook URL') if (!urlValidation.isValid) { logger.warn(`[${requestId}] Invalid webhook URL`, { error: urlValidation.error }) return NextResponse.json( diff --git a/apps/sim/app/api/tools/confluence/upload-attachment/route.ts b/apps/sim/app/api/tools/confluence/upload-attachment/route.ts index f6be92f3f..599d70b75 100644 --- a/apps/sim/app/api/tools/confluence/upload-attachment/route.ts +++ b/apps/sim/app/api/tools/confluence/upload-attachment/route.ts @@ -92,6 +92,9 @@ export async function POST(request: NextRequest) { formData.append('comment', comment) } + // Add minorEdit field as required by Confluence API + formData.append('minorEdit', 'false') + const response = await fetch(url, { method: 'POST', headers: { diff --git a/apps/sim/app/api/tools/discord/send-message/route.ts b/apps/sim/app/api/tools/discord/send-message/route.ts index 273657a61..f5bf7d27f 100644 --- a/apps/sim/app/api/tools/discord/send-message/route.ts +++ b/apps/sim/app/api/tools/discord/send-message/route.ts @@ -4,6 +4,7 @@ import { z } from 'zod' import { checkInternalAuth } from '@/lib/auth/hybrid' import { validateNumericId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' +import { RawFileInputArraySchema } from '@/lib/uploads/utils/file-schemas' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -15,7 +16,7 @@ const DiscordSendMessageSchema = z.object({ botToken: z.string().min(1, 'Bot token is required'), channelId: z.string().min(1, 'Channel ID is required'), content: z.string().optional().nullable(), - files: z.array(z.any()).optional().nullable(), + files: RawFileInputArraySchema.optional().nullable(), }) export async function POST(request: NextRequest) { @@ -101,6 +102,12 @@ export async function POST(request: NextRequest) { logger.info(`[${requestId}] Processing ${validatedData.files.length} file(s)`) const userFiles = processFilesToUserFiles(validatedData.files, requestId, logger) + const filesOutput: Array<{ + name: string + mimeType: string + data: string + size: number + }> = [] if (userFiles.length === 0) { logger.warn(`[${requestId}] No valid files to upload, falling back to text-only`) @@ -137,6 +144,12 @@ export async function POST(request: NextRequest) { logger.info(`[${requestId}] Downloading file ${i}: ${userFile.name}`) const buffer = await downloadFileFromStorage(userFile, requestId, logger) + filesOutput.push({ + name: userFile.name, + mimeType: userFile.type || 'application/octet-stream', + data: buffer.toString('base64'), + size: buffer.length, + }) const blob = new Blob([new Uint8Array(buffer)], { type: userFile.type }) formData.append(`files[${i}]`, blob, userFile.name) @@ -173,6 +186,7 @@ export async function POST(request: NextRequest) { message: data.content, data: data, fileCount: userFiles.length, + files: filesOutput, }, }) } catch (error) { diff --git a/apps/sim/app/api/tools/dropbox/upload/route.ts b/apps/sim/app/api/tools/dropbox/upload/route.ts new file mode 100644 index 000000000..629a1dfbd --- /dev/null +++ b/apps/sim/app/api/tools/dropbox/upload/route.ts @@ -0,0 +1,141 @@ +import { createLogger } from '@sim/logger' +import { type NextRequest, NextResponse } from 'next/server' +import { z } from 'zod' +import { checkInternalAuth } from '@/lib/auth/hybrid' +import { generateRequestId } from '@/lib/core/utils/request' +import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' +import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' +import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' + +export const dynamic = 'force-dynamic' + +const logger = createLogger('DropboxUploadAPI') + +/** + * Escapes non-ASCII characters in JSON string for HTTP header safety. + * Dropbox API requires characters 0x7F and all non-ASCII to be escaped as \uXXXX. + */ +function httpHeaderSafeJson(value: object): string { + return JSON.stringify(value).replace(/[\u007f-\uffff]/g, (c) => { + return '\\u' + ('0000' + c.charCodeAt(0).toString(16)).slice(-4) + }) +} + +const DropboxUploadSchema = z.object({ + accessToken: z.string().min(1, 'Access token is required'), + path: z.string().min(1, 'Destination path is required'), + file: FileInputSchema.optional().nullable(), + // Legacy field for backwards compatibility + fileContent: z.string().optional().nullable(), + fileName: z.string().optional().nullable(), + mode: z.enum(['add', 'overwrite']).optional().nullable(), + autorename: z.boolean().optional().nullable(), + mute: z.boolean().optional().nullable(), +}) + +export async function POST(request: NextRequest) { + const requestId = generateRequestId() + + try { + const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) + + if (!authResult.success) { + logger.warn(`[${requestId}] Unauthorized Dropbox upload attempt: ${authResult.error}`) + return NextResponse.json( + { success: false, error: authResult.error || 'Authentication required' }, + { status: 401 } + ) + } + + logger.info(`[${requestId}] Authenticated Dropbox upload request via ${authResult.authType}`) + + const body = await request.json() + const validatedData = DropboxUploadSchema.parse(body) + + let fileBuffer: Buffer + let fileName: string + + // Prefer UserFile input, fall back to legacy base64 string + if (validatedData.file) { + // Process UserFile input + const userFiles = processFilesToUserFiles( + [validatedData.file as RawFileInput], + requestId, + logger + ) + + if (userFiles.length === 0) { + return NextResponse.json({ success: false, error: 'Invalid file input' }, { status: 400 }) + } + + const userFile = userFiles[0] + logger.info(`[${requestId}] Downloading file: ${userFile.name} (${userFile.size} bytes)`) + + fileBuffer = await downloadFileFromStorage(userFile, requestId, logger) + fileName = userFile.name + } else if (validatedData.fileContent) { + // Legacy: base64 string input (backwards compatibility) + logger.info(`[${requestId}] Using legacy base64 content input`) + fileBuffer = Buffer.from(validatedData.fileContent, 'base64') + fileName = validatedData.fileName || 'file' + } else { + return NextResponse.json({ success: false, error: 'File is required' }, { status: 400 }) + } + + // Determine final path + let finalPath = validatedData.path + if (finalPath.endsWith('/')) { + finalPath = `${finalPath}${fileName}` + } + + logger.info(`[${requestId}] Uploading to Dropbox: ${finalPath} (${fileBuffer.length} bytes)`) + + const dropboxApiArg = { + path: finalPath, + mode: validatedData.mode || 'add', + autorename: validatedData.autorename ?? true, + mute: validatedData.mute ?? false, + } + + const response = await fetch('https://content.dropboxapi.com/2/files/upload', { + method: 'POST', + headers: { + Authorization: `Bearer ${validatedData.accessToken}`, + 'Content-Type': 'application/octet-stream', + 'Dropbox-API-Arg': httpHeaderSafeJson(dropboxApiArg), + }, + body: new Uint8Array(fileBuffer), + }) + + const data = await response.json() + + if (!response.ok) { + const errorMessage = data.error_summary || data.error?.message || 'Failed to upload file' + logger.error(`[${requestId}] Dropbox API error:`, { status: response.status, data }) + return NextResponse.json({ success: false, error: errorMessage }, { status: response.status }) + } + + logger.info(`[${requestId}] File uploaded successfully to ${data.path_display}`) + + return NextResponse.json({ + success: true, + output: { + file: data, + }, + }) + } catch (error) { + if (error instanceof z.ZodError) { + logger.warn(`[${requestId}] Validation error:`, error.errors) + return NextResponse.json( + { success: false, error: error.errors[0]?.message || 'Validation failed' }, + { status: 400 } + ) + } + + logger.error(`[${requestId}] Unexpected error:`, error) + return NextResponse.json( + { success: false, error: error instanceof Error ? error.message : 'Unknown error' }, + { status: 500 } + ) + } +} diff --git a/apps/sim/app/api/tools/github/latest-commit/route.ts b/apps/sim/app/api/tools/github/latest-commit/route.ts new file mode 100644 index 000000000..39d088dbe --- /dev/null +++ b/apps/sim/app/api/tools/github/latest-commit/route.ts @@ -0,0 +1,195 @@ +import { createLogger } from '@sim/logger' +import { type NextRequest, NextResponse } from 'next/server' +import { z } from 'zod' +import { checkInternalAuth } from '@/lib/auth/hybrid' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { generateRequestId } from '@/lib/core/utils/request' + +export const dynamic = 'force-dynamic' + +const logger = createLogger('GitHubLatestCommitAPI') + +interface GitHubErrorResponse { + message?: string +} + +interface GitHubCommitResponse { + sha: string + html_url: string + commit: { + message: string + author: { name: string; email: string; date: string } + committer: { name: string; email: string; date: string } + } + author?: { login: string; avatar_url: string; html_url: string } + committer?: { login: string; avatar_url: string; html_url: string } + stats?: { additions: number; deletions: number; total: number } + files?: Array<{ + filename: string + status: string + additions: number + deletions: number + changes: number + patch?: string + raw_url?: string + blob_url?: string + }> +} + +const GitHubLatestCommitSchema = z.object({ + owner: z.string().min(1, 'Owner is required'), + repo: z.string().min(1, 'Repo is required'), + branch: z.string().optional().nullable(), + apiKey: z.string().min(1, 'API key is required'), +}) + +export async function POST(request: NextRequest) { + const requestId = generateRequestId() + + try { + const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) + + if (!authResult.success) { + logger.warn(`[${requestId}] Unauthorized GitHub latest commit attempt: ${authResult.error}`) + return NextResponse.json( + { + success: false, + error: authResult.error || 'Authentication required', + }, + { status: 401 } + ) + } + + const body = await request.json() + const validatedData = GitHubLatestCommitSchema.parse(body) + + const { owner, repo, branch, apiKey } = validatedData + + const baseUrl = `https://api.github.com/repos/${owner}/${repo}` + const commitUrl = branch ? `${baseUrl}/commits/${branch}` : `${baseUrl}/commits/HEAD` + + logger.info(`[${requestId}] Fetching latest commit from GitHub`, { owner, repo, branch }) + + const urlValidation = await validateUrlWithDNS(commitUrl, 'commitUrl') + if (!urlValidation.isValid) { + return NextResponse.json({ success: false, error: urlValidation.error }, { status: 400 }) + } + + const response = await secureFetchWithPinnedIP(commitUrl, urlValidation.resolvedIP!, { + method: 'GET', + headers: { + Accept: 'application/vnd.github.v3+json', + Authorization: `Bearer ${apiKey}`, + 'X-GitHub-Api-Version': '2022-11-28', + }, + }) + + if (!response.ok) { + const errorData = (await response.json().catch(() => ({}))) as GitHubErrorResponse + logger.error(`[${requestId}] GitHub API error`, { + status: response.status, + error: errorData, + }) + return NextResponse.json( + { success: false, error: errorData.message || `GitHub API error: ${response.status}` }, + { status: 400 } + ) + } + + const data = (await response.json()) as GitHubCommitResponse + + const content = `Latest commit: "${data.commit.message}" by ${data.commit.author.name} on ${data.commit.author.date}. SHA: ${data.sha}` + + const files = data.files || [] + const fileDetailsWithContent = [] + + for (const file of files) { + const fileDetail: Record = { + filename: file.filename, + additions: file.additions, + deletions: file.deletions, + changes: file.changes, + status: file.status, + raw_url: file.raw_url, + blob_url: file.blob_url, + patch: file.patch, + content: undefined, + } + + if (file.status !== 'removed' && file.raw_url) { + try { + const rawUrlValidation = await validateUrlWithDNS(file.raw_url, 'rawUrl') + if (rawUrlValidation.isValid) { + const contentResponse = await secureFetchWithPinnedIP( + file.raw_url, + rawUrlValidation.resolvedIP!, + { + headers: { + Authorization: `Bearer ${apiKey}`, + 'X-GitHub-Api-Version': '2022-11-28', + }, + } + ) + + if (contentResponse.ok) { + fileDetail.content = await contentResponse.text() + } + } + } catch (error) { + logger.warn(`[${requestId}] Failed to fetch content for ${file.filename}:`, error) + } + } + + fileDetailsWithContent.push(fileDetail) + } + + logger.info(`[${requestId}] Latest commit fetched successfully`, { + sha: data.sha, + fileCount: files.length, + }) + + return NextResponse.json({ + success: true, + output: { + content, + metadata: { + sha: data.sha, + html_url: data.html_url, + commit_message: data.commit.message, + author: { + name: data.commit.author.name, + login: data.author?.login || 'Unknown', + avatar_url: data.author?.avatar_url || '', + html_url: data.author?.html_url || '', + }, + committer: { + name: data.commit.committer.name, + login: data.committer?.login || 'Unknown', + avatar_url: data.committer?.avatar_url || '', + html_url: data.committer?.html_url || '', + }, + stats: data.stats + ? { + additions: data.stats.additions, + deletions: data.stats.deletions, + total: data.stats.total, + } + : undefined, + files: fileDetailsWithContent.length > 0 ? fileDetailsWithContent : undefined, + }, + }, + }) + } catch (error) { + logger.error(`[${requestId}] Error fetching GitHub latest commit:`, error) + return NextResponse.json( + { + success: false, + error: error instanceof Error ? error.message : 'Unknown error occurred', + }, + { status: 500 } + ) + } +} diff --git a/apps/sim/app/api/tools/gmail/draft/route.ts b/apps/sim/app/api/tools/gmail/draft/route.ts index 627ab0ad4..7a6c6cf0c 100644 --- a/apps/sim/app/api/tools/gmail/draft/route.ts +++ b/apps/sim/app/api/tools/gmail/draft/route.ts @@ -3,6 +3,7 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { RawFileInputArraySchema } from '@/lib/uploads/utils/file-schemas' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { @@ -28,7 +29,7 @@ const GmailDraftSchema = z.object({ replyToMessageId: z.string().optional().nullable(), cc: z.string().optional().nullable(), bcc: z.string().optional().nullable(), - attachments: z.array(z.any()).optional().nullable(), + attachments: RawFileInputArraySchema.optional().nullable(), }) export async function POST(request: NextRequest) { diff --git a/apps/sim/app/api/tools/gmail/send/route.ts b/apps/sim/app/api/tools/gmail/send/route.ts index 535587aa0..26c0ce3f7 100644 --- a/apps/sim/app/api/tools/gmail/send/route.ts +++ b/apps/sim/app/api/tools/gmail/send/route.ts @@ -3,6 +3,7 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { RawFileInputArraySchema } from '@/lib/uploads/utils/file-schemas' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { @@ -28,7 +29,7 @@ const GmailSendSchema = z.object({ replyToMessageId: z.string().optional().nullable(), cc: z.string().optional().nullable(), bcc: z.string().optional().nullable(), - attachments: z.array(z.any()).optional().nullable(), + attachments: RawFileInputArraySchema.optional().nullable(), }) export async function POST(request: NextRequest) { diff --git a/apps/sim/app/api/tools/google_drive/download/route.ts b/apps/sim/app/api/tools/google_drive/download/route.ts new file mode 100644 index 000000000..2a9730dca --- /dev/null +++ b/apps/sim/app/api/tools/google_drive/download/route.ts @@ -0,0 +1,252 @@ +import { createLogger } from '@sim/logger' +import { type NextRequest, NextResponse } from 'next/server' +import { z } from 'zod' +import { checkInternalAuth } from '@/lib/auth/hybrid' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { generateRequestId } from '@/lib/core/utils/request' +import type { GoogleDriveFile, GoogleDriveRevision } from '@/tools/google_drive/types' +import { + ALL_FILE_FIELDS, + ALL_REVISION_FIELDS, + DEFAULT_EXPORT_FORMATS, + GOOGLE_WORKSPACE_MIME_TYPES, +} from '@/tools/google_drive/utils' + +export const dynamic = 'force-dynamic' + +const logger = createLogger('GoogleDriveDownloadAPI') + +/** Google API error response structure */ +interface GoogleApiErrorResponse { + error?: { + message?: string + code?: number + status?: string + } +} + +/** Google Drive revisions list response */ +interface GoogleDriveRevisionsResponse { + revisions?: GoogleDriveRevision[] + nextPageToken?: string +} + +const GoogleDriveDownloadSchema = z.object({ + accessToken: z.string().min(1, 'Access token is required'), + fileId: z.string().min(1, 'File ID is required'), + mimeType: z.string().optional().nullable(), + fileName: z.string().optional().nullable(), + includeRevisions: z.boolean().optional().default(true), +}) + +export async function POST(request: NextRequest) { + const requestId = generateRequestId() + + try { + const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) + + if (!authResult.success) { + logger.warn(`[${requestId}] Unauthorized Google Drive download attempt: ${authResult.error}`) + return NextResponse.json( + { + success: false, + error: authResult.error || 'Authentication required', + }, + { status: 401 } + ) + } + + const body = await request.json() + const validatedData = GoogleDriveDownloadSchema.parse(body) + + const { + accessToken, + fileId, + mimeType: exportMimeType, + fileName, + includeRevisions, + } = validatedData + const authHeader = `Bearer ${accessToken}` + + logger.info(`[${requestId}] Getting file metadata from Google Drive`, { fileId }) + + const metadataUrl = `https://www.googleapis.com/drive/v3/files/${fileId}?fields=${ALL_FILE_FIELDS}&supportsAllDrives=true` + const metadataUrlValidation = await validateUrlWithDNS(metadataUrl, 'metadataUrl') + if (!metadataUrlValidation.isValid) { + return NextResponse.json( + { success: false, error: metadataUrlValidation.error }, + { status: 400 } + ) + } + + const metadataResponse = await secureFetchWithPinnedIP( + metadataUrl, + metadataUrlValidation.resolvedIP!, + { + headers: { Authorization: authHeader }, + } + ) + + if (!metadataResponse.ok) { + const errorDetails = (await metadataResponse + .json() + .catch(() => ({}))) as GoogleApiErrorResponse + logger.error(`[${requestId}] Failed to get file metadata`, { + status: metadataResponse.status, + error: errorDetails, + }) + return NextResponse.json( + { success: false, error: errorDetails.error?.message || 'Failed to get file metadata' }, + { status: 400 } + ) + } + + const metadata = (await metadataResponse.json()) as GoogleDriveFile + const fileMimeType = metadata.mimeType + + let fileBuffer: Buffer + let finalMimeType = fileMimeType + + if (GOOGLE_WORKSPACE_MIME_TYPES.includes(fileMimeType)) { + const exportFormat = exportMimeType || DEFAULT_EXPORT_FORMATS[fileMimeType] || 'text/plain' + finalMimeType = exportFormat + + logger.info(`[${requestId}] Exporting Google Workspace file`, { + fileId, + mimeType: fileMimeType, + exportFormat, + }) + + const exportUrl = `https://www.googleapis.com/drive/v3/files/${fileId}/export?mimeType=${encodeURIComponent(exportFormat)}&supportsAllDrives=true` + const exportUrlValidation = await validateUrlWithDNS(exportUrl, 'exportUrl') + if (!exportUrlValidation.isValid) { + return NextResponse.json( + { success: false, error: exportUrlValidation.error }, + { status: 400 } + ) + } + + const exportResponse = await secureFetchWithPinnedIP( + exportUrl, + exportUrlValidation.resolvedIP!, + { headers: { Authorization: authHeader } } + ) + + if (!exportResponse.ok) { + const exportError = (await exportResponse + .json() + .catch(() => ({}))) as GoogleApiErrorResponse + logger.error(`[${requestId}] Failed to export file`, { + status: exportResponse.status, + error: exportError, + }) + return NextResponse.json( + { + success: false, + error: exportError.error?.message || 'Failed to export Google Workspace file', + }, + { status: 400 } + ) + } + + const arrayBuffer = await exportResponse.arrayBuffer() + fileBuffer = Buffer.from(arrayBuffer) + } else { + logger.info(`[${requestId}] Downloading regular file`, { fileId, mimeType: fileMimeType }) + + const downloadUrl = `https://www.googleapis.com/drive/v3/files/${fileId}?alt=media&supportsAllDrives=true` + const downloadUrlValidation = await validateUrlWithDNS(downloadUrl, 'downloadUrl') + if (!downloadUrlValidation.isValid) { + return NextResponse.json( + { success: false, error: downloadUrlValidation.error }, + { status: 400 } + ) + } + + const downloadResponse = await secureFetchWithPinnedIP( + downloadUrl, + downloadUrlValidation.resolvedIP!, + { headers: { Authorization: authHeader } } + ) + + if (!downloadResponse.ok) { + const downloadError = (await downloadResponse + .json() + .catch(() => ({}))) as GoogleApiErrorResponse + logger.error(`[${requestId}] Failed to download file`, { + status: downloadResponse.status, + error: downloadError, + }) + return NextResponse.json( + { success: false, error: downloadError.error?.message || 'Failed to download file' }, + { status: 400 } + ) + } + + const arrayBuffer = await downloadResponse.arrayBuffer() + fileBuffer = Buffer.from(arrayBuffer) + } + + const canReadRevisions = metadata.capabilities?.canReadRevisions === true + if (includeRevisions && canReadRevisions) { + try { + const revisionsUrl = `https://www.googleapis.com/drive/v3/files/${fileId}/revisions?fields=revisions(${ALL_REVISION_FIELDS})&pageSize=100` + const revisionsUrlValidation = await validateUrlWithDNS(revisionsUrl, 'revisionsUrl') + if (revisionsUrlValidation.isValid) { + const revisionsResponse = await secureFetchWithPinnedIP( + revisionsUrl, + revisionsUrlValidation.resolvedIP!, + { headers: { Authorization: authHeader } } + ) + + if (revisionsResponse.ok) { + const revisionsData = (await revisionsResponse.json()) as GoogleDriveRevisionsResponse + metadata.revisions = revisionsData.revisions + logger.info(`[${requestId}] Fetched file revisions`, { + fileId, + revisionCount: metadata.revisions?.length || 0, + }) + } + } + } catch (error) { + logger.warn(`[${requestId}] Error fetching revisions, continuing without them`, { error }) + } + } + + const resolvedName = fileName || metadata.name || 'download' + + logger.info(`[${requestId}] File downloaded successfully`, { + fileId, + name: resolvedName, + size: fileBuffer.length, + mimeType: finalMimeType, + }) + + const base64Data = fileBuffer.toString('base64') + + return NextResponse.json({ + success: true, + output: { + file: { + name: resolvedName, + mimeType: finalMimeType, + data: base64Data, + size: fileBuffer.length, + }, + metadata, + }, + }) + } catch (error) { + logger.error(`[${requestId}] Error downloading Google Drive file:`, error) + return NextResponse.json( + { + success: false, + error: error instanceof Error ? error.message : 'Unknown error occurred', + }, + { status: 500 } + ) + } +} diff --git a/apps/sim/app/api/tools/google_drive/upload/route.ts b/apps/sim/app/api/tools/google_drive/upload/route.ts index 9cf53e41d..3549245fd 100644 --- a/apps/sim/app/api/tools/google_drive/upload/route.ts +++ b/apps/sim/app/api/tools/google_drive/upload/route.ts @@ -3,6 +3,7 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { @@ -20,7 +21,7 @@ const GOOGLE_DRIVE_API_BASE = 'https://www.googleapis.com/upload/drive/v3/files' const GoogleDriveUploadSchema = z.object({ accessToken: z.string().min(1, 'Access token is required'), fileName: z.string().min(1, 'File name is required'), - file: z.any().optional().nullable(), + file: RawFileInputSchema.optional().nullable(), mimeType: z.string().optional().nullable(), folderId: z.string().optional().nullable(), }) diff --git a/apps/sim/app/api/tools/google_vault/download-export-file/route.ts b/apps/sim/app/api/tools/google_vault/download-export-file/route.ts new file mode 100644 index 000000000..01bdfd3f5 --- /dev/null +++ b/apps/sim/app/api/tools/google_vault/download-export-file/route.ts @@ -0,0 +1,131 @@ +import { createLogger } from '@sim/logger' +import { type NextRequest, NextResponse } from 'next/server' +import { z } from 'zod' +import { checkInternalAuth } from '@/lib/auth/hybrid' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { generateRequestId } from '@/lib/core/utils/request' +import { enhanceGoogleVaultError } from '@/tools/google_vault/utils' + +export const dynamic = 'force-dynamic' + +const logger = createLogger('GoogleVaultDownloadExportFileAPI') + +const GoogleVaultDownloadExportFileSchema = z.object({ + accessToken: z.string().min(1, 'Access token is required'), + bucketName: z.string().min(1, 'Bucket name is required'), + objectName: z.string().min(1, 'Object name is required'), + fileName: z.string().optional().nullable(), +}) + +export async function POST(request: NextRequest) { + const requestId = generateRequestId() + + try { + const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) + + if (!authResult.success) { + logger.warn(`[${requestId}] Unauthorized Google Vault download attempt: ${authResult.error}`) + return NextResponse.json( + { + success: false, + error: authResult.error || 'Authentication required', + }, + { status: 401 } + ) + } + + const body = await request.json() + const validatedData = GoogleVaultDownloadExportFileSchema.parse(body) + + const { accessToken, bucketName, objectName, fileName } = validatedData + + const bucket = encodeURIComponent(bucketName) + const object = encodeURIComponent(objectName) + const downloadUrl = `https://storage.googleapis.com/storage/v1/b/${bucket}/o/${object}?alt=media` + + logger.info(`[${requestId}] Downloading file from Google Vault`, { bucketName, objectName }) + + const urlValidation = await validateUrlWithDNS(downloadUrl, 'downloadUrl') + if (!urlValidation.isValid) { + return NextResponse.json( + { success: false, error: enhanceGoogleVaultError(urlValidation.error || 'Invalid URL') }, + { status: 400 } + ) + } + + const downloadResponse = await secureFetchWithPinnedIP(downloadUrl, urlValidation.resolvedIP!, { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + }, + }) + + if (!downloadResponse.ok) { + const errorText = await downloadResponse.text().catch(() => '') + const errorMessage = `Failed to download file: ${errorText || downloadResponse.statusText}` + logger.error(`[${requestId}] Failed to download Vault export file`, { + status: downloadResponse.status, + error: errorText, + }) + return NextResponse.json( + { success: false, error: enhanceGoogleVaultError(errorMessage) }, + { status: 400 } + ) + } + + const contentType = downloadResponse.headers.get('content-type') || 'application/octet-stream' + const disposition = downloadResponse.headers.get('content-disposition') || '' + const match = disposition.match(/filename\*=UTF-8''([^;]+)|filename="([^"]+)"/) + + let resolvedName = fileName + if (!resolvedName) { + if (match?.[1]) { + try { + resolvedName = decodeURIComponent(match[1]) + } catch { + resolvedName = match[1] + } + } else if (match?.[2]) { + resolvedName = match[2] + } else if (objectName) { + const parts = objectName.split('/') + resolvedName = parts[parts.length - 1] || 'vault-export.bin' + } else { + resolvedName = 'vault-export.bin' + } + } + + const arrayBuffer = await downloadResponse.arrayBuffer() + const buffer = Buffer.from(arrayBuffer) + + logger.info(`[${requestId}] Vault export file downloaded successfully`, { + name: resolvedName, + size: buffer.length, + mimeType: contentType, + }) + + return NextResponse.json({ + success: true, + output: { + file: { + name: resolvedName, + mimeType: contentType, + data: buffer.toString('base64'), + size: buffer.length, + }, + }, + }) + } catch (error) { + logger.error(`[${requestId}] Error downloading Google Vault export file:`, error) + return NextResponse.json( + { + success: false, + error: error instanceof Error ? error.message : 'Unknown error occurred', + }, + { status: 500 } + ) + } +} diff --git a/apps/sim/app/api/tools/image/route.ts b/apps/sim/app/api/tools/image/route.ts index 633e61068..475a9de5c 100644 --- a/apps/sim/app/api/tools/image/route.ts +++ b/apps/sim/app/api/tools/image/route.ts @@ -1,7 +1,10 @@ import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateImageUrl } from '@/lib/core/security/input-validation' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' const logger = createLogger('ImageProxyAPI') @@ -26,7 +29,7 @@ export async function GET(request: NextRequest) { return new NextResponse('Missing URL parameter', { status: 400 }) } - const urlValidation = validateImageUrl(imageUrl) + const urlValidation = await validateUrlWithDNS(imageUrl, 'imageUrl') if (!urlValidation.isValid) { logger.warn(`[${requestId}] Blocked image proxy request`, { url: imageUrl.substring(0, 100), @@ -38,7 +41,8 @@ export async function GET(request: NextRequest) { logger.info(`[${requestId}] Proxying image request for: ${imageUrl}`) try { - const imageResponse = await fetch(imageUrl, { + const imageResponse = await secureFetchWithPinnedIP(imageUrl, urlValidation.resolvedIP!, { + method: 'GET', headers: { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', @@ -64,14 +68,14 @@ export async function GET(request: NextRequest) { const contentType = imageResponse.headers.get('content-type') || 'image/jpeg' - const imageBlob = await imageResponse.blob() + const imageArrayBuffer = await imageResponse.arrayBuffer() - if (imageBlob.size === 0) { - logger.error(`[${requestId}] Empty image blob received`) + if (imageArrayBuffer.byteLength === 0) { + logger.error(`[${requestId}] Empty image received`) return new NextResponse('Empty image received', { status: 404 }) } - return new NextResponse(imageBlob, { + return new NextResponse(imageArrayBuffer, { headers: { 'Content-Type': contentType, 'Access-Control-Allow-Origin': '*', diff --git a/apps/sim/app/api/tools/jira/add-attachment/route.ts b/apps/sim/app/api/tools/jira/add-attachment/route.ts new file mode 100644 index 000000000..52b36b24a --- /dev/null +++ b/apps/sim/app/api/tools/jira/add-attachment/route.ts @@ -0,0 +1,121 @@ +import { createLogger } from '@sim/logger' +import { type NextRequest, NextResponse } from 'next/server' +import { z } from 'zod' +import { checkInternalAuth } from '@/lib/auth/hybrid' +import { RawFileInputArraySchema } from '@/lib/uploads/utils/file-schemas' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { getJiraCloudId } from '@/tools/jira/utils' + +const logger = createLogger('JiraAddAttachmentAPI') + +export const dynamic = 'force-dynamic' + +const JiraAddAttachmentSchema = z.object({ + accessToken: z.string().min(1, 'Access token is required'), + domain: z.string().min(1, 'Domain is required'), + issueKey: z.string().min(1, 'Issue key is required'), + files: RawFileInputArraySchema, + cloudId: z.string().optional().nullable(), +}) + +export async function POST(request: NextRequest) { + const requestId = `jira-attach-${Date.now()}` + + try { + const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) + if (!authResult.success) { + return NextResponse.json( + { success: false, error: authResult.error || 'Unauthorized' }, + { status: 401 } + ) + } + + const body = await request.json() + const validatedData = JiraAddAttachmentSchema.parse(body) + + const userFiles = processFilesToUserFiles(validatedData.files, requestId, logger) + if (userFiles.length === 0) { + return NextResponse.json( + { success: false, error: 'No valid files provided for upload' }, + { status: 400 } + ) + } + + const cloudId = + validatedData.cloudId || + (await getJiraCloudId(validatedData.domain, validatedData.accessToken)) + + const formData = new FormData() + const filesOutput: Array<{ name: string; mimeType: string; data: string; size: number }> = [] + + for (const file of userFiles) { + const buffer = await downloadFileFromStorage(file, requestId, logger) + filesOutput.push({ + name: file.name, + mimeType: file.type || 'application/octet-stream', + data: buffer.toString('base64'), + size: buffer.length, + }) + const blob = new Blob([new Uint8Array(buffer)], { + type: file.type || 'application/octet-stream', + }) + formData.append('file', blob, file.name) + } + + const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${validatedData.issueKey}/attachments` + + const response = await fetch(url, { + method: 'POST', + headers: { + Authorization: `Bearer ${validatedData.accessToken}`, + 'X-Atlassian-Token': 'no-check', + }, + body: formData, + }) + + if (!response.ok) { + const errorText = await response.text() + logger.error(`[${requestId}] Jira attachment upload failed`, { + status: response.status, + statusText: response.statusText, + error: errorText, + }) + return NextResponse.json( + { + success: false, + error: `Failed to upload attachments: ${response.statusText}`, + }, + { status: response.status } + ) + } + + const attachments = await response.json() + const attachmentIds = Array.isArray(attachments) + ? attachments.map((attachment) => attachment.id).filter(Boolean) + : [] + + return NextResponse.json({ + success: true, + output: { + ts: new Date().toISOString(), + issueKey: validatedData.issueKey, + attachmentIds, + files: filesOutput, + }, + }) + } catch (error) { + if (error instanceof z.ZodError) { + return NextResponse.json( + { success: false, error: 'Invalid request data', details: error.errors }, + { status: 400 } + ) + } + + logger.error(`[${requestId}] Jira attachment upload error`, error) + return NextResponse.json( + { success: false, error: error instanceof Error ? error.message : 'Internal server error' }, + { status: 500 } + ) + } +} diff --git a/apps/sim/app/api/tools/microsoft_teams/write_channel/route.ts b/apps/sim/app/api/tools/microsoft_teams/write_channel/route.ts index dcaa0f738..a477f68d8 100644 --- a/apps/sim/app/api/tools/microsoft_teams/write_channel/route.ts +++ b/apps/sim/app/api/tools/microsoft_teams/write_channel/route.ts @@ -2,9 +2,11 @@ import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkInternalAuth } from '@/lib/auth/hybrid' +import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' -import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { RawFileInputArraySchema } from '@/lib/uploads/utils/file-schemas' +import { uploadFilesForTeamsMessage } from '@/tools/microsoft_teams/server-utils' +import type { GraphApiErrorResponse, GraphChatMessage } from '@/tools/microsoft_teams/types' import { resolveMentionsForChannel, type TeamsMention } from '@/tools/microsoft_teams/utils' export const dynamic = 'force-dynamic' @@ -16,7 +18,7 @@ const TeamsWriteChannelSchema = z.object({ teamId: z.string().min(1, 'Team ID is required'), channelId: z.string().min(1, 'Channel ID is required'), content: z.string().min(1, 'Message content is required'), - files: z.array(z.any()).optional().nullable(), + files: RawFileInputArraySchema.optional().nullable(), }) export async function POST(request: NextRequest) { @@ -53,93 +55,12 @@ export async function POST(request: NextRequest) { fileCount: validatedData.files?.length || 0, }) - const attachments: any[] = [] - if (validatedData.files && validatedData.files.length > 0) { - const rawFiles = validatedData.files - logger.info(`[${requestId}] Processing ${rawFiles.length} file(s) for upload to OneDrive`) - - const userFiles = processFilesToUserFiles(rawFiles, requestId, logger) - - for (const file of userFiles) { - try { - logger.info(`[${requestId}] Uploading file to Teams: ${file.name} (${file.size} bytes)`) - - const buffer = await downloadFileFromStorage(file, requestId, logger) - - const uploadUrl = - 'https://graph.microsoft.com/v1.0/me/drive/root:/TeamsAttachments/' + - encodeURIComponent(file.name) + - ':/content' - - logger.info(`[${requestId}] Uploading to Teams: ${uploadUrl}`) - - const uploadResponse = await fetch(uploadUrl, { - method: 'PUT', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'Content-Type': file.type || 'application/octet-stream', - }, - body: new Uint8Array(buffer), - }) - - if (!uploadResponse.ok) { - const errorData = await uploadResponse.json().catch(() => ({})) - logger.error(`[${requestId}] Teams upload failed:`, errorData) - throw new Error( - `Failed to upload file to Teams: ${errorData.error?.message || 'Unknown error'}` - ) - } - - const uploadedFile = await uploadResponse.json() - logger.info(`[${requestId}] File uploaded to Teams successfully`, { - id: uploadedFile.id, - webUrl: uploadedFile.webUrl, - }) - - const fileDetailsUrl = `https://graph.microsoft.com/v1.0/me/drive/items/${uploadedFile.id}?$select=id,name,webDavUrl,eTag,size` - - const fileDetailsResponse = await fetch(fileDetailsUrl, { - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - }, - }) - - if (!fileDetailsResponse.ok) { - const errorData = await fileDetailsResponse.json().catch(() => ({})) - logger.error(`[${requestId}] Failed to get file details:`, errorData) - throw new Error( - `Failed to get file details: ${errorData.error?.message || 'Unknown error'}` - ) - } - - const fileDetails = await fileDetailsResponse.json() - logger.info(`[${requestId}] Got file details`, { - webDavUrl: fileDetails.webDavUrl, - eTag: fileDetails.eTag, - }) - - const attachmentId = fileDetails.eTag?.match(/\{([a-f0-9-]+)\}/i)?.[1] || fileDetails.id - - attachments.push({ - id: attachmentId, - contentType: 'reference', - contentUrl: fileDetails.webDavUrl, - name: file.name, - }) - - logger.info(`[${requestId}] Created attachment reference for ${file.name}`) - } catch (error) { - logger.error(`[${requestId}] Failed to process file ${file.name}:`, error) - throw new Error( - `Failed to process file "${file.name}": ${error instanceof Error ? error.message : 'Unknown error'}` - ) - } - } - - logger.info( - `[${requestId}] All ${attachments.length} file(s) uploaded and attachment references created` - ) - } + const { attachments, filesOutput } = await uploadFilesForTeamsMessage({ + rawFiles: validatedData.files || [], + accessToken: validatedData.accessToken, + requestId, + logger, + }) let messageContent = validatedData.content let contentType: 'text' | 'html' = 'text' @@ -197,17 +118,21 @@ export async function POST(request: NextRequest) { const teamsUrl = `https://graph.microsoft.com/v1.0/teams/${encodeURIComponent(validatedData.teamId)}/channels/${encodeURIComponent(validatedData.channelId)}/messages` - const teamsResponse = await fetch(teamsUrl, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${validatedData.accessToken}`, + const teamsResponse = await secureFetchWithValidation( + teamsUrl, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${validatedData.accessToken}`, + }, + body: JSON.stringify(messageBody), }, - body: JSON.stringify(messageBody), - }) + 'teamsUrl' + ) if (!teamsResponse.ok) { - const errorData = await teamsResponse.json().catch(() => ({})) + const errorData = (await teamsResponse.json().catch(() => ({}))) as GraphApiErrorResponse logger.error(`[${requestId}] Microsoft Teams API error:`, errorData) return NextResponse.json( { @@ -218,7 +143,7 @@ export async function POST(request: NextRequest) { ) } - const responseData = await teamsResponse.json() + const responseData = (await teamsResponse.json()) as GraphChatMessage logger.info(`[${requestId}] Teams channel message sent successfully`, { messageId: responseData.id, attachmentCount: attachments.length, @@ -237,6 +162,7 @@ export async function POST(request: NextRequest) { url: responseData.webUrl || '', attachmentCount: attachments.length, }, + files: filesOutput, }, }) } catch (error) { diff --git a/apps/sim/app/api/tools/microsoft_teams/write_chat/route.ts b/apps/sim/app/api/tools/microsoft_teams/write_chat/route.ts index 14454fafa..67df1e402 100644 --- a/apps/sim/app/api/tools/microsoft_teams/write_chat/route.ts +++ b/apps/sim/app/api/tools/microsoft_teams/write_chat/route.ts @@ -2,9 +2,11 @@ import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkInternalAuth } from '@/lib/auth/hybrid' +import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' -import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { RawFileInputArraySchema } from '@/lib/uploads/utils/file-schemas' +import { uploadFilesForTeamsMessage } from '@/tools/microsoft_teams/server-utils' +import type { GraphApiErrorResponse, GraphChatMessage } from '@/tools/microsoft_teams/types' import { resolveMentionsForChat, type TeamsMention } from '@/tools/microsoft_teams/utils' export const dynamic = 'force-dynamic' @@ -15,7 +17,7 @@ const TeamsWriteChatSchema = z.object({ accessToken: z.string().min(1, 'Access token is required'), chatId: z.string().min(1, 'Chat ID is required'), content: z.string().min(1, 'Message content is required'), - files: z.array(z.any()).optional().nullable(), + files: RawFileInputArraySchema.optional().nullable(), }) export async function POST(request: NextRequest) { @@ -51,93 +53,12 @@ export async function POST(request: NextRequest) { fileCount: validatedData.files?.length || 0, }) - const attachments: any[] = [] - if (validatedData.files && validatedData.files.length > 0) { - const rawFiles = validatedData.files - logger.info(`[${requestId}] Processing ${rawFiles.length} file(s) for upload to Teams`) - - const userFiles = processFilesToUserFiles(rawFiles, requestId, logger) - - for (const file of userFiles) { - try { - logger.info(`[${requestId}] Uploading file to Teams: ${file.name} (${file.size} bytes)`) - - const buffer = await downloadFileFromStorage(file, requestId, logger) - - const uploadUrl = - 'https://graph.microsoft.com/v1.0/me/drive/root:/TeamsAttachments/' + - encodeURIComponent(file.name) + - ':/content' - - logger.info(`[${requestId}] Uploading to Teams: ${uploadUrl}`) - - const uploadResponse = await fetch(uploadUrl, { - method: 'PUT', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'Content-Type': file.type || 'application/octet-stream', - }, - body: new Uint8Array(buffer), - }) - - if (!uploadResponse.ok) { - const errorData = await uploadResponse.json().catch(() => ({})) - logger.error(`[${requestId}] Teams upload failed:`, errorData) - throw new Error( - `Failed to upload file to Teams: ${errorData.error?.message || 'Unknown error'}` - ) - } - - const uploadedFile = await uploadResponse.json() - logger.info(`[${requestId}] File uploaded to Teams successfully`, { - id: uploadedFile.id, - webUrl: uploadedFile.webUrl, - }) - - const fileDetailsUrl = `https://graph.microsoft.com/v1.0/me/drive/items/${uploadedFile.id}?$select=id,name,webDavUrl,eTag,size` - - const fileDetailsResponse = await fetch(fileDetailsUrl, { - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - }, - }) - - if (!fileDetailsResponse.ok) { - const errorData = await fileDetailsResponse.json().catch(() => ({})) - logger.error(`[${requestId}] Failed to get file details:`, errorData) - throw new Error( - `Failed to get file details: ${errorData.error?.message || 'Unknown error'}` - ) - } - - const fileDetails = await fileDetailsResponse.json() - logger.info(`[${requestId}] Got file details`, { - webDavUrl: fileDetails.webDavUrl, - eTag: fileDetails.eTag, - }) - - const attachmentId = fileDetails.eTag?.match(/\{([a-f0-9-]+)\}/i)?.[1] || fileDetails.id - - attachments.push({ - id: attachmentId, - contentType: 'reference', - contentUrl: fileDetails.webDavUrl, - name: file.name, - }) - - logger.info(`[${requestId}] Created attachment reference for ${file.name}`) - } catch (error) { - logger.error(`[${requestId}] Failed to process file ${file.name}:`, error) - throw new Error( - `Failed to process file "${file.name}": ${error instanceof Error ? error.message : 'Unknown error'}` - ) - } - } - - logger.info( - `[${requestId}] All ${attachments.length} file(s) uploaded and attachment references created` - ) - } + const { attachments, filesOutput } = await uploadFilesForTeamsMessage({ + rawFiles: validatedData.files || [], + accessToken: validatedData.accessToken, + requestId, + logger, + }) let messageContent = validatedData.content let contentType: 'text' | 'html' = 'text' @@ -194,17 +115,21 @@ export async function POST(request: NextRequest) { const teamsUrl = `https://graph.microsoft.com/v1.0/chats/${encodeURIComponent(validatedData.chatId)}/messages` - const teamsResponse = await fetch(teamsUrl, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${validatedData.accessToken}`, + const teamsResponse = await secureFetchWithValidation( + teamsUrl, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${validatedData.accessToken}`, + }, + body: JSON.stringify(messageBody), }, - body: JSON.stringify(messageBody), - }) + 'teamsUrl' + ) if (!teamsResponse.ok) { - const errorData = await teamsResponse.json().catch(() => ({})) + const errorData = (await teamsResponse.json().catch(() => ({}))) as GraphApiErrorResponse logger.error(`[${requestId}] Microsoft Teams API error:`, errorData) return NextResponse.json( { @@ -215,7 +140,7 @@ export async function POST(request: NextRequest) { ) } - const responseData = await teamsResponse.json() + const responseData = (await teamsResponse.json()) as GraphChatMessage logger.info(`[${requestId}] Teams message sent successfully`, { messageId: responseData.id, attachmentCount: attachments.length, @@ -233,6 +158,7 @@ export async function POST(request: NextRequest) { url: responseData.webUrl || '', attachmentCount: attachments.length, }, + files: filesOutput, }, }) } catch (error) { diff --git a/apps/sim/app/api/tools/mistral/parse/route.ts b/apps/sim/app/api/tools/mistral/parse/route.ts index a40e5d502..f8b0c1191 100644 --- a/apps/sim/app/api/tools/mistral/parse/route.ts +++ b/apps/sim/app/api/tools/mistral/parse/route.ts @@ -2,15 +2,17 @@ import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { getBaseUrl } from '@/lib/core/utils/urls' -import { StorageService } from '@/lib/uploads' import { - extractStorageKey, - inferContextFromKey, - isInternalFileUrl, -} from '@/lib/uploads/utils/file-utils' -import { verifyFileAccess } from '@/app/api/files/authorization' + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { generateRequestId } from '@/lib/core/utils/request' +import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' +import { isInternalFileUrl, processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' +import { + downloadFileFromStorage, + resolveInternalFileUrl, +} from '@/lib/uploads/utils/file-utils.server' export const dynamic = 'force-dynamic' @@ -18,7 +20,9 @@ const logger = createLogger('MistralParseAPI') const MistralParseSchema = z.object({ apiKey: z.string().min(1, 'API key is required'), - filePath: z.string().min(1, 'File path is required'), + filePath: z.string().min(1, 'File path is required').optional(), + fileData: FileInputSchema.optional(), + file: FileInputSchema.optional(), resultType: z.string().optional(), pages: z.array(z.number()).optional(), includeImageBase64: z.boolean().optional(), @@ -49,66 +53,140 @@ export async function POST(request: NextRequest) { const body = await request.json() const validatedData = MistralParseSchema.parse(body) + const fileData = validatedData.file || validatedData.fileData + const filePath = typeof fileData === 'string' ? fileData : validatedData.filePath + + if (!fileData && (!filePath || filePath.trim() === '')) { + return NextResponse.json( + { + success: false, + error: 'File input is required', + }, + { status: 400 } + ) + } + logger.info(`[${requestId}] Mistral parse request`, { - filePath: validatedData.filePath, - isWorkspaceFile: isInternalFileUrl(validatedData.filePath), + hasFileData: Boolean(fileData), + filePath, + isWorkspaceFile: filePath ? isInternalFileUrl(filePath) : false, userId, }) - let fileUrl = validatedData.filePath + const mistralBody: any = { + model: 'mistral-ocr-latest', + } - if (isInternalFileUrl(validatedData.filePath)) { + if (fileData && typeof fileData === 'object') { + const rawFile = fileData + let userFile try { - const storageKey = extractStorageKey(validatedData.filePath) - - const context = inferContextFromKey(storageKey) - - const hasAccess = await verifyFileAccess( - storageKey, - userId, - undefined, // customConfig - context, // context - false // isLocal - ) - - if (!hasAccess) { - logger.warn(`[${requestId}] Unauthorized presigned URL generation attempt`, { - userId, - key: storageKey, - context, - }) - return NextResponse.json( - { - success: false, - error: 'File not found', - }, - { status: 404 } - ) - } - - fileUrl = await StorageService.generatePresignedDownloadUrl(storageKey, context, 5 * 60) - logger.info(`[${requestId}] Generated presigned URL for ${context} file`) + userFile = processSingleFileToUserFile(rawFile, requestId, logger) } catch (error) { - logger.error(`[${requestId}] Failed to generate presigned URL:`, error) return NextResponse.json( { success: false, - error: 'Failed to generate file access URL', + error: error instanceof Error ? error.message : 'Failed to process file', }, - { status: 500 } + { status: 400 } ) } - } else if (validatedData.filePath?.startsWith('/')) { - const baseUrl = getBaseUrl() - fileUrl = `${baseUrl}${validatedData.filePath}` - } - const mistralBody: any = { - model: 'mistral-ocr-latest', - document: { - type: 'document_url', - document_url: fileUrl, - }, + let mimeType = userFile.type + if (!mimeType || mimeType === 'application/octet-stream') { + const filename = userFile.name?.toLowerCase() || '' + if (filename.endsWith('.pdf')) { + mimeType = 'application/pdf' + } else if (filename.endsWith('.png')) { + mimeType = 'image/png' + } else if (filename.endsWith('.jpg') || filename.endsWith('.jpeg')) { + mimeType = 'image/jpeg' + } else if (filename.endsWith('.gif')) { + mimeType = 'image/gif' + } else if (filename.endsWith('.webp')) { + mimeType = 'image/webp' + } else { + mimeType = 'application/pdf' + } + } + let base64 = userFile.base64 + if (!base64) { + const buffer = await downloadFileFromStorage(userFile, requestId, logger) + base64 = buffer.toString('base64') + } + const base64Payload = base64.startsWith('data:') + ? base64 + : `data:${mimeType};base64,${base64}` + + // Mistral API uses different document types for images vs documents + const isImage = mimeType.startsWith('image/') + if (isImage) { + mistralBody.document = { + type: 'image_url', + image_url: base64Payload, + } + } else { + mistralBody.document = { + type: 'document_url', + document_url: base64Payload, + } + } + } else if (filePath) { + let fileUrl = filePath + + const isInternalFilePath = isInternalFileUrl(filePath) + if (isInternalFilePath) { + const resolution = await resolveInternalFileUrl(filePath, userId, requestId, logger) + if (resolution.error) { + return NextResponse.json( + { + success: false, + error: resolution.error.message, + }, + { status: resolution.error.status } + ) + } + fileUrl = resolution.fileUrl || fileUrl + } else if (filePath.startsWith('/')) { + logger.warn(`[${requestId}] Invalid internal path`, { + userId, + path: filePath.substring(0, 50), + }) + return NextResponse.json( + { + success: false, + error: 'Invalid file path. Only uploaded files are supported for internal paths.', + }, + { status: 400 } + ) + } else { + const urlValidation = await validateUrlWithDNS(fileUrl, 'filePath') + if (!urlValidation.isValid) { + return NextResponse.json( + { + success: false, + error: urlValidation.error, + }, + { status: 400 } + ) + } + } + + const imageExtensions = ['.png', '.jpg', '.jpeg', '.gif', '.webp', '.avif'] + const pathname = new URL(fileUrl).pathname.toLowerCase() + const isImageUrl = imageExtensions.some((ext) => pathname.endsWith(ext)) + + if (isImageUrl) { + mistralBody.document = { + type: 'image_url', + image_url: fileUrl, + } + } else { + mistralBody.document = { + type: 'document_url', + document_url: fileUrl, + } + } } if (validatedData.pages) { @@ -124,15 +202,34 @@ export async function POST(request: NextRequest) { mistralBody.image_min_size = validatedData.imageMinSize } - const mistralResponse = await fetch('https://api.mistral.ai/v1/ocr', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Accept: 'application/json', - Authorization: `Bearer ${validatedData.apiKey}`, - }, - body: JSON.stringify(mistralBody), - }) + const mistralEndpoint = 'https://api.mistral.ai/v1/ocr' + const mistralValidation = await validateUrlWithDNS(mistralEndpoint, 'Mistral API URL') + if (!mistralValidation.isValid) { + logger.error(`[${requestId}] Mistral API URL validation failed`, { + error: mistralValidation.error, + }) + return NextResponse.json( + { + success: false, + error: 'Failed to reach Mistral API', + }, + { status: 502 } + ) + } + + const mistralResponse = await secureFetchWithPinnedIP( + mistralEndpoint, + mistralValidation.resolvedIP!, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + Authorization: `Bearer ${validatedData.apiKey}`, + }, + body: JSON.stringify(mistralBody), + } + ) if (!mistralResponse.ok) { const errorText = await mistralResponse.text() diff --git a/apps/sim/app/api/tools/onedrive/download/route.ts b/apps/sim/app/api/tools/onedrive/download/route.ts new file mode 100644 index 000000000..a50338af5 --- /dev/null +++ b/apps/sim/app/api/tools/onedrive/download/route.ts @@ -0,0 +1,177 @@ +import { createLogger } from '@sim/logger' +import { type NextRequest, NextResponse } from 'next/server' +import { z } from 'zod' +import { checkInternalAuth } from '@/lib/auth/hybrid' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { generateRequestId } from '@/lib/core/utils/request' + +export const dynamic = 'force-dynamic' + +/** Microsoft Graph API error response structure */ +interface GraphApiError { + error?: { + code?: string + message?: string + } +} + +/** Microsoft Graph API drive item metadata response */ +interface DriveItemMetadata { + id?: string + name?: string + folder?: Record + file?: { + mimeType?: string + } +} + +const logger = createLogger('OneDriveDownloadAPI') + +const OneDriveDownloadSchema = z.object({ + accessToken: z.string().min(1, 'Access token is required'), + fileId: z.string().min(1, 'File ID is required'), + fileName: z.string().optional().nullable(), +}) + +export async function POST(request: NextRequest) { + const requestId = generateRequestId() + + try { + const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) + + if (!authResult.success) { + logger.warn(`[${requestId}] Unauthorized OneDrive download attempt: ${authResult.error}`) + return NextResponse.json( + { + success: false, + error: authResult.error || 'Authentication required', + }, + { status: 401 } + ) + } + + const body = await request.json() + const validatedData = OneDriveDownloadSchema.parse(body) + + const { accessToken, fileId, fileName } = validatedData + const authHeader = `Bearer ${accessToken}` + + logger.info(`[${requestId}] Getting file metadata from OneDrive`, { fileId }) + + const metadataUrl = `https://graph.microsoft.com/v1.0/me/drive/items/${fileId}` + const metadataUrlValidation = await validateUrlWithDNS(metadataUrl, 'metadataUrl') + if (!metadataUrlValidation.isValid) { + return NextResponse.json( + { success: false, error: metadataUrlValidation.error }, + { status: 400 } + ) + } + + const metadataResponse = await secureFetchWithPinnedIP( + metadataUrl, + metadataUrlValidation.resolvedIP!, + { + headers: { Authorization: authHeader }, + } + ) + + if (!metadataResponse.ok) { + const errorDetails = (await metadataResponse.json().catch(() => ({}))) as GraphApiError + logger.error(`[${requestId}] Failed to get file metadata`, { + status: metadataResponse.status, + error: errorDetails, + }) + return NextResponse.json( + { success: false, error: errorDetails.error?.message || 'Failed to get file metadata' }, + { status: 400 } + ) + } + + const metadata = (await metadataResponse.json()) as DriveItemMetadata + + if (metadata.folder && !metadata.file) { + logger.error(`[${requestId}] Attempted to download a folder`, { + itemId: metadata.id, + itemName: metadata.name, + }) + return NextResponse.json( + { + success: false, + error: `Cannot download folder "${metadata.name}". Please select a file instead.`, + }, + { status: 400 } + ) + } + + const mimeType = metadata.file?.mimeType || 'application/octet-stream' + + logger.info(`[${requestId}] Downloading file from OneDrive`, { fileId, mimeType }) + + const downloadUrl = `https://graph.microsoft.com/v1.0/me/drive/items/${fileId}/content` + const downloadUrlValidation = await validateUrlWithDNS(downloadUrl, 'downloadUrl') + if (!downloadUrlValidation.isValid) { + return NextResponse.json( + { success: false, error: downloadUrlValidation.error }, + { status: 400 } + ) + } + + const downloadResponse = await secureFetchWithPinnedIP( + downloadUrl, + downloadUrlValidation.resolvedIP!, + { + headers: { Authorization: authHeader }, + } + ) + + if (!downloadResponse.ok) { + const downloadError = (await downloadResponse.json().catch(() => ({}))) as GraphApiError + logger.error(`[${requestId}] Failed to download file`, { + status: downloadResponse.status, + error: downloadError, + }) + return NextResponse.json( + { success: false, error: downloadError.error?.message || 'Failed to download file' }, + { status: 400 } + ) + } + + const arrayBuffer = await downloadResponse.arrayBuffer() + const fileBuffer = Buffer.from(arrayBuffer) + + const resolvedName = fileName || metadata.name || 'download' + + logger.info(`[${requestId}] File downloaded successfully`, { + fileId, + name: resolvedName, + size: fileBuffer.length, + mimeType, + }) + + const base64Data = fileBuffer.toString('base64') + + return NextResponse.json({ + success: true, + output: { + file: { + name: resolvedName, + mimeType, + data: base64Data, + size: fileBuffer.length, + }, + }, + }) + } catch (error) { + logger.error(`[${requestId}] Error downloading OneDrive file:`, error) + return NextResponse.json( + { + success: false, + error: error instanceof Error ? error.message : 'Unknown error occurred', + }, + { status: 500 } + ) + } +} diff --git a/apps/sim/app/api/tools/onedrive/upload/route.ts b/apps/sim/app/api/tools/onedrive/upload/route.ts index 759b41da3..8919b528c 100644 --- a/apps/sim/app/api/tools/onedrive/upload/route.ts +++ b/apps/sim/app/api/tools/onedrive/upload/route.ts @@ -4,7 +4,9 @@ import * as XLSX from 'xlsx' import { z } from 'zod' import { checkInternalAuth } from '@/lib/auth/hybrid' import { validateMicrosoftGraphId } from '@/lib/core/security/input-validation' +import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' +import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' import { getExtensionFromMimeType, processSingleFileToUserFile, @@ -29,12 +31,33 @@ const ExcelValuesSchema = z.union([ const OneDriveUploadSchema = z.object({ accessToken: z.string().min(1, 'Access token is required'), fileName: z.string().min(1, 'File name is required'), - file: z.any().optional(), + file: RawFileInputSchema.optional(), folderId: z.string().optional().nullable(), mimeType: z.string().nullish(), values: ExcelValuesSchema.optional().nullable(), + conflictBehavior: z.enum(['fail', 'replace', 'rename']).optional().nullable(), }) +/** Microsoft Graph DriveItem response */ +interface OneDriveFileData { + id: string + name: string + size: number + webUrl: string + createdDateTime: string + lastModifiedDateTime: string + file?: { mimeType: string } + parentReference?: { id: string; path: string } + '@microsoft.graph.downloadUrl'?: string +} + +/** Microsoft Graph Excel range response */ +interface ExcelRangeData { + address?: string + addressLocal?: string + values?: unknown[][] +} + export async function POST(request: NextRequest) { const requestId = generateRequestId() @@ -88,25 +111,9 @@ export async function POST(request: NextRequest) { ) } - let fileToProcess - if (Array.isArray(rawFile)) { - if (rawFile.length === 0) { - return NextResponse.json( - { - success: false, - error: 'No file provided', - }, - { status: 400 } - ) - } - fileToProcess = rawFile[0] - } else { - fileToProcess = rawFile - } - let userFile try { - userFile = processSingleFileToUserFile(fileToProcess, requestId, logger) + userFile = processSingleFileToUserFile(rawFile, requestId, logger) } catch (error) { return NextResponse.json( { @@ -179,14 +186,23 @@ export async function POST(request: NextRequest) { uploadUrl = `${MICROSOFT_GRAPH_BASE}/me/drive/root:/${encodeURIComponent(fileName)}:/content` } - const uploadResponse = await fetch(uploadUrl, { - method: 'PUT', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'Content-Type': mimeType, + // Add conflict behavior if specified (defaults to replace by Microsoft Graph API) + if (validatedData.conflictBehavior) { + uploadUrl += `?@microsoft.graph.conflictBehavior=${validatedData.conflictBehavior}` + } + + const uploadResponse = await secureFetchWithValidation( + uploadUrl, + { + method: 'PUT', + headers: { + Authorization: `Bearer ${validatedData.accessToken}`, + 'Content-Type': mimeType, + }, + body: fileBuffer, }, - body: new Uint8Array(fileBuffer), - }) + 'uploadUrl' + ) if (!uploadResponse.ok) { const errorText = await uploadResponse.text() @@ -200,7 +216,7 @@ export async function POST(request: NextRequest) { ) } - const fileData = await uploadResponse.json() + const fileData = (await uploadResponse.json()) as OneDriveFileData let excelWriteResult: any | undefined const shouldWriteExcelContent = @@ -209,8 +225,11 @@ export async function POST(request: NextRequest) { if (shouldWriteExcelContent) { try { let workbookSessionId: string | undefined - const sessionResp = await fetch( - `${MICROSOFT_GRAPH_BASE}/me/drive/items/${encodeURIComponent(fileData.id)}/workbook/createSession`, + const sessionUrl = `${MICROSOFT_GRAPH_BASE}/me/drive/items/${encodeURIComponent( + fileData.id + )}/workbook/createSession` + const sessionResp = await secureFetchWithValidation( + sessionUrl, { method: 'POST', headers: { @@ -218,11 +237,12 @@ export async function POST(request: NextRequest) { 'Content-Type': 'application/json', }, body: JSON.stringify({ persistChanges: true }), - } + }, + 'sessionUrl' ) if (sessionResp.ok) { - const sessionData = await sessionResp.json() + const sessionData = (await sessionResp.json()) as { id?: string } workbookSessionId = sessionData?.id } @@ -231,14 +251,19 @@ export async function POST(request: NextRequest) { const listUrl = `${MICROSOFT_GRAPH_BASE}/me/drive/items/${encodeURIComponent( fileData.id )}/workbook/worksheets?$select=name&$orderby=position&$top=1` - const listResp = await fetch(listUrl, { - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - ...(workbookSessionId ? { 'workbook-session-id': workbookSessionId } : {}), + const listResp = await secureFetchWithValidation( + listUrl, + { + method: 'GET', + headers: { + Authorization: `Bearer ${validatedData.accessToken}`, + ...(workbookSessionId ? { 'workbook-session-id': workbookSessionId } : {}), + }, }, - }) + 'listUrl' + ) if (listResp.ok) { - const listData = await listResp.json() + const listData = (await listResp.json()) as { value?: Array<{ name?: string }> } const firstSheetName = listData?.value?.[0]?.name if (firstSheetName) { sheetName = firstSheetName @@ -297,15 +322,19 @@ export async function POST(request: NextRequest) { )}')/range(address='${encodeURIComponent(computedRangeAddress)}')` ) - const excelWriteResponse = await fetch(url.toString(), { - method: 'PATCH', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'Content-Type': 'application/json', - ...(workbookSessionId ? { 'workbook-session-id': workbookSessionId } : {}), + const excelWriteResponse = await secureFetchWithValidation( + url.toString(), + { + method: 'PATCH', + headers: { + Authorization: `Bearer ${validatedData.accessToken}`, + 'Content-Type': 'application/json', + ...(workbookSessionId ? { 'workbook-session-id': workbookSessionId } : {}), + }, + body: JSON.stringify({ values: processedValues }), }, - body: JSON.stringify({ values: processedValues }), - }) + 'excelWriteUrl' + ) if (!excelWriteResponse || !excelWriteResponse.ok) { const errorText = excelWriteResponse ? await excelWriteResponse.text() : 'no response' @@ -320,7 +349,7 @@ export async function POST(request: NextRequest) { details: errorText, } } else { - const writeData = await excelWriteResponse.json() + const writeData = (await excelWriteResponse.json()) as ExcelRangeData const addr = writeData.address || writeData.addressLocal const v = writeData.values || [] excelWriteResult = { @@ -328,21 +357,25 @@ export async function POST(request: NextRequest) { updatedRange: addr, updatedRows: Array.isArray(v) ? v.length : undefined, updatedColumns: Array.isArray(v) && v[0] ? v[0].length : undefined, - updatedCells: Array.isArray(v) && v[0] ? v.length * (v[0] as any[]).length : undefined, + updatedCells: Array.isArray(v) && v[0] ? v.length * v[0].length : undefined, } } if (workbookSessionId) { try { - const closeResp = await fetch( - `${MICROSOFT_GRAPH_BASE}/me/drive/items/${encodeURIComponent(fileData.id)}/workbook/closeSession`, + const closeUrl = `${MICROSOFT_GRAPH_BASE}/me/drive/items/${encodeURIComponent( + fileData.id + )}/workbook/closeSession` + const closeResp = await secureFetchWithValidation( + closeUrl, { method: 'POST', headers: { Authorization: `Bearer ${validatedData.accessToken}`, 'workbook-session-id': workbookSessionId, }, - } + }, + 'closeSessionUrl' ) if (!closeResp.ok) { const closeText = await closeResp.text() diff --git a/apps/sim/app/api/tools/outlook/draft/route.ts b/apps/sim/app/api/tools/outlook/draft/route.ts index 39bb3f5ef..eeee0f14e 100644 --- a/apps/sim/app/api/tools/outlook/draft/route.ts +++ b/apps/sim/app/api/tools/outlook/draft/route.ts @@ -3,6 +3,7 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { RawFileInputArraySchema } from '@/lib/uploads/utils/file-schemas' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -18,7 +19,7 @@ const OutlookDraftSchema = z.object({ contentType: z.enum(['text', 'html']).optional().nullable(), cc: z.string().optional().nullable(), bcc: z.string().optional().nullable(), - attachments: z.array(z.any()).optional().nullable(), + attachments: RawFileInputArraySchema.optional().nullable(), }) export async function POST(request: NextRequest) { diff --git a/apps/sim/app/api/tools/outlook/send/route.ts b/apps/sim/app/api/tools/outlook/send/route.ts index 329318880..f90f62518 100644 --- a/apps/sim/app/api/tools/outlook/send/route.ts +++ b/apps/sim/app/api/tools/outlook/send/route.ts @@ -3,6 +3,7 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { RawFileInputArraySchema } from '@/lib/uploads/utils/file-schemas' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -20,7 +21,7 @@ const OutlookSendSchema = z.object({ bcc: z.string().optional().nullable(), replyToMessageId: z.string().optional().nullable(), conversationId: z.string().optional().nullable(), - attachments: z.array(z.any()).optional().nullable(), + attachments: RawFileInputArraySchema.optional().nullable(), }) export async function POST(request: NextRequest) { @@ -95,14 +96,14 @@ export async function POST(request: NextRequest) { if (attachments.length > 0) { const totalSize = attachments.reduce((sum, file) => sum + file.size, 0) - const maxSize = 4 * 1024 * 1024 // 4MB + const maxSize = 3 * 1024 * 1024 // 3MB - Microsoft Graph API limit for inline attachments if (totalSize > maxSize) { const sizeMB = (totalSize / (1024 * 1024)).toFixed(2) return NextResponse.json( { success: false, - error: `Total attachment size (${sizeMB}MB) exceeds Outlook's limit of 4MB per request`, + error: `Total attachment size (${sizeMB}MB) exceeds Microsoft Graph API limit of 3MB per request`, }, { status: 400 } ) diff --git a/apps/sim/app/api/tools/pipedrive/get-files/route.ts b/apps/sim/app/api/tools/pipedrive/get-files/route.ts new file mode 100644 index 000000000..93111c94b --- /dev/null +++ b/apps/sim/app/api/tools/pipedrive/get-files/route.ts @@ -0,0 +1,165 @@ +import { createLogger } from '@sim/logger' +import { type NextRequest, NextResponse } from 'next/server' +import { z } from 'zod' +import { checkInternalAuth } from '@/lib/auth/hybrid' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { generateRequestId } from '@/lib/core/utils/request' +import { getFileExtension, getMimeTypeFromExtension } from '@/lib/uploads/utils/file-utils' + +export const dynamic = 'force-dynamic' + +const logger = createLogger('PipedriveGetFilesAPI') + +interface PipedriveFile { + id?: number + name?: string + url?: string +} + +interface PipedriveApiResponse { + success: boolean + data?: PipedriveFile[] + error?: string +} + +const PipedriveGetFilesSchema = z.object({ + accessToken: z.string().min(1, 'Access token is required'), + deal_id: z.string().optional().nullable(), + person_id: z.string().optional().nullable(), + org_id: z.string().optional().nullable(), + limit: z.string().optional().nullable(), + downloadFiles: z.boolean().optional().default(false), +}) + +export async function POST(request: NextRequest) { + const requestId = generateRequestId() + + try { + const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) + + if (!authResult.success) { + logger.warn(`[${requestId}] Unauthorized Pipedrive get files attempt: ${authResult.error}`) + return NextResponse.json( + { + success: false, + error: authResult.error || 'Authentication required', + }, + { status: 401 } + ) + } + + const body = await request.json() + const validatedData = PipedriveGetFilesSchema.parse(body) + + const { accessToken, deal_id, person_id, org_id, limit, downloadFiles } = validatedData + + const baseUrl = 'https://api.pipedrive.com/v1/files' + const queryParams = new URLSearchParams() + + if (deal_id) queryParams.append('deal_id', deal_id) + if (person_id) queryParams.append('person_id', person_id) + if (org_id) queryParams.append('org_id', org_id) + if (limit) queryParams.append('limit', limit) + + const queryString = queryParams.toString() + const apiUrl = queryString ? `${baseUrl}?${queryString}` : baseUrl + + logger.info(`[${requestId}] Fetching files from Pipedrive`, { deal_id, person_id, org_id }) + + const urlValidation = await validateUrlWithDNS(apiUrl, 'apiUrl') + if (!urlValidation.isValid) { + return NextResponse.json({ success: false, error: urlValidation.error }, { status: 400 }) + } + + const response = await secureFetchWithPinnedIP(apiUrl, urlValidation.resolvedIP!, { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, + }) + + const data = (await response.json()) as PipedriveApiResponse + + if (!data.success) { + logger.error(`[${requestId}] Pipedrive API request failed`, { data }) + return NextResponse.json( + { success: false, error: data.error || 'Failed to fetch files from Pipedrive' }, + { status: 400 } + ) + } + + const files = data.data || [] + const downloadedFiles: Array<{ + name: string + mimeType: string + data: string + size: number + }> = [] + + if (downloadFiles) { + for (const file of files) { + if (!file?.url) continue + + try { + const fileUrlValidation = await validateUrlWithDNS(file.url, 'fileUrl') + if (!fileUrlValidation.isValid) continue + + const downloadResponse = await secureFetchWithPinnedIP( + file.url, + fileUrlValidation.resolvedIP!, + { + method: 'GET', + headers: { Authorization: `Bearer ${accessToken}` }, + } + ) + + if (!downloadResponse.ok) continue + + const arrayBuffer = await downloadResponse.arrayBuffer() + const buffer = Buffer.from(arrayBuffer) + const extension = getFileExtension(file.name || '') + const mimeType = + downloadResponse.headers.get('content-type') || getMimeTypeFromExtension(extension) + const fileName = file.name || `pipedrive-file-${file.id || Date.now()}` + + downloadedFiles.push({ + name: fileName, + mimeType, + data: buffer.toString('base64'), + size: buffer.length, + }) + } catch (error) { + logger.warn(`[${requestId}] Failed to download file ${file.id}:`, error) + } + } + } + + logger.info(`[${requestId}] Pipedrive files fetched successfully`, { + fileCount: files.length, + downloadedCount: downloadedFiles.length, + }) + + return NextResponse.json({ + success: true, + output: { + files, + downloadedFiles: downloadedFiles.length > 0 ? downloadedFiles : undefined, + total_items: files.length, + success: true, + }, + }) + } catch (error) { + logger.error(`[${requestId}] Error fetching Pipedrive files:`, error) + return NextResponse.json( + { + success: false, + error: error instanceof Error ? error.message : 'Unknown error occurred', + }, + { status: 500 } + ) + } +} diff --git a/apps/sim/app/api/tools/pulse/parse/route.ts b/apps/sim/app/api/tools/pulse/parse/route.ts index 59adeec15..39dc9259a 100644 --- a/apps/sim/app/api/tools/pulse/parse/route.ts +++ b/apps/sim/app/api/tools/pulse/parse/route.ts @@ -2,15 +2,14 @@ import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { getBaseUrl } from '@/lib/core/utils/urls' -import { StorageService } from '@/lib/uploads' import { - extractStorageKey, - inferContextFromKey, - isInternalFileUrl, -} from '@/lib/uploads/utils/file-utils' -import { verifyFileAccess } from '@/app/api/files/authorization' + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { generateRequestId } from '@/lib/core/utils/request' +import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' +import { isInternalFileUrl } from '@/lib/uploads/utils/file-utils' +import { resolveFileInputToUrl } from '@/lib/uploads/utils/file-utils.server' export const dynamic = 'force-dynamic' @@ -18,7 +17,8 @@ const logger = createLogger('PulseParseAPI') const PulseParseSchema = z.object({ apiKey: z.string().min(1, 'API key is required'), - filePath: z.string().min(1, 'File path is required'), + filePath: z.string().optional(), + file: RawFileInputSchema.optional(), pages: z.string().optional(), extractFigure: z.boolean().optional(), figureDescription: z.boolean().optional(), @@ -51,50 +51,30 @@ export async function POST(request: NextRequest) { const validatedData = PulseParseSchema.parse(body) logger.info(`[${requestId}] Pulse parse request`, { + fileName: validatedData.file?.name, filePath: validatedData.filePath, - isWorkspaceFile: isInternalFileUrl(validatedData.filePath), + isWorkspaceFile: validatedData.filePath ? isInternalFileUrl(validatedData.filePath) : false, userId, }) - let fileUrl = validatedData.filePath + const resolution = await resolveFileInputToUrl({ + file: validatedData.file, + filePath: validatedData.filePath, + userId, + requestId, + logger, + }) - if (isInternalFileUrl(validatedData.filePath)) { - try { - const storageKey = extractStorageKey(validatedData.filePath) - const context = inferContextFromKey(storageKey) + if (resolution.error) { + return NextResponse.json( + { success: false, error: resolution.error.message }, + { status: resolution.error.status } + ) + } - const hasAccess = await verifyFileAccess(storageKey, userId, undefined, context, false) - - if (!hasAccess) { - logger.warn(`[${requestId}] Unauthorized presigned URL generation attempt`, { - userId, - key: storageKey, - context, - }) - return NextResponse.json( - { - success: false, - error: 'File not found', - }, - { status: 404 } - ) - } - - fileUrl = await StorageService.generatePresignedDownloadUrl(storageKey, context, 5 * 60) - logger.info(`[${requestId}] Generated presigned URL for ${context} file`) - } catch (error) { - logger.error(`[${requestId}] Failed to generate presigned URL:`, error) - return NextResponse.json( - { - success: false, - error: 'Failed to generate file access URL', - }, - { status: 500 } - ) - } - } else if (validatedData.filePath?.startsWith('/')) { - const baseUrl = getBaseUrl() - fileUrl = `${baseUrl}${validatedData.filePath}` + const fileUrl = resolution.fileUrl + if (!fileUrl) { + return NextResponse.json({ success: false, error: 'File input is required' }, { status: 400 }) } const formData = new FormData() @@ -119,13 +99,36 @@ export async function POST(request: NextRequest) { formData.append('chunk_size', String(validatedData.chunkSize)) } - const pulseResponse = await fetch('https://api.runpulse.com/extract', { - method: 'POST', - headers: { - 'x-api-key': validatedData.apiKey, - }, - body: formData, - }) + const pulseEndpoint = 'https://api.runpulse.com/extract' + const pulseValidation = await validateUrlWithDNS(pulseEndpoint, 'Pulse API URL') + if (!pulseValidation.isValid) { + logger.error(`[${requestId}] Pulse API URL validation failed`, { + error: pulseValidation.error, + }) + return NextResponse.json( + { + success: false, + error: 'Failed to reach Pulse API', + }, + { status: 502 } + ) + } + + const pulsePayload = new Response(formData) + const contentType = pulsePayload.headers.get('content-type') || 'multipart/form-data' + const bodyBuffer = Buffer.from(await pulsePayload.arrayBuffer()) + const pulseResponse = await secureFetchWithPinnedIP( + pulseEndpoint, + pulseValidation.resolvedIP!, + { + method: 'POST', + headers: { + 'x-api-key': validatedData.apiKey, + 'Content-Type': contentType, + }, + body: bodyBuffer, + } + ) if (!pulseResponse.ok) { const errorText = await pulseResponse.text() diff --git a/apps/sim/app/api/tools/reducto/parse/route.ts b/apps/sim/app/api/tools/reducto/parse/route.ts index e8fd960ff..c526c8f2a 100644 --- a/apps/sim/app/api/tools/reducto/parse/route.ts +++ b/apps/sim/app/api/tools/reducto/parse/route.ts @@ -2,15 +2,14 @@ import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { getBaseUrl } from '@/lib/core/utils/urls' -import { StorageService } from '@/lib/uploads' import { - extractStorageKey, - inferContextFromKey, - isInternalFileUrl, -} from '@/lib/uploads/utils/file-utils' -import { verifyFileAccess } from '@/app/api/files/authorization' + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { generateRequestId } from '@/lib/core/utils/request' +import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' +import { isInternalFileUrl } from '@/lib/uploads/utils/file-utils' +import { resolveFileInputToUrl } from '@/lib/uploads/utils/file-utils.server' export const dynamic = 'force-dynamic' @@ -18,7 +17,8 @@ const logger = createLogger('ReductoParseAPI') const ReductoParseSchema = z.object({ apiKey: z.string().min(1, 'API key is required'), - filePath: z.string().min(1, 'File path is required'), + filePath: z.string().optional(), + file: RawFileInputSchema.optional(), pages: z.array(z.number()).optional(), tableOutputFormat: z.enum(['html', 'md']).optional(), }) @@ -47,56 +47,30 @@ export async function POST(request: NextRequest) { const validatedData = ReductoParseSchema.parse(body) logger.info(`[${requestId}] Reducto parse request`, { + fileName: validatedData.file?.name, filePath: validatedData.filePath, - isWorkspaceFile: isInternalFileUrl(validatedData.filePath), + isWorkspaceFile: validatedData.filePath ? isInternalFileUrl(validatedData.filePath) : false, userId, }) - let fileUrl = validatedData.filePath + const resolution = await resolveFileInputToUrl({ + file: validatedData.file, + filePath: validatedData.filePath, + userId, + requestId, + logger, + }) - if (isInternalFileUrl(validatedData.filePath)) { - try { - const storageKey = extractStorageKey(validatedData.filePath) - const context = inferContextFromKey(storageKey) + if (resolution.error) { + return NextResponse.json( + { success: false, error: resolution.error.message }, + { status: resolution.error.status } + ) + } - const hasAccess = await verifyFileAccess( - storageKey, - userId, - undefined, // customConfig - context, // context - false // isLocal - ) - - if (!hasAccess) { - logger.warn(`[${requestId}] Unauthorized presigned URL generation attempt`, { - userId, - key: storageKey, - context, - }) - return NextResponse.json( - { - success: false, - error: 'File not found', - }, - { status: 404 } - ) - } - - fileUrl = await StorageService.generatePresignedDownloadUrl(storageKey, context, 5 * 60) - logger.info(`[${requestId}] Generated presigned URL for ${context} file`) - } catch (error) { - logger.error(`[${requestId}] Failed to generate presigned URL:`, error) - return NextResponse.json( - { - success: false, - error: 'Failed to generate file access URL', - }, - { status: 500 } - ) - } - } else if (validatedData.filePath?.startsWith('/')) { - const baseUrl = getBaseUrl() - fileUrl = `${baseUrl}${validatedData.filePath}` + const fileUrl = resolution.fileUrl + if (!fileUrl) { + return NextResponse.json({ success: false, error: 'File input is required' }, { status: 400 }) } const reductoBody: Record = { @@ -104,8 +78,13 @@ export async function POST(request: NextRequest) { } if (validatedData.pages && validatedData.pages.length > 0) { + // Reducto API expects page_range as an object with start/end, not an array + const pages = validatedData.pages reductoBody.settings = { - page_range: validatedData.pages, + page_range: { + start: Math.min(...pages), + end: Math.max(...pages), + }, } } @@ -115,15 +94,34 @@ export async function POST(request: NextRequest) { } } - const reductoResponse = await fetch('https://platform.reducto.ai/parse', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Accept: 'application/json', - Authorization: `Bearer ${validatedData.apiKey}`, - }, - body: JSON.stringify(reductoBody), - }) + const reductoEndpoint = 'https://platform.reducto.ai/parse' + const reductoValidation = await validateUrlWithDNS(reductoEndpoint, 'Reducto API URL') + if (!reductoValidation.isValid) { + logger.error(`[${requestId}] Reducto API URL validation failed`, { + error: reductoValidation.error, + }) + return NextResponse.json( + { + success: false, + error: 'Failed to reach Reducto API', + }, + { status: 502 } + ) + } + + const reductoResponse = await secureFetchWithPinnedIP( + reductoEndpoint, + reductoValidation.resolvedIP!, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + Authorization: `Bearer ${validatedData.apiKey}`, + }, + body: JSON.stringify(reductoBody), + } + ) if (!reductoResponse.ok) { const errorText = await reductoResponse.text() diff --git a/apps/sim/app/api/tools/s3/put-object/route.ts b/apps/sim/app/api/tools/s3/put-object/route.ts index c33f250bc..c55950bc9 100644 --- a/apps/sim/app/api/tools/s3/put-object/route.ts +++ b/apps/sim/app/api/tools/s3/put-object/route.ts @@ -4,6 +4,7 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -17,7 +18,7 @@ const S3PutObjectSchema = z.object({ region: z.string().min(1, 'Region is required'), bucketName: z.string().min(1, 'Bucket name is required'), objectKey: z.string().min(1, 'Object key is required'), - file: z.any().optional().nullable(), + file: RawFileInputSchema.optional().nullable(), content: z.string().optional().nullable(), contentType: z.string().optional().nullable(), acl: z.string().optional().nullable(), diff --git a/apps/sim/app/api/tools/sendgrid/send-mail/route.ts b/apps/sim/app/api/tools/sendgrid/send-mail/route.ts new file mode 100644 index 000000000..362960b89 --- /dev/null +++ b/apps/sim/app/api/tools/sendgrid/send-mail/route.ts @@ -0,0 +1,188 @@ +import { createLogger } from '@sim/logger' +import { type NextRequest, NextResponse } from 'next/server' +import { z } from 'zod' +import { checkInternalAuth } from '@/lib/auth/hybrid' +import { generateRequestId } from '@/lib/core/utils/request' +import { RawFileInputArraySchema } from '@/lib/uploads/utils/file-schemas' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' + +export const dynamic = 'force-dynamic' + +const logger = createLogger('SendGridSendMailAPI') + +const SendGridSendMailSchema = z.object({ + apiKey: z.string().min(1, 'API key is required'), + from: z.string().min(1, 'From email is required'), + fromName: z.string().optional().nullable(), + to: z.string().min(1, 'To email is required'), + toName: z.string().optional().nullable(), + subject: z.string().optional().nullable(), + content: z.string().optional().nullable(), + contentType: z.string().optional().nullable(), + cc: z.string().optional().nullable(), + bcc: z.string().optional().nullable(), + replyTo: z.string().optional().nullable(), + replyToName: z.string().optional().nullable(), + templateId: z.string().optional().nullable(), + dynamicTemplateData: z.any().optional().nullable(), + attachments: RawFileInputArraySchema.optional().nullable(), +}) + +export async function POST(request: NextRequest) { + const requestId = generateRequestId() + + try { + const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) + + if (!authResult.success) { + logger.warn(`[${requestId}] Unauthorized SendGrid send attempt: ${authResult.error}`) + return NextResponse.json( + { success: false, error: authResult.error || 'Authentication required' }, + { status: 401 } + ) + } + + logger.info(`[${requestId}] Authenticated SendGrid send request via ${authResult.authType}`) + + const body = await request.json() + const validatedData = SendGridSendMailSchema.parse(body) + + logger.info(`[${requestId}] Sending SendGrid email`, { + to: validatedData.to, + subject: validatedData.subject || '(template)', + hasAttachments: !!(validatedData.attachments && validatedData.attachments.length > 0), + attachmentCount: validatedData.attachments?.length || 0, + }) + + // Build personalizations + const personalizations: Record = { + to: [ + { email: validatedData.to, ...(validatedData.toName && { name: validatedData.toName }) }, + ], + } + + if (validatedData.cc) { + personalizations.cc = [{ email: validatedData.cc }] + } + + if (validatedData.bcc) { + personalizations.bcc = [{ email: validatedData.bcc }] + } + + if (validatedData.templateId && validatedData.dynamicTemplateData) { + personalizations.dynamic_template_data = + typeof validatedData.dynamicTemplateData === 'string' + ? JSON.parse(validatedData.dynamicTemplateData) + : validatedData.dynamicTemplateData + } + + // Build mail body + const mailBody: Record = { + personalizations: [personalizations], + from: { + email: validatedData.from, + ...(validatedData.fromName && { name: validatedData.fromName }), + }, + subject: validatedData.subject, + } + + if (validatedData.templateId) { + mailBody.template_id = validatedData.templateId + } else { + mailBody.content = [ + { + type: validatedData.contentType || 'text/plain', + value: validatedData.content, + }, + ] + } + + if (validatedData.replyTo) { + mailBody.reply_to = { + email: validatedData.replyTo, + ...(validatedData.replyToName && { name: validatedData.replyToName }), + } + } + + // Process attachments from UserFile objects + if (validatedData.attachments && validatedData.attachments.length > 0) { + const rawAttachments = validatedData.attachments + logger.info(`[${requestId}] Processing ${rawAttachments.length} attachment(s)`) + + const userFiles = processFilesToUserFiles(rawAttachments, requestId, logger) + + if (userFiles.length > 0) { + const sendGridAttachments = await Promise.all( + userFiles.map(async (file) => { + try { + logger.info( + `[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)` + ) + const buffer = await downloadFileFromStorage(file, requestId, logger) + + return { + content: buffer.toString('base64'), + filename: file.name, + type: file.type || 'application/octet-stream', + disposition: 'attachment', + } + } catch (error) { + logger.error(`[${requestId}] Failed to download attachment ${file.name}:`, error) + throw new Error( + `Failed to download attachment "${file.name}": ${error instanceof Error ? error.message : 'Unknown error'}` + ) + } + }) + ) + + mailBody.attachments = sendGridAttachments + } + } + + // Send to SendGrid + const response = await fetch('https://api.sendgrid.com/v3/mail/send', { + method: 'POST', + headers: { + Authorization: `Bearer ${validatedData.apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(mailBody), + }) + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})) + const errorMessage = + errorData.errors?.[0]?.message || errorData.message || 'Failed to send email' + logger.error(`[${requestId}] SendGrid API error:`, { status: response.status, errorData }) + return NextResponse.json({ success: false, error: errorMessage }, { status: response.status }) + } + + const messageId = response.headers.get('X-Message-Id') + logger.info(`[${requestId}] Email sent successfully`, { messageId }) + + return NextResponse.json({ + success: true, + output: { + success: true, + messageId: messageId || undefined, + to: validatedData.to, + subject: validatedData.subject || '', + }, + }) + } catch (error) { + if (error instanceof z.ZodError) { + logger.warn(`[${requestId}] Validation error:`, error.errors) + return NextResponse.json( + { success: false, error: error.errors[0]?.message || 'Validation failed' }, + { status: 400 } + ) + } + + logger.error(`[${requestId}] Unexpected error:`, error) + return NextResponse.json( + { success: false, error: error instanceof Error ? error.message : 'Unknown error' }, + { status: 500 } + ) + } +} diff --git a/apps/sim/app/api/tools/sftp/download/route.ts b/apps/sim/app/api/tools/sftp/download/route.ts index 4914703fc..849e1ee09 100644 --- a/apps/sim/app/api/tools/sftp/download/route.ts +++ b/apps/sim/app/api/tools/sftp/download/route.ts @@ -4,6 +4,7 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { getFileExtension, getMimeTypeFromExtension } from '@/lib/uploads/utils/file-utils' import { createSftpConnection, getSftp, isPathSafe, sanitizePath } from '@/app/api/tools/sftp/utils' export const dynamic = 'force-dynamic' @@ -111,6 +112,8 @@ export async function POST(request: NextRequest) { const buffer = Buffer.concat(chunks) const fileName = path.basename(remotePath) + const extension = getFileExtension(fileName) + const mimeType = getMimeTypeFromExtension(extension) let content: string if (params.encoding === 'base64') { @@ -124,6 +127,12 @@ export async function POST(request: NextRequest) { return NextResponse.json({ success: true, fileName, + file: { + name: fileName, + mimeType, + data: buffer.toString('base64'), + size: buffer.length, + }, content, size: buffer.length, encoding: params.encoding, diff --git a/apps/sim/app/api/tools/sftp/upload/route.ts b/apps/sim/app/api/tools/sftp/upload/route.ts index 90f5e6ab7..54851e595 100644 --- a/apps/sim/app/api/tools/sftp/upload/route.ts +++ b/apps/sim/app/api/tools/sftp/upload/route.ts @@ -3,6 +3,7 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { RawFileInputArraySchema } from '@/lib/uploads/utils/file-schemas' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { @@ -26,14 +27,7 @@ const UploadSchema = z.object({ privateKey: z.string().nullish(), passphrase: z.string().nullish(), remotePath: z.string().min(1, 'Remote path is required'), - files: z - .union([z.array(z.any()), z.string(), z.number(), z.null(), z.undefined()]) - .transform((val) => { - if (Array.isArray(val)) return val - if (val === null || val === undefined || val === '') return undefined - return undefined - }) - .nullish(), + files: RawFileInputArraySchema.optional().nullable(), fileContent: z.string().nullish(), fileName: z.string().nullish(), overwrite: z.boolean().default(true), diff --git a/apps/sim/app/api/tools/sharepoint/upload/route.ts b/apps/sim/app/api/tools/sharepoint/upload/route.ts index 26ce0b1d2..df8f33712 100644 --- a/apps/sim/app/api/tools/sharepoint/upload/route.ts +++ b/apps/sim/app/api/tools/sharepoint/upload/route.ts @@ -2,9 +2,12 @@ import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkInternalAuth } from '@/lib/auth/hybrid' +import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' +import { RawFileInputArraySchema } from '@/lib/uploads/utils/file-schemas' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import type { MicrosoftGraphDriveItem } from '@/tools/onedrive/types' export const dynamic = 'force-dynamic' @@ -16,7 +19,7 @@ const SharepointUploadSchema = z.object({ driveId: z.string().optional().nullable(), folderPath: z.string().optional().nullable(), fileName: z.string().optional().nullable(), - files: z.array(z.any()).optional().nullable(), + files: RawFileInputArraySchema.optional().nullable(), }) export async function POST(request: NextRequest) { @@ -79,18 +82,23 @@ export async function POST(request: NextRequest) { let effectiveDriveId = validatedData.driveId if (!effectiveDriveId) { logger.info(`[${requestId}] No driveId provided, fetching default drive for site`) - const driveResponse = await fetch( - `https://graph.microsoft.com/v1.0/sites/${validatedData.siteId}/drive`, + const driveUrl = `https://graph.microsoft.com/v1.0/sites/${validatedData.siteId}/drive` + const driveResponse = await secureFetchWithValidation( + driveUrl, { + method: 'GET', headers: { Authorization: `Bearer ${validatedData.accessToken}`, Accept: 'application/json', }, - } + }, + 'driveUrl' ) if (!driveResponse.ok) { - const errorData = await driveResponse.json().catch(() => ({})) + const errorData = (await driveResponse.json().catch(() => ({}))) as { + error?: { message?: string } + } logger.error(`[${requestId}] Failed to get default drive:`, errorData) return NextResponse.json( { @@ -101,7 +109,7 @@ export async function POST(request: NextRequest) { ) } - const driveData = await driveResponse.json() + const driveData = (await driveResponse.json()) as { id: string } effectiveDriveId = driveData.id logger.info(`[${requestId}] Using default drive: ${effectiveDriveId}`) } @@ -145,34 +153,87 @@ export async function POST(request: NextRequest) { logger.info(`[${requestId}] Uploading to: ${uploadUrl}`) - const uploadResponse = await fetch(uploadUrl, { - method: 'PUT', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'Content-Type': userFile.type || 'application/octet-stream', + const uploadResponse = await secureFetchWithValidation( + uploadUrl, + { + method: 'PUT', + headers: { + Authorization: `Bearer ${validatedData.accessToken}`, + 'Content-Type': userFile.type || 'application/octet-stream', + }, + body: buffer, }, - body: new Uint8Array(buffer), - }) + 'uploadUrl' + ) if (!uploadResponse.ok) { const errorData = await uploadResponse.json().catch(() => ({})) logger.error(`[${requestId}] Failed to upload file ${fileName}:`, errorData) if (uploadResponse.status === 409) { - logger.warn(`[${requestId}] File ${fileName} already exists, attempting to replace`) + // File exists - retry with conflict behavior set to replace + logger.warn(`[${requestId}] File ${fileName} already exists, retrying with replace`) + const replaceUrl = `${uploadUrl}?@microsoft.graph.conflictBehavior=replace` + const replaceResponse = await secureFetchWithValidation( + replaceUrl, + { + method: 'PUT', + headers: { + Authorization: `Bearer ${validatedData.accessToken}`, + 'Content-Type': userFile.type || 'application/octet-stream', + }, + body: buffer, + }, + 'replaceUrl' + ) + + if (!replaceResponse.ok) { + const replaceErrorData = (await replaceResponse.json().catch(() => ({}))) as { + error?: { message?: string } + } + logger.error(`[${requestId}] Failed to replace file ${fileName}:`, replaceErrorData) + return NextResponse.json( + { + success: false, + error: replaceErrorData.error?.message || `Failed to replace file: ${fileName}`, + }, + { status: replaceResponse.status } + ) + } + + const replaceData = (await replaceResponse.json()) as { + id: string + name: string + webUrl: string + size: number + createdDateTime: string + lastModifiedDateTime: string + } + logger.info(`[${requestId}] File replaced successfully: ${fileName}`) + + uploadedFiles.push({ + id: replaceData.id, + name: replaceData.name, + webUrl: replaceData.webUrl, + size: replaceData.size, + createdDateTime: replaceData.createdDateTime, + lastModifiedDateTime: replaceData.lastModifiedDateTime, + }) continue } return NextResponse.json( { success: false, - error: errorData.error?.message || `Failed to upload file: ${fileName}`, + error: + (errorData as { error?: { message?: string } }).error?.message || + `Failed to upload file: ${fileName}`, }, { status: uploadResponse.status } ) } - const uploadData = await uploadResponse.json() + const uploadData = (await uploadResponse.json()) as MicrosoftGraphDriveItem logger.info(`[${requestId}] File uploaded successfully: ${fileName}`) uploadedFiles.push({ diff --git a/apps/sim/app/api/tools/slack/download/route.ts b/apps/sim/app/api/tools/slack/download/route.ts new file mode 100644 index 000000000..45c34bcd1 --- /dev/null +++ b/apps/sim/app/api/tools/slack/download/route.ts @@ -0,0 +1,170 @@ +import { createLogger } from '@sim/logger' +import { type NextRequest, NextResponse } from 'next/server' +import { z } from 'zod' +import { checkInternalAuth } from '@/lib/auth/hybrid' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { generateRequestId } from '@/lib/core/utils/request' + +export const dynamic = 'force-dynamic' + +const logger = createLogger('SlackDownloadAPI') + +const SlackDownloadSchema = z.object({ + accessToken: z.string().min(1, 'Access token is required'), + fileId: z.string().min(1, 'File ID is required'), + fileName: z.string().optional().nullable(), +}) + +export async function POST(request: NextRequest) { + const requestId = generateRequestId() + + try { + const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) + + if (!authResult.success) { + logger.warn(`[${requestId}] Unauthorized Slack download attempt: ${authResult.error}`) + return NextResponse.json( + { + success: false, + error: authResult.error || 'Authentication required', + }, + { status: 401 } + ) + } + + logger.info(`[${requestId}] Authenticated Slack download request via ${authResult.authType}`, { + userId: authResult.userId, + }) + + const body = await request.json() + const validatedData = SlackDownloadSchema.parse(body) + + const { accessToken, fileId, fileName } = validatedData + + logger.info(`[${requestId}] Getting file info from Slack`, { fileId }) + + const infoResponse = await fetch(`https://slack.com/api/files.info?file=${fileId}`, { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + }, + }) + + if (!infoResponse.ok) { + const errorDetails = await infoResponse.json().catch(() => ({})) + logger.error(`[${requestId}] Failed to get file info from Slack`, { + status: infoResponse.status, + statusText: infoResponse.statusText, + error: errorDetails, + }) + return NextResponse.json( + { + success: false, + error: errorDetails.error || 'Failed to get file info', + }, + { status: 400 } + ) + } + + const data = await infoResponse.json() + + if (!data.ok) { + logger.error(`[${requestId}] Slack API returned error`, { error: data.error }) + return NextResponse.json( + { + success: false, + error: data.error || 'Slack API error', + }, + { status: 400 } + ) + } + + const file = data.file + const resolvedFileName = fileName || file.name || 'download' + const mimeType = file.mimetype || 'application/octet-stream' + const urlPrivate = file.url_private + + if (!urlPrivate) { + return NextResponse.json( + { + success: false, + error: 'File does not have a download URL', + }, + { status: 400 } + ) + } + + const urlValidation = await validateUrlWithDNS(urlPrivate, 'urlPrivate') + if (!urlValidation.isValid) { + return NextResponse.json( + { + success: false, + error: urlValidation.error, + }, + { status: 400 } + ) + } + + logger.info(`[${requestId}] Downloading file from Slack`, { + fileId, + fileName: resolvedFileName, + mimeType, + }) + + const downloadResponse = await secureFetchWithPinnedIP(urlPrivate, urlValidation.resolvedIP!, { + headers: { + Authorization: `Bearer ${accessToken}`, + }, + }) + + if (!downloadResponse.ok) { + logger.error(`[${requestId}] Failed to download file content`, { + status: downloadResponse.status, + statusText: downloadResponse.statusText, + }) + return NextResponse.json( + { + success: false, + error: 'Failed to download file content', + }, + { status: 400 } + ) + } + + const arrayBuffer = await downloadResponse.arrayBuffer() + const fileBuffer = Buffer.from(arrayBuffer) + + logger.info(`[${requestId}] File downloaded successfully`, { + fileId, + name: resolvedFileName, + size: fileBuffer.length, + mimeType, + }) + + const base64Data = fileBuffer.toString('base64') + + return NextResponse.json({ + success: true, + output: { + file: { + name: resolvedFileName, + mimeType, + data: base64Data, + size: fileBuffer.length, + }, + }, + }) + } catch (error) { + logger.error(`[${requestId}] Error downloading Slack file:`, error) + return NextResponse.json( + { + success: false, + error: error instanceof Error ? error.message : 'Unknown error occurred', + }, + { status: 500 } + ) + } +} diff --git a/apps/sim/app/api/tools/slack/send-message/route.ts b/apps/sim/app/api/tools/slack/send-message/route.ts index 3938b89d1..21f60faf6 100644 --- a/apps/sim/app/api/tools/slack/send-message/route.ts +++ b/apps/sim/app/api/tools/slack/send-message/route.ts @@ -3,6 +3,7 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { RawFileInputArraySchema } from '@/lib/uploads/utils/file-schemas' import { sendSlackMessage } from '../utils' export const dynamic = 'force-dynamic' @@ -16,7 +17,7 @@ const SlackSendMessageSchema = z userId: z.string().optional().nullable(), text: z.string().min(1, 'Message text is required'), thread_ts: z.string().optional().nullable(), - files: z.array(z.any()).optional().nullable(), + files: RawFileInputArraySchema.optional().nullable(), }) .refine((data) => data.channel || data.userId, { message: 'Either channel or userId is required', diff --git a/apps/sim/app/api/tools/slack/utils.ts b/apps/sim/app/api/tools/slack/utils.ts index 4a18071bf..b635c49d8 100644 --- a/apps/sim/app/api/tools/slack/utils.ts +++ b/apps/sim/app/api/tools/slack/utils.ts @@ -1,6 +1,8 @@ import type { Logger } from '@sim/logger' +import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import type { ToolFileData } from '@/tools/types' /** * Sends a message to a Slack channel using chat.postMessage @@ -70,9 +72,10 @@ export async function uploadFilesToSlack( accessToken: string, requestId: string, logger: Logger -): Promise { +): Promise<{ fileIds: string[]; files: ToolFileData[] }> { const userFiles = processFilesToUserFiles(files, requestId, logger) const uploadedFileIds: string[] = [] + const uploadedFiles: ToolFileData[] = [] for (const userFile of userFiles) { logger.info(`[${requestId}] Uploading file: ${userFile.name}`) @@ -100,10 +103,14 @@ export async function uploadFilesToSlack( logger.info(`[${requestId}] Got upload URL for ${userFile.name}, file_id: ${urlData.file_id}`) - const uploadResponse = await fetch(urlData.upload_url, { - method: 'POST', - body: new Uint8Array(buffer), - }) + const uploadResponse = await secureFetchWithValidation( + urlData.upload_url, + { + method: 'POST', + body: buffer, + }, + 'uploadUrl' + ) if (!uploadResponse.ok) { logger.error(`[${requestId}] Failed to upload file data: ${uploadResponse.status}`) @@ -112,9 +119,16 @@ export async function uploadFilesToSlack( logger.info(`[${requestId}] File data uploaded successfully`) uploadedFileIds.push(urlData.file_id) + // Only add to uploadedFiles after successful upload to keep arrays in sync + uploadedFiles.push({ + name: userFile.name, + mimeType: userFile.type || 'application/octet-stream', + data: buffer.toString('base64'), + size: buffer.length, + }) } - return uploadedFileIds + return { fileIds: uploadedFileIds, files: uploadedFiles } } /** @@ -124,7 +138,8 @@ export async function completeSlackFileUpload( uploadedFileIds: string[], channel: string, text: string, - accessToken: string + accessToken: string, + threadTs?: string | null ): Promise<{ ok: boolean; files?: any[]; error?: string }> { const response = await fetch('https://slack.com/api/files.completeUploadExternal', { method: 'POST', @@ -136,6 +151,7 @@ export async function completeSlackFileUpload( files: uploadedFileIds.map((id) => ({ id })), channel_id: channel, initial_comment: text, + ...(threadTs && { thread_ts: threadTs }), }), }) @@ -217,7 +233,13 @@ export async function sendSlackMessage( logger: Logger ): Promise<{ success: boolean - output?: { message: any; ts: string; channel: string; fileCount?: number } + output?: { + message: any + ts: string + channel: string + fileCount?: number + files?: ToolFileData[] + } error?: string }> { const { accessToken, text, threadTs, files } = params @@ -249,10 +271,15 @@ export async function sendSlackMessage( // Process files logger.info(`[${requestId}] Processing ${files.length} file(s)`) - const uploadedFileIds = await uploadFilesToSlack(files, accessToken, requestId, logger) + const { fileIds, files: uploadedFiles } = await uploadFilesToSlack( + files, + accessToken, + requestId, + logger + ) // No valid files uploaded - send text-only - if (uploadedFileIds.length === 0) { + if (fileIds.length === 0) { logger.warn(`[${requestId}] No valid files to upload, sending text-only message`) const data = await postSlackMessage(accessToken, channel, text, threadTs) @@ -264,8 +291,8 @@ export async function sendSlackMessage( return { success: true, output: formatMessageSuccessResponse(data, text) } } - // Complete file upload - const completeData = await completeSlackFileUpload(uploadedFileIds, channel, text, accessToken) + // Complete file upload with thread support + const completeData = await completeSlackFileUpload(fileIds, channel, text, accessToken, threadTs) if (!completeData.ok) { logger.error(`[${requestId}] Failed to complete upload:`, completeData.error) @@ -282,7 +309,8 @@ export async function sendSlackMessage( message: fileMessage, ts: fileMessage.ts, channel, - fileCount: uploadedFileIds.length, + fileCount: fileIds.length, + files: uploadedFiles, }, } } diff --git a/apps/sim/app/api/tools/smtp/send/route.ts b/apps/sim/app/api/tools/smtp/send/route.ts index 910ae4368..ca2fdf41c 100644 --- a/apps/sim/app/api/tools/smtp/send/route.ts +++ b/apps/sim/app/api/tools/smtp/send/route.ts @@ -4,6 +4,7 @@ import nodemailer from 'nodemailer' import { z } from 'zod' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { RawFileInputArraySchema } from '@/lib/uploads/utils/file-schemas' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -28,7 +29,7 @@ const SmtpSendSchema = z.object({ cc: z.string().optional().nullable(), bcc: z.string().optional().nullable(), replyTo: z.string().optional().nullable(), - attachments: z.array(z.any()).optional().nullable(), + attachments: RawFileInputArraySchema.optional().nullable(), }) export async function POST(request: NextRequest) { diff --git a/apps/sim/app/api/tools/ssh/download-file/route.ts b/apps/sim/app/api/tools/ssh/download-file/route.ts index e3bffd29d..cd908a1b9 100644 --- a/apps/sim/app/api/tools/ssh/download-file/route.ts +++ b/apps/sim/app/api/tools/ssh/download-file/route.ts @@ -5,6 +5,7 @@ import { type NextRequest, NextResponse } from 'next/server' import type { Client, SFTPWrapper } from 'ssh2' import { z } from 'zod' import { checkInternalAuth } from '@/lib/auth/hybrid' +import { getFileExtension, getMimeTypeFromExtension } from '@/lib/uploads/utils/file-utils' import { createSSHConnection, sanitizePath } from '@/app/api/tools/ssh/utils' const logger = createLogger('SSHDownloadFileAPI') @@ -79,6 +80,16 @@ export async function POST(request: NextRequest) { }) }) + // Check file size limit (50MB to prevent memory exhaustion) + const maxSize = 50 * 1024 * 1024 + if (stats.size > maxSize) { + const sizeMB = (stats.size / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { error: `File size (${sizeMB}MB) exceeds download limit of 50MB` }, + { status: 400 } + ) + } + // Read file content const content = await new Promise((resolve, reject) => { const chunks: Buffer[] = [] @@ -96,6 +107,8 @@ export async function POST(request: NextRequest) { }) const fileName = path.basename(remotePath) + const extension = getFileExtension(fileName) + const mimeType = getMimeTypeFromExtension(extension) // Encode content as base64 for binary safety const base64Content = content.toString('base64') @@ -104,6 +117,12 @@ export async function POST(request: NextRequest) { return NextResponse.json({ downloaded: true, + file: { + name: fileName, + mimeType, + data: base64Content, + size: stats.size, + }, content: base64Content, fileName: fileName, remotePath: remotePath, diff --git a/apps/sim/app/api/tools/stagehand/agent/route.ts b/apps/sim/app/api/tools/stagehand/agent/route.ts index f8cddf143..0d6f69765 100644 --- a/apps/sim/app/api/tools/stagehand/agent/route.ts +++ b/apps/sim/app/api/tools/stagehand/agent/route.ts @@ -3,6 +3,7 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkInternalAuth } from '@/lib/auth/hybrid' import { env } from '@/lib/core/config/env' +import { validateUrlWithDNS } from '@/lib/core/security/input-validation.server' import { isSensitiveKey, REDACTED_MARKER } from '@/lib/core/security/redaction' import { ensureZodObject, normalizeUrl } from '@/app/api/tools/stagehand/utils' @@ -123,6 +124,10 @@ export async function POST(request: NextRequest) { const variablesObject = processVariables(params.variables) const startUrl = normalizeUrl(rawStartUrl) + const urlValidation = await validateUrlWithDNS(startUrl, 'startUrl') + if (!urlValidation.isValid) { + return NextResponse.json({ error: urlValidation.error }, { status: 400 }) + } logger.info('Starting Stagehand agent process', { rawStartUrl, diff --git a/apps/sim/app/api/tools/stagehand/extract/route.ts b/apps/sim/app/api/tools/stagehand/extract/route.ts index b663f575d..8523db6c7 100644 --- a/apps/sim/app/api/tools/stagehand/extract/route.ts +++ b/apps/sim/app/api/tools/stagehand/extract/route.ts @@ -3,6 +3,7 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkInternalAuth } from '@/lib/auth/hybrid' import { env } from '@/lib/core/config/env' +import { validateUrlWithDNS } from '@/lib/core/security/input-validation.server' import { ensureZodObject, normalizeUrl } from '@/app/api/tools/stagehand/utils' const logger = createLogger('StagehandExtractAPI') @@ -51,6 +52,10 @@ export async function POST(request: NextRequest) { const params = validationResult.data const { url: rawUrl, instruction, selector, provider, apiKey, schema } = params const url = normalizeUrl(rawUrl) + const urlValidation = await validateUrlWithDNS(url, 'url') + if (!urlValidation.isValid) { + return NextResponse.json({ error: urlValidation.error }, { status: 400 }) + } logger.info('Starting Stagehand extraction process', { rawUrl, diff --git a/apps/sim/app/api/tools/stt/route.ts b/apps/sim/app/api/tools/stt/route.ts index 8a3ed3ef2..1317d8453 100644 --- a/apps/sim/app/api/tools/stt/route.ts +++ b/apps/sim/app/api/tools/stt/route.ts @@ -2,7 +2,15 @@ import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' import { extractAudioFromVideo, isVideoFile } from '@/lib/audio/extractor' import { checkInternalAuth } from '@/lib/auth/hybrid' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { getMimeTypeFromExtension, isInternalFileUrl } from '@/lib/uploads/utils/file-utils' +import { + downloadFileFromStorage, + resolveInternalFileUrl, +} from '@/lib/uploads/utils/file-utils.server' import type { UserFile } from '@/executor/types' import type { TranscriptSegment } from '@/tools/stt/types' @@ -45,6 +53,7 @@ export async function POST(request: NextRequest) { return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) } + const userId = authResult.userId const body: SttRequestBody = await request.json() const { provider, @@ -72,13 +81,25 @@ export async function POST(request: NextRequest) { let audioMimeType: string if (body.audioFile) { + if (Array.isArray(body.audioFile) && body.audioFile.length !== 1) { + return NextResponse.json({ error: 'audioFile must be a single file' }, { status: 400 }) + } const file = Array.isArray(body.audioFile) ? body.audioFile[0] : body.audioFile logger.info(`[${requestId}] Processing uploaded file: ${file.name}`) audioBuffer = await downloadFileFromStorage(file, requestId, logger) audioFileName = file.name - audioMimeType = file.type + // file.type may be missing if the file came from a block that doesn't preserve it + // Infer from filename extension as fallback + const ext = file.name.split('.').pop()?.toLowerCase() || '' + audioMimeType = file.type || getMimeTypeFromExtension(ext) } else if (body.audioFileReference) { + if (Array.isArray(body.audioFileReference) && body.audioFileReference.length !== 1) { + return NextResponse.json( + { error: 'audioFileReference must be a single file' }, + { status: 400 } + ) + } const file = Array.isArray(body.audioFileReference) ? body.audioFileReference[0] : body.audioFileReference @@ -86,18 +107,54 @@ export async function POST(request: NextRequest) { audioBuffer = await downloadFileFromStorage(file, requestId, logger) audioFileName = file.name - audioMimeType = file.type + + const ext = file.name.split('.').pop()?.toLowerCase() || '' + audioMimeType = file.type || getMimeTypeFromExtension(ext) } else if (body.audioUrl) { logger.info(`[${requestId}] Downloading from URL: ${body.audioUrl}`) - const response = await fetch(body.audioUrl) + let audioUrl = body.audioUrl.trim() + if (audioUrl.startsWith('/') && !isInternalFileUrl(audioUrl)) { + return NextResponse.json( + { + error: 'Invalid file path. Only uploaded files are supported for internal paths.', + }, + { status: 400 } + ) + } + + if (isInternalFileUrl(audioUrl)) { + if (!userId) { + return NextResponse.json( + { error: 'Authentication required for internal file access' }, + { status: 401 } + ) + } + const resolution = await resolveInternalFileUrl(audioUrl, userId, requestId, logger) + if (resolution.error) { + return NextResponse.json( + { error: resolution.error.message }, + { status: resolution.error.status } + ) + } + audioUrl = resolution.fileUrl || audioUrl + } + + const urlValidation = await validateUrlWithDNS(audioUrl, 'audioUrl') + if (!urlValidation.isValid) { + return NextResponse.json({ error: urlValidation.error }, { status: 400 }) + } + + const response = await secureFetchWithPinnedIP(audioUrl, urlValidation.resolvedIP!, { + method: 'GET', + }) if (!response.ok) { throw new Error(`Failed to download audio from URL: ${response.statusText}`) } const arrayBuffer = await response.arrayBuffer() audioBuffer = Buffer.from(arrayBuffer) - audioFileName = body.audioUrl.split('/').pop() || 'audio_file' + audioFileName = audioUrl.split('/').pop() || 'audio_file' audioMimeType = response.headers.get('content-type') || 'audio/mpeg' } else { return NextResponse.json( @@ -149,7 +206,9 @@ export async function POST(request: NextRequest) { translateToEnglish, model, body.prompt, - body.temperature + body.temperature, + audioMimeType, + audioFileName ) transcript = result.transcript segments = result.segments @@ -162,7 +221,8 @@ export async function POST(request: NextRequest) { language, timestamps, diarization, - model + model, + audioMimeType ) transcript = result.transcript segments = result.segments @@ -252,7 +312,9 @@ async function transcribeWithWhisper( translate?: boolean, model?: string, prompt?: string, - temperature?: number + temperature?: number, + mimeType?: string, + fileName?: string ): Promise<{ transcript: string segments?: TranscriptSegment[] @@ -261,8 +323,11 @@ async function transcribeWithWhisper( }> { const formData = new FormData() - const blob = new Blob([new Uint8Array(audioBuffer)], { type: 'audio/mpeg' }) - formData.append('file', blob, 'audio.mp3') + // Use actual MIME type and filename if provided + const actualMimeType = mimeType || 'audio/mpeg' + const actualFileName = fileName || 'audio.mp3' + const blob = new Blob([new Uint8Array(audioBuffer)], { type: actualMimeType }) + formData.append('file', blob, actualFileName) formData.append('model', model || 'whisper-1') if (language && language !== 'auto') { @@ -279,10 +344,11 @@ async function transcribeWithWhisper( formData.append('response_format', 'verbose_json') + // OpenAI API uses array notation for timestamp_granularities if (timestamps === 'word') { - formData.append('timestamp_granularities', 'word') + formData.append('timestamp_granularities[]', 'word') } else if (timestamps === 'sentence') { - formData.append('timestamp_granularities', 'segment') + formData.append('timestamp_granularities[]', 'segment') } const endpoint = translate ? 'translations' : 'transcriptions' @@ -325,7 +391,8 @@ async function transcribeWithDeepgram( language?: string, timestamps?: 'none' | 'sentence' | 'word', diarization?: boolean, - model?: string + model?: string, + mimeType?: string ): Promise<{ transcript: string segments?: TranscriptSegment[] @@ -357,7 +424,7 @@ async function transcribeWithDeepgram( method: 'POST', headers: { Authorization: `Token ${apiKey}`, - 'Content-Type': 'audio/mpeg', + 'Content-Type': mimeType || 'audio/mpeg', }, body: new Uint8Array(audioBuffer), }) @@ -513,7 +580,8 @@ async function transcribeWithAssemblyAI( audio_url: upload_url, } - if (model === 'best' || model === 'nano') { + // AssemblyAI supports 'best', 'slam-1', or 'universal' for speech_model + if (model === 'best' || model === 'slam-1' || model === 'universal') { transcriptRequest.speech_model = model } diff --git a/apps/sim/app/api/tools/supabase/storage-upload/route.ts b/apps/sim/app/api/tools/supabase/storage-upload/route.ts index 46122fc19..c0677bb35 100644 --- a/apps/sim/app/api/tools/supabase/storage-upload/route.ts +++ b/apps/sim/app/api/tools/supabase/storage-upload/route.ts @@ -3,6 +3,7 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -16,7 +17,7 @@ const SupabaseStorageUploadSchema = z.object({ bucket: z.string().min(1, 'Bucket name is required'), fileName: z.string().min(1, 'File name is required'), path: z.string().optional().nullable(), - fileData: z.any(), + fileData: FileInputSchema, contentType: z.string().optional().nullable(), upsert: z.boolean().optional().default(false), }) diff --git a/apps/sim/app/api/tools/telegram/send-document/route.ts b/apps/sim/app/api/tools/telegram/send-document/route.ts index 8435ee68f..0ddaac702 100644 --- a/apps/sim/app/api/tools/telegram/send-document/route.ts +++ b/apps/sim/app/api/tools/telegram/send-document/route.ts @@ -3,6 +3,7 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { RawFileInputArraySchema } from '@/lib/uploads/utils/file-schemas' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { convertMarkdownToHTML } from '@/tools/telegram/utils' @@ -14,7 +15,7 @@ const logger = createLogger('TelegramSendDocumentAPI') const TelegramSendDocumentSchema = z.object({ botToken: z.string().min(1, 'Bot token is required'), chatId: z.string().min(1, 'Chat ID is required'), - files: z.array(z.any()).optional().nullable(), + files: RawFileInputArraySchema.optional().nullable(), caption: z.string().optional().nullable(), }) @@ -93,6 +94,14 @@ export async function POST(request: NextRequest) { logger.info(`[${requestId}] Uploading document: ${userFile.name}`) const buffer = await downloadFileFromStorage(userFile, requestId, logger) + const filesOutput = [ + { + name: userFile.name, + mimeType: userFile.type || 'application/octet-stream', + data: buffer.toString('base64'), + size: buffer.length, + }, + ] logger.info(`[${requestId}] Downloaded file: ${buffer.length} bytes`) @@ -135,6 +144,7 @@ export async function POST(request: NextRequest) { output: { message: 'Document sent successfully', data: data.result, + files: filesOutput, }, }) } catch (error) { diff --git a/apps/sim/app/api/tools/textract/parse/route.ts b/apps/sim/app/api/tools/textract/parse/route.ts index 86fa83512..ad19aeb95 100644 --- a/apps/sim/app/api/tools/textract/parse/route.ts +++ b/apps/sim/app/api/tools/textract/parse/route.ts @@ -3,19 +3,18 @@ import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkInternalAuth } from '@/lib/auth/hybrid' +import { validateAwsRegion, validateS3BucketName } from '@/lib/core/security/input-validation' import { - validateAwsRegion, - validateExternalUrl, - validateS3BucketName, -} from '@/lib/core/security/input-validation' + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' -import { StorageService } from '@/lib/uploads' +import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' +import { isInternalFileUrl, processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' import { - extractStorageKey, - inferContextFromKey, - isInternalFileUrl, -} from '@/lib/uploads/utils/file-utils' -import { verifyFileAccess } from '@/app/api/files/authorization' + downloadFileFromStorage, + resolveInternalFileUrl, +} from '@/lib/uploads/utils/file-utils.server' export const dynamic = 'force-dynamic' export const maxDuration = 300 // 5 minutes for large multi-page PDF processing @@ -35,6 +34,7 @@ const TextractParseSchema = z region: z.string().min(1, 'AWS region is required'), processingMode: z.enum(['sync', 'async']).optional().default('sync'), filePath: z.string().optional(), + file: RawFileInputSchema.optional(), s3Uri: z.string().optional(), featureTypes: z .array(z.enum(['TABLES', 'FORMS', 'QUERIES', 'SIGNATURES', 'LAYOUT'])) @@ -50,6 +50,20 @@ const TextractParseSchema = z path: ['region'], }) } + if (data.processingMode === 'async' && !data.s3Uri) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'S3 URI is required for multi-page processing (s3://bucket/key)', + path: ['s3Uri'], + }) + } + if (data.processingMode !== 'async' && !data.file && !data.filePath) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'File input is required for single-page processing', + path: ['filePath'], + }) + } }) function getSignatureKey( @@ -111,7 +125,14 @@ function signAwsRequest( } async function fetchDocumentBytes(url: string): Promise<{ bytes: string; contentType: string }> { - const response = await fetch(url) + const urlValidation = await validateUrlWithDNS(url, 'Document URL') + if (!urlValidation.isValid) { + throw new Error(urlValidation.error || 'Invalid document URL') + } + + const response = await secureFetchWithPinnedIP(url, urlValidation.resolvedIP!, { + method: 'GET', + }) if (!response.ok) { throw new Error(`Failed to fetch document: ${response.statusText}`) } @@ -318,8 +339,8 @@ export async function POST(request: NextRequest) { logger.info(`[${requestId}] Textract parse request`, { processingMode, - filePath: validatedData.filePath?.substring(0, 50), - s3Uri: validatedData.s3Uri?.substring(0, 50), + hasFile: Boolean(validatedData.file), + hasS3Uri: Boolean(validatedData.s3Uri), featureTypes, userId, }) @@ -414,90 +435,89 @@ export async function POST(request: NextRequest) { }) } - if (!validatedData.filePath) { - return NextResponse.json( - { - success: false, - error: 'File path is required for single-page processing', - }, - { status: 400 } - ) - } + let bytes = '' + let contentType = 'application/octet-stream' + let isPdf = false - let fileUrl = validatedData.filePath - - const isInternalFilePath = validatedData.filePath && isInternalFileUrl(validatedData.filePath) - - if (isInternalFilePath) { + if (validatedData.file) { + let userFile try { - const storageKey = extractStorageKey(validatedData.filePath) - const context = inferContextFromKey(storageKey) - - const hasAccess = await verifyFileAccess(storageKey, userId, undefined, context, false) - - if (!hasAccess) { - logger.warn(`[${requestId}] Unauthorized presigned URL generation attempt`, { - userId, - key: storageKey, - context, - }) - return NextResponse.json( - { - success: false, - error: 'File not found', - }, - { status: 404 } - ) - } - - fileUrl = await StorageService.generatePresignedDownloadUrl(storageKey, context, 5 * 60) - logger.info(`[${requestId}] Generated presigned URL for ${context} file`) + userFile = processSingleFileToUserFile(validatedData.file, requestId, logger) } catch (error) { - logger.error(`[${requestId}] Failed to generate presigned URL:`, error) return NextResponse.json( { success: false, - error: 'Failed to generate file access URL', - }, - { status: 500 } - ) - } - } else if (validatedData.filePath?.startsWith('/')) { - // Reject arbitrary absolute paths that don't contain /api/files/serve/ - logger.warn(`[${requestId}] Invalid internal path`, { - userId, - path: validatedData.filePath.substring(0, 50), - }) - return NextResponse.json( - { - success: false, - error: 'Invalid file path. Only uploaded files are supported for internal paths.', - }, - { status: 400 } - ) - } else { - const urlValidation = validateExternalUrl(fileUrl, 'Document URL') - if (!urlValidation.isValid) { - logger.warn(`[${requestId}] SSRF attempt blocked`, { - userId, - url: fileUrl.substring(0, 100), - error: urlValidation.error, - }) - return NextResponse.json( - { - success: false, - error: urlValidation.error, + error: error instanceof Error ? error.message : 'Failed to process file', }, { status: 400 } ) } + + const buffer = await downloadFileFromStorage(userFile, requestId, logger) + bytes = buffer.toString('base64') + contentType = userFile.type || 'application/octet-stream' + isPdf = contentType.includes('pdf') || userFile.name?.toLowerCase().endsWith('.pdf') + } else if (validatedData.filePath) { + let fileUrl = validatedData.filePath + + const isInternalFilePath = isInternalFileUrl(fileUrl) + + if (isInternalFilePath) { + const resolution = await resolveInternalFileUrl(fileUrl, userId, requestId, logger) + if (resolution.error) { + return NextResponse.json( + { + success: false, + error: resolution.error.message, + }, + { status: resolution.error.status } + ) + } + fileUrl = resolution.fileUrl || fileUrl + } else if (fileUrl.startsWith('/')) { + logger.warn(`[${requestId}] Invalid internal path`, { + userId, + path: fileUrl.substring(0, 50), + }) + return NextResponse.json( + { + success: false, + error: 'Invalid file path. Only uploaded files are supported for internal paths.', + }, + { status: 400 } + ) + } else { + const urlValidation = await validateUrlWithDNS(fileUrl, 'Document URL') + if (!urlValidation.isValid) { + logger.warn(`[${requestId}] SSRF attempt blocked`, { + userId, + url: fileUrl.substring(0, 100), + error: urlValidation.error, + }) + return NextResponse.json( + { + success: false, + error: urlValidation.error, + }, + { status: 400 } + ) + } + } + + const fetched = await fetchDocumentBytes(fileUrl) + bytes = fetched.bytes + contentType = fetched.contentType + isPdf = contentType.includes('pdf') || fileUrl.toLowerCase().endsWith('.pdf') + } else { + return NextResponse.json( + { + success: false, + error: 'File input is required for single-page processing', + }, + { status: 400 } + ) } - const { bytes, contentType } = await fetchDocumentBytes(fileUrl) - - // Track if this is a PDF for better error messaging - const isPdf = contentType.includes('pdf') || fileUrl.toLowerCase().endsWith('.pdf') - const uri = '/' let textractBody: Record diff --git a/apps/sim/app/api/tools/twilio/get-recording/route.ts b/apps/sim/app/api/tools/twilio/get-recording/route.ts new file mode 100644 index 000000000..b5562307e --- /dev/null +++ b/apps/sim/app/api/tools/twilio/get-recording/route.ts @@ -0,0 +1,250 @@ +import { createLogger } from '@sim/logger' +import { type NextRequest, NextResponse } from 'next/server' +import { z } from 'zod' +import { checkInternalAuth } from '@/lib/auth/hybrid' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { generateRequestId } from '@/lib/core/utils/request' +import { getExtensionFromMimeType } from '@/lib/uploads/utils/file-utils' + +export const dynamic = 'force-dynamic' + +const logger = createLogger('TwilioGetRecordingAPI') + +interface TwilioRecordingResponse { + sid?: string + call_sid?: string + duration?: string + status?: string + channels?: number + source?: string + price?: string + price_unit?: string + uri?: string + error_code?: number + message?: string + error_message?: string +} + +interface TwilioErrorResponse { + message?: string +} + +interface TwilioTranscription { + transcription_text?: string + status?: string + price?: string + price_unit?: string +} + +interface TwilioTranscriptionsResponse { + transcriptions?: TwilioTranscription[] +} + +const TwilioGetRecordingSchema = z.object({ + accountSid: z.string().min(1, 'Account SID is required'), + authToken: z.string().min(1, 'Auth token is required'), + recordingSid: z.string().min(1, 'Recording SID is required'), +}) + +export async function POST(request: NextRequest) { + const requestId = generateRequestId() + + try { + const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) + + if (!authResult.success) { + logger.warn(`[${requestId}] Unauthorized Twilio get recording attempt: ${authResult.error}`) + return NextResponse.json( + { + success: false, + error: authResult.error || 'Authentication required', + }, + { status: 401 } + ) + } + + const body = await request.json() + const validatedData = TwilioGetRecordingSchema.parse(body) + + const { accountSid, authToken, recordingSid } = validatedData + + if (!accountSid.startsWith('AC')) { + return NextResponse.json( + { + success: false, + error: `Invalid Account SID format. Account SID must start with "AC" (you provided: ${accountSid.substring(0, 2)}...)`, + }, + { status: 400 } + ) + } + + const twilioAuth = Buffer.from(`${accountSid}:${authToken}`).toString('base64') + + logger.info(`[${requestId}] Getting recording info from Twilio`, { recordingSid }) + + const infoUrl = `https://api.twilio.com/2010-04-01/Accounts/${accountSid}/Recordings/${recordingSid}.json` + const infoUrlValidation = await validateUrlWithDNS(infoUrl, 'infoUrl') + if (!infoUrlValidation.isValid) { + return NextResponse.json({ success: false, error: infoUrlValidation.error }, { status: 400 }) + } + + const infoResponse = await secureFetchWithPinnedIP(infoUrl, infoUrlValidation.resolvedIP!, { + method: 'GET', + headers: { Authorization: `Basic ${twilioAuth}` }, + }) + + if (!infoResponse.ok) { + const errorData = (await infoResponse.json().catch(() => ({}))) as TwilioErrorResponse + logger.error(`[${requestId}] Twilio API error`, { + status: infoResponse.status, + error: errorData, + }) + return NextResponse.json( + { success: false, error: errorData.message || `Twilio API error: ${infoResponse.status}` }, + { status: 400 } + ) + } + + const data = (await infoResponse.json()) as TwilioRecordingResponse + + if (data.error_code) { + return NextResponse.json({ + success: false, + output: { + success: false, + error: data.message || data.error_message || 'Failed to retrieve recording', + }, + error: data.message || data.error_message || 'Failed to retrieve recording', + }) + } + + const baseUrl = 'https://api.twilio.com' + const mediaUrl = data.uri ? `${baseUrl}${data.uri.replace('.json', '')}` : undefined + + let transcriptionText: string | undefined + let transcriptionStatus: string | undefined + let transcriptionPrice: string | undefined + let transcriptionPriceUnit: string | undefined + let file: + | { + name: string + mimeType: string + data: string + size: number + } + | undefined + + try { + const transcriptionUrl = `https://api.twilio.com/2010-04-01/Accounts/${accountSid}/Transcriptions.json?RecordingSid=${data.sid}` + logger.info(`[${requestId}] Checking for transcriptions`) + + const transcriptionUrlValidation = await validateUrlWithDNS( + transcriptionUrl, + 'transcriptionUrl' + ) + if (transcriptionUrlValidation.isValid) { + const transcriptionResponse = await secureFetchWithPinnedIP( + transcriptionUrl, + transcriptionUrlValidation.resolvedIP!, + { + method: 'GET', + headers: { Authorization: `Basic ${twilioAuth}` }, + } + ) + + if (transcriptionResponse.ok) { + const transcriptionData = + (await transcriptionResponse.json()) as TwilioTranscriptionsResponse + + if (transcriptionData.transcriptions && transcriptionData.transcriptions.length > 0) { + const transcription = transcriptionData.transcriptions[0] + transcriptionText = transcription.transcription_text + transcriptionStatus = transcription.status + transcriptionPrice = transcription.price + transcriptionPriceUnit = transcription.price_unit + logger.info(`[${requestId}] Transcription found`, { + status: transcriptionStatus, + textLength: transcriptionText?.length, + }) + } + } + } + } catch (error) { + logger.warn(`[${requestId}] Failed to fetch transcription:`, error) + } + + if (mediaUrl) { + try { + const mediaUrlValidation = await validateUrlWithDNS(mediaUrl, 'mediaUrl') + if (mediaUrlValidation.isValid) { + const mediaResponse = await secureFetchWithPinnedIP( + mediaUrl, + mediaUrlValidation.resolvedIP!, + { + method: 'GET', + headers: { Authorization: `Basic ${twilioAuth}` }, + } + ) + + if (mediaResponse.ok) { + const contentType = + mediaResponse.headers.get('content-type') || 'application/octet-stream' + const extension = getExtensionFromMimeType(contentType) || 'dat' + const arrayBuffer = await mediaResponse.arrayBuffer() + const buffer = Buffer.from(arrayBuffer) + const fileName = `${data.sid || recordingSid}.${extension}` + + file = { + name: fileName, + mimeType: contentType, + data: buffer.toString('base64'), + size: buffer.length, + } + } + } + } catch (error) { + logger.warn(`[${requestId}] Failed to download recording media:`, error) + } + } + + logger.info(`[${requestId}] Twilio recording fetched successfully`, { + recordingSid: data.sid, + hasFile: !!file, + hasTranscription: !!transcriptionText, + }) + + return NextResponse.json({ + success: true, + output: { + success: true, + recordingSid: data.sid, + callSid: data.call_sid, + duration: data.duration ? Number.parseInt(data.duration, 10) : undefined, + status: data.status, + channels: data.channels, + source: data.source, + mediaUrl, + file, + price: data.price, + priceUnit: data.price_unit, + uri: data.uri, + transcriptionText, + transcriptionStatus, + transcriptionPrice, + transcriptionPriceUnit, + }, + }) + } catch (error) { + logger.error(`[${requestId}] Error fetching Twilio recording:`, error) + return NextResponse.json( + { + success: false, + error: error instanceof Error ? error.message : 'Unknown error occurred', + }, + { status: 500 } + ) + } +} diff --git a/apps/sim/app/api/tools/vision/analyze/route.ts b/apps/sim/app/api/tools/vision/analyze/route.ts index 165005142..684094b2b 100644 --- a/apps/sim/app/api/tools/vision/analyze/route.ts +++ b/apps/sim/app/api/tools/vision/analyze/route.ts @@ -1,10 +1,20 @@ +import { GoogleGenAI } from '@google/genai' import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkInternalAuth } from '@/lib/auth/hybrid' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' -import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' +import { isInternalFileUrl, processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' +import { + downloadFileFromStorage, + resolveInternalFileUrl, +} from '@/lib/uploads/utils/file-utils.server' +import { convertUsageMetadata, extractTextContent } from '@/providers/google/utils' export const dynamic = 'force-dynamic' @@ -13,8 +23,8 @@ const logger = createLogger('VisionAnalyzeAPI') const VisionAnalyzeSchema = z.object({ apiKey: z.string().min(1, 'API key is required'), imageUrl: z.string().optional().nullable(), - imageFile: z.any().optional().nullable(), - model: z.string().optional().default('gpt-4o'), + imageFile: RawFileInputSchema.optional().nullable(), + model: z.string().optional().default('gpt-5.2'), prompt: z.string().optional().nullable(), }) @@ -39,6 +49,7 @@ export async function POST(request: NextRequest) { userId: authResult.userId, }) + const userId = authResult.userId const body = await request.json() const validatedData = VisionAnalyzeSchema.parse(body) @@ -77,18 +88,72 @@ export async function POST(request: NextRequest) { ) } - const buffer = await downloadFileFromStorage(userFile, requestId, logger) - - const base64 = buffer.toString('base64') + let base64 = userFile.base64 + let bufferLength = 0 + if (!base64) { + const buffer = await downloadFileFromStorage(userFile, requestId, logger) + base64 = buffer.toString('base64') + bufferLength = buffer.length + } const mimeType = userFile.type || 'image/jpeg' imageSource = `data:${mimeType};base64,${base64}` - logger.info(`[${requestId}] Converted image to base64 (${buffer.length} bytes)`) + if (bufferLength > 0) { + logger.info(`[${requestId}] Converted image to base64 (${bufferLength} bytes)`) + } + } + + let imageUrlValidation: Awaited> | null = null + if (imageSource && !imageSource.startsWith('data:')) { + if (imageSource.startsWith('/') && !isInternalFileUrl(imageSource)) { + return NextResponse.json( + { + success: false, + error: 'Invalid file path. Only uploaded files are supported for internal paths.', + }, + { status: 400 } + ) + } + + if (isInternalFileUrl(imageSource)) { + if (!userId) { + return NextResponse.json( + { + success: false, + error: 'Authentication required for internal file access', + }, + { status: 401 } + ) + } + const resolution = await resolveInternalFileUrl(imageSource, userId, requestId, logger) + if (resolution.error) { + return NextResponse.json( + { + success: false, + error: resolution.error.message, + }, + { status: resolution.error.status } + ) + } + imageSource = resolution.fileUrl || imageSource + } + + imageUrlValidation = await validateUrlWithDNS(imageSource, 'imageUrl') + if (!imageUrlValidation.isValid) { + return NextResponse.json( + { + success: false, + error: imageUrlValidation.error, + }, + { status: 400 } + ) + } } const defaultPrompt = 'Please analyze this image and describe what you see in detail.' const prompt = validatedData.prompt || defaultPrompt - const isClaude = validatedData.model.startsWith('claude-3') + const isClaude = validatedData.model.startsWith('claude-') + const isGemini = validatedData.model.startsWith('gemini-') const apiUrl = isClaude ? 'https://api.anthropic.com/v1/messages' : 'https://api.openai.com/v1/chat/completions' @@ -106,6 +171,72 @@ export async function POST(request: NextRequest) { let requestBody: any + if (isGemini) { + let base64Payload = imageSource + if (!base64Payload.startsWith('data:')) { + const urlValidation = + imageUrlValidation || (await validateUrlWithDNS(base64Payload, 'imageUrl')) + if (!urlValidation.isValid) { + return NextResponse.json({ success: false, error: urlValidation.error }, { status: 400 }) + } + + const response = await secureFetchWithPinnedIP(base64Payload, urlValidation.resolvedIP!, { + method: 'GET', + }) + if (!response.ok) { + return NextResponse.json( + { success: false, error: 'Failed to fetch image for Gemini' }, + { status: 400 } + ) + } + const contentType = + response.headers.get('content-type') || validatedData.imageFile?.type || 'image/jpeg' + const arrayBuffer = await response.arrayBuffer() + const base64 = Buffer.from(arrayBuffer).toString('base64') + base64Payload = `data:${contentType};base64,${base64}` + } + const base64Marker = ';base64,' + const markerIndex = base64Payload.indexOf(base64Marker) + if (!base64Payload.startsWith('data:') || markerIndex === -1) { + return NextResponse.json( + { success: false, error: 'Invalid base64 image format' }, + { status: 400 } + ) + } + const rawMimeType = base64Payload.slice('data:'.length, markerIndex) + const mediaType = rawMimeType.split(';')[0] || 'image/jpeg' + const base64Data = base64Payload.slice(markerIndex + base64Marker.length) + if (!base64Data) { + return NextResponse.json( + { success: false, error: 'Invalid base64 image format' }, + { status: 400 } + ) + } + + const ai = new GoogleGenAI({ apiKey: validatedData.apiKey }) + const geminiResponse = await ai.models.generateContent({ + model: validatedData.model, + contents: [ + { + role: 'user', + parts: [{ text: prompt }, { inlineData: { mimeType: mediaType, data: base64Data } }], + }, + ], + }) + + const content = extractTextContent(geminiResponse.candidates?.[0]) + const usage = convertUsageMetadata(geminiResponse.usageMetadata) + + return NextResponse.json({ + success: true, + output: { + content, + model: validatedData.model, + tokens: usage.totalTokenCount || undefined, + }, + }) + } + if (isClaude) { if (imageSource.startsWith('data:')) { const base64Match = imageSource.match(/^data:([^;]+);base64,(.+)$/) @@ -172,7 +303,7 @@ export async function POST(request: NextRequest) { ], }, ], - max_tokens: 1000, + max_completion_tokens: 1000, } } diff --git a/apps/sim/app/api/tools/wordpress/upload/route.ts b/apps/sim/app/api/tools/wordpress/upload/route.ts index 8c2604bce..5cf9a1b6f 100644 --- a/apps/sim/app/api/tools/wordpress/upload/route.ts +++ b/apps/sim/app/api/tools/wordpress/upload/route.ts @@ -3,6 +3,7 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' import { getFileExtension, getMimeTypeFromExtension, @@ -19,7 +20,7 @@ const WORDPRESS_COM_API_BASE = 'https://public-api.wordpress.com/wp/v2/sites' const WordPressUploadSchema = z.object({ accessToken: z.string().min(1, 'Access token is required'), siteId: z.string().min(1, 'Site ID is required'), - file: z.any().optional().nullable(), + file: RawFileInputSchema.optional().nullable(), filename: z.string().optional().nullable(), title: z.string().optional().nullable(), caption: z.string().optional().nullable(), diff --git a/apps/sim/app/api/tools/zoom/get-recordings/route.ts b/apps/sim/app/api/tools/zoom/get-recordings/route.ts new file mode 100644 index 000000000..2247612fd --- /dev/null +++ b/apps/sim/app/api/tools/zoom/get-recordings/route.ts @@ -0,0 +1,216 @@ +import { createLogger } from '@sim/logger' +import { type NextRequest, NextResponse } from 'next/server' +import { z } from 'zod' +import { checkInternalAuth } from '@/lib/auth/hybrid' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { generateRequestId } from '@/lib/core/utils/request' +import { getExtensionFromMimeType } from '@/lib/uploads/utils/file-utils' + +export const dynamic = 'force-dynamic' + +const logger = createLogger('ZoomGetRecordingsAPI') + +interface ZoomRecordingFile { + id?: string + meeting_id?: string + recording_start?: string + recording_end?: string + file_type?: string + file_extension?: string + file_size?: number + play_url?: string + download_url?: string + status?: string + recording_type?: string +} + +interface ZoomRecordingsResponse { + uuid?: string + id?: string | number + account_id?: string + host_id?: string + topic?: string + type?: number + start_time?: string + duration?: number + total_size?: number + recording_count?: number + share_url?: string + recording_files?: ZoomRecordingFile[] +} + +interface ZoomErrorResponse { + message?: string + code?: number +} + +const ZoomGetRecordingsSchema = z.object({ + accessToken: z.string().min(1, 'Access token is required'), + meetingId: z.string().min(1, 'Meeting ID is required'), + includeFolderItems: z.boolean().optional(), + ttl: z.number().optional(), + downloadFiles: z.boolean().optional().default(false), +}) + +export async function POST(request: NextRequest) { + const requestId = generateRequestId() + + try { + const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) + + if (!authResult.success) { + logger.warn(`[${requestId}] Unauthorized Zoom get recordings attempt: ${authResult.error}`) + return NextResponse.json( + { + success: false, + error: authResult.error || 'Authentication required', + }, + { status: 401 } + ) + } + + const body = await request.json() + const validatedData = ZoomGetRecordingsSchema.parse(body) + + const { accessToken, meetingId, includeFolderItems, ttl, downloadFiles } = validatedData + + const baseUrl = `https://api.zoom.us/v2/meetings/${encodeURIComponent(meetingId)}/recordings` + const queryParams = new URLSearchParams() + + if (includeFolderItems != null) { + queryParams.append('include_folder_items', String(includeFolderItems)) + } + if (ttl) { + queryParams.append('ttl', String(ttl)) + } + + const queryString = queryParams.toString() + const apiUrl = queryString ? `${baseUrl}?${queryString}` : baseUrl + + logger.info(`[${requestId}] Fetching recordings from Zoom`, { meetingId }) + + const urlValidation = await validateUrlWithDNS(apiUrl, 'apiUrl') + if (!urlValidation.isValid) { + return NextResponse.json({ success: false, error: urlValidation.error }, { status: 400 }) + } + + const response = await secureFetchWithPinnedIP(apiUrl, urlValidation.resolvedIP!, { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${accessToken}`, + }, + }) + + if (!response.ok) { + const errorData = (await response.json().catch(() => ({}))) as ZoomErrorResponse + logger.error(`[${requestId}] Zoom API error`, { + status: response.status, + error: errorData, + }) + return NextResponse.json( + { success: false, error: errorData.message || `Zoom API error: ${response.status}` }, + { status: 400 } + ) + } + + const data = (await response.json()) as ZoomRecordingsResponse + const files: Array<{ + name: string + mimeType: string + data: string + size: number + }> = [] + + if (downloadFiles && Array.isArray(data.recording_files)) { + for (const file of data.recording_files) { + if (!file?.download_url) continue + + try { + const fileUrlValidation = await validateUrlWithDNS(file.download_url, 'downloadUrl') + if (!fileUrlValidation.isValid) continue + + const downloadResponse = await secureFetchWithPinnedIP( + file.download_url, + fileUrlValidation.resolvedIP!, + { + method: 'GET', + headers: { Authorization: `Bearer ${accessToken}` }, + } + ) + + if (!downloadResponse.ok) continue + + const contentType = + downloadResponse.headers.get('content-type') || 'application/octet-stream' + const arrayBuffer = await downloadResponse.arrayBuffer() + const buffer = Buffer.from(arrayBuffer) + const extension = + file.file_extension?.toString().toLowerCase() || + getExtensionFromMimeType(contentType) || + 'dat' + const fileName = `zoom-recording-${file.id || file.recording_start || Date.now()}.${extension}` + + files.push({ + name: fileName, + mimeType: contentType, + data: buffer.toString('base64'), + size: buffer.length, + }) + } catch (error) { + logger.warn(`[${requestId}] Failed to download recording file:`, error) + } + } + } + + logger.info(`[${requestId}] Zoom recordings fetched successfully`, { + recordingCount: data.recording_files?.length || 0, + downloadedCount: files.length, + }) + + return NextResponse.json({ + success: true, + output: { + recording: { + uuid: data.uuid, + id: data.id, + account_id: data.account_id, + host_id: data.host_id, + topic: data.topic, + type: data.type, + start_time: data.start_time, + duration: data.duration, + total_size: data.total_size, + recording_count: data.recording_count, + share_url: data.share_url, + recording_files: (data.recording_files || []).map((file: ZoomRecordingFile) => ({ + id: file.id, + meeting_id: file.meeting_id, + recording_start: file.recording_start, + recording_end: file.recording_end, + file_type: file.file_type, + file_extension: file.file_extension, + file_size: file.file_size, + play_url: file.play_url, + download_url: file.download_url, + status: file.status, + recording_type: file.recording_type, + })), + }, + files: files.length > 0 ? files : undefined, + }, + }) + } catch (error) { + logger.error(`[${requestId}] Error fetching Zoom recordings:`, error) + return NextResponse.json( + { + success: false, + error: error instanceof Error ? error.message : 'Unknown error occurred', + }, + { status: 500 } + ) + } +} diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/chat/chat.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/chat/chat.tsx index 1c1d468be..7b66b8369 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/chat/chat.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/chat/chat.tsx @@ -807,7 +807,7 @@ export function Chat() { const newReservedFields: StartInputFormatField[] = missingStartReservedFields.map( (fieldName) => { - const defaultType = fieldName === 'files' ? 'files' : 'string' + const defaultType = fieldName === 'files' ? 'file[]' : 'string' return { id: crypto.randomUUID(), diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/deploy/components/deploy-modal/components/a2a/a2a.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/deploy/components/deploy-modal/components/a2a/a2a.tsx index 86be4ba5e..50bcc9c6a 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/deploy/components/deploy-modal/components/a2a/a2a.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/deploy/components/deploy-modal/components/a2a/a2a.tsx @@ -179,7 +179,7 @@ export function A2aDeploy({ newFields.push({ id: crypto.randomUUID(), name: 'files', - type: 'files', + type: 'file[]', value: '', collapsed: false, }) diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/file-upload/file-upload.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/file-upload/file-upload.tsx index 234498d01..39ad880d9 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/file-upload/file-upload.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/file-upload/file-upload.tsx @@ -368,6 +368,7 @@ export function FileUpload({ const uploadedFile: UploadedFile = { name: selectedFile.name, path: selectedFile.path, + key: selectedFile.key, size: selectedFile.size, type: selectedFile.type, } diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/starter/input-format.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/starter/input-format.tsx index 8900c2318..f12ceb3e2 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/starter/input-format.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/starter/input-format.tsx @@ -26,7 +26,7 @@ import { useAccessibleReferencePrefixes } from '@/app/workspace/[workspaceId]/w/ interface Field { id: string name: string - type?: 'string' | 'number' | 'boolean' | 'object' | 'array' | 'files' + type?: 'string' | 'number' | 'boolean' | 'object' | 'array' | 'file[]' value?: string description?: string collapsed?: boolean @@ -57,7 +57,7 @@ const TYPE_OPTIONS: ComboboxOption[] = [ { label: 'Boolean', value: 'boolean' }, { label: 'Object', value: 'object' }, { label: 'Array', value: 'array' }, - { label: 'Files', value: 'files' }, + { label: 'Files', value: 'file[]' }, ] /** @@ -448,7 +448,7 @@ export function FieldFormat({ ) } - if (field.type === 'files') { + if (field.type === 'file[]') { const lineCount = fieldValue.split('\n').length const gutterWidth = calculateGutterWidth(lineCount) diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/tag-dropdown/tag-dropdown.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/tag-dropdown/tag-dropdown.tsx index f233fe025..fbec4fe0a 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/tag-dropdown/tag-dropdown.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/tag-dropdown/tag-dropdown.tsx @@ -225,7 +225,7 @@ const getOutputTypeForPath = ( const chatModeTypes: Record = { input: 'string', conversationId: 'string', - files: 'files', + files: 'file[]', } return chatModeTypes[outputPath] || 'any' } @@ -1568,16 +1568,11 @@ export const TagDropdown: React.FC = ({ blockTagGroups.sort((a, b) => a.distance - b.distance) finalBlockTagGroups.push(...blockTagGroups) - const contextualTags: string[] = [] - if (loopBlockGroup) { - contextualTags.push(...loopBlockGroup.tags) - } - if (parallelBlockGroup) { - contextualTags.push(...parallelBlockGroup.tags) - } + const groupTags = finalBlockTagGroups.flatMap((group) => group.tags) + const tags = [...groupTags, ...variableTags] return { - tags: [...allBlockTags, ...variableTags, ...contextualTags], + tags, variableInfoMap, blockTagGroups: finalBlockTagGroups, } @@ -1751,7 +1746,7 @@ export const TagDropdown: React.FC = ({ mergedSubBlocks ) - if (fieldType === 'files' || fieldType === 'file[]' || fieldType === 'array') { + if (fieldType === 'file' || fieldType === 'file[]' || fieldType === 'array') { const blockName = parts[0] const remainingPath = parts.slice(2).join('.') processedTag = `${blockName}.${arrayFieldName}[0].${remainingPath}` diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-block-output-fields.ts b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-block-output-fields.ts index af5f67529..233f06e58 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-block-output-fields.ts +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-block-output-fields.ts @@ -188,7 +188,7 @@ export function useBlockOutputFields({ baseOutputs = { input: { type: 'string', description: 'User message' }, conversationId: { type: 'string', description: 'Conversation ID' }, - files: { type: 'files', description: 'Uploaded files' }, + files: { type: 'file[]', description: 'Uploaded files' }, } } else { const inputFormatValue = mergedSubBlocks?.inputFormat?.value diff --git a/apps/sim/background/webhook-execution.ts b/apps/sim/background/webhook-execution.ts index e5e3d3007..40f60971b 100644 --- a/apps/sim/background/webhook-execution.ts +++ b/apps/sim/background/webhook-execution.ts @@ -417,11 +417,11 @@ async function executeWebhookJobInternal( if (triggerBlock?.subBlocks?.inputFormat?.value) { const inputFormat = triggerBlock.subBlocks.inputFormat.value as unknown as Array<{ name: string - type: 'string' | 'number' | 'boolean' | 'object' | 'array' | 'files' + type: 'string' | 'number' | 'boolean' | 'object' | 'array' | 'file[]' }> logger.debug(`[${requestId}] Processing generic webhook files from inputFormat`) - const fileFields = inputFormat.filter((field) => field.type === 'files') + const fileFields = inputFormat.filter((field) => field.type === 'file[]') if (fileFields.length > 0 && typeof input === 'object' && input !== null) { const executionContext = { diff --git a/apps/sim/blocks/blocks.test.ts b/apps/sim/blocks/blocks.test.ts index eb59b46c2..5790ca6d0 100644 --- a/apps/sim/blocks/blocks.test.ts +++ b/apps/sim/blocks/blocks.test.ts @@ -442,7 +442,16 @@ describe('Blocks Module', () => { }) it('should have valid output types', () => { - const validPrimitiveTypes = ['string', 'number', 'boolean', 'json', 'array', 'files', 'any'] + const validPrimitiveTypes = [ + 'string', + 'number', + 'boolean', + 'json', + 'array', + 'file', + 'file[]', + 'any', + ] const blocks = getAllBlocks() for (const block of blocks) { for (const [key, outputConfig] of Object.entries(block.outputs)) { diff --git a/apps/sim/blocks/blocks/a2a.ts b/apps/sim/blocks/blocks/a2a.ts index 86c98ac9b..7426ea917 100644 --- a/apps/sim/blocks/blocks/a2a.ts +++ b/apps/sim/blocks/blocks/a2a.ts @@ -1,5 +1,6 @@ import { A2AIcon } from '@/components/icons' import type { BlockConfig } from '@/blocks/types' +import { normalizeFileInput } from '@/blocks/utils' import type { ToolResponse } from '@/tools/types' export interface A2AResponse extends ToolResponse { @@ -214,6 +215,14 @@ export const A2ABlock: BlockConfig = { ], config: { tool: (params) => params.operation as string, + params: (params) => { + const { fileUpload, fileReference, ...rest } = params + const normalizedFiles = normalizeFileInput(fileUpload || fileReference || params.files) + return { + ...rest, + ...(normalizedFiles && { files: normalizedFiles }), + } + }, }, }, inputs: { diff --git a/apps/sim/blocks/blocks/chat_trigger.ts b/apps/sim/blocks/blocks/chat_trigger.ts index 2efb6612f..34fa5d0cc 100644 --- a/apps/sim/blocks/blocks/chat_trigger.ts +++ b/apps/sim/blocks/blocks/chat_trigger.ts @@ -26,7 +26,7 @@ export const ChatTriggerBlock: BlockConfig = { outputs: { input: { type: 'string', description: 'User message' }, conversationId: { type: 'string', description: 'Conversation ID' }, - files: { type: 'files', description: 'Uploaded files' }, + files: { type: 'file[]', description: 'Uploaded files' }, }, triggers: { enabled: true, diff --git a/apps/sim/blocks/blocks/confluence.ts b/apps/sim/blocks/blocks/confluence.ts index 5f9436f5c..5bdb21e5e 100644 --- a/apps/sim/blocks/blocks/confluence.ts +++ b/apps/sim/blocks/blocks/confluence.ts @@ -1,6 +1,7 @@ import { ConfluenceIcon } from '@/components/icons' import type { BlockConfig } from '@/blocks/types' import { AuthMode } from '@/blocks/types' +import { normalizeFileInput } from '@/blocks/utils' import type { ConfluenceResponse } from '@/tools/confluence/types' export const ConfluenceBlock: BlockConfig = { @@ -651,14 +652,15 @@ export const ConfluenceV2Block: BlockConfig = { if (operation === 'upload_attachment') { const fileInput = attachmentFileUpload || attachmentFileReference || attachmentFile - if (!fileInput) { + const normalizedFile = normalizeFileInput(fileInput, { single: true }) + if (!normalizedFile) { throw new Error('File is required for upload attachment operation.') } return { credential, pageId: effectivePageId, operation, - file: fileInput, + file: normalizedFile, fileName: attachmentFileName, comment: attachmentComment, ...rest, diff --git a/apps/sim/blocks/blocks/discord.ts b/apps/sim/blocks/blocks/discord.ts index 0d1108a09..79331eaac 100644 --- a/apps/sim/blocks/blocks/discord.ts +++ b/apps/sim/blocks/blocks/discord.ts @@ -1,6 +1,7 @@ import { DiscordIcon } from '@/components/icons' import type { BlockConfig } from '@/blocks/types' import { AuthMode } from '@/blocks/types' +import { normalizeFileInput } from '@/blocks/utils' import type { DiscordResponse } from '@/tools/discord/types' export const DiscordBlock: BlockConfig = { @@ -578,13 +579,14 @@ export const DiscordBlock: BlockConfig = { if (!params.serverId) throw new Error('Server ID is required') switch (params.operation) { - case 'discord_send_message': + case 'discord_send_message': { return { ...commonParams, channelId: params.channelId, content: params.content, - files: params.attachmentFiles || params.files, + files: normalizeFileInput(params.attachmentFiles || params.files), } + } case 'discord_get_messages': return { ...commonParams, @@ -789,6 +791,7 @@ export const DiscordBlock: BlockConfig = { }, outputs: { message: { type: 'string', description: 'Status message' }, + files: { type: 'file[]', description: 'Files attached to the message' }, data: { type: 'json', description: 'Response data' }, }, } diff --git a/apps/sim/blocks/blocks/dropbox.ts b/apps/sim/blocks/blocks/dropbox.ts index cffc5ac84..e7127c118 100644 --- a/apps/sim/blocks/blocks/dropbox.ts +++ b/apps/sim/blocks/blocks/dropbox.ts @@ -1,6 +1,7 @@ import { DropboxIcon } from '@/components/icons' import type { BlockConfig } from '@/blocks/types' import { AuthMode } from '@/blocks/types' +import { normalizeFileInput } from '@/blocks/utils' import type { DropboxResponse } from '@/tools/dropbox/types' export const DropboxBlock: BlockConfig = { @@ -60,12 +61,25 @@ export const DropboxBlock: BlockConfig = { required: true, }, { - id: 'fileContent', - title: 'File Content', - type: 'long-input', - placeholder: 'Base64 encoded file content or file reference', - condition: { field: 'operation', value: 'dropbox_upload' }, + id: 'uploadFile', + title: 'File', + type: 'file-upload', + canonicalParamId: 'file', + placeholder: 'Upload file to send to Dropbox', + mode: 'basic', + multiple: false, required: true, + condition: { field: 'operation', value: 'dropbox_upload' }, + }, + { + id: 'fileRef', + title: 'File', + type: 'short-input', + canonicalParamId: 'file', + placeholder: 'Reference file from previous blocks', + mode: 'advanced', + required: true, + condition: { field: 'operation', value: 'dropbox_upload' }, }, { id: 'mode', @@ -303,6 +317,16 @@ Return ONLY the timestamp string - no explanations, no quotes, no extra text.`, params.maxResults = Number(params.maxResults) } + // Normalize file input for upload operation + // Check all possible field IDs: uploadFile (basic), fileRef (advanced), fileContent (legacy) + const normalizedFile = normalizeFileInput( + params.uploadFile || params.fileRef || params.fileContent, + { single: true } + ) + if (normalizedFile) { + params.file = normalizedFile + } + switch (params.operation) { case 'dropbox_upload': return 'dropbox_upload' @@ -337,7 +361,10 @@ Return ONLY the timestamp string - no explanations, no quotes, no extra text.`, path: { type: 'string', description: 'Path in Dropbox' }, autorename: { type: 'boolean', description: 'Auto-rename on conflict' }, // Upload inputs - fileContent: { type: 'string', description: 'Base64 encoded file content' }, + uploadFile: { type: 'json', description: 'Uploaded file (UserFile)' }, + file: { type: 'json', description: 'File to upload (UserFile object)' }, + fileRef: { type: 'json', description: 'File reference from previous block' }, + fileContent: { type: 'string', description: 'Legacy: base64 encoded file content' }, fileName: { type: 'string', description: 'Optional filename' }, mode: { type: 'string', description: 'Write mode: add or overwrite' }, mute: { type: 'boolean', description: 'Mute notifications' }, @@ -360,7 +387,7 @@ Return ONLY the timestamp string - no explanations, no quotes, no extra text.`, }, outputs: { // Upload/Download outputs - file: { type: 'json', description: 'File metadata' }, + file: { type: 'file', description: 'Downloaded file stored in execution files' }, content: { type: 'string', description: 'File content (base64)' }, temporaryLink: { type: 'string', description: 'Temporary download link' }, // List folder outputs diff --git a/apps/sim/blocks/blocks/elevenlabs.ts b/apps/sim/blocks/blocks/elevenlabs.ts index 9589a9f47..58d79fe67 100644 --- a/apps/sim/blocks/blocks/elevenlabs.ts +++ b/apps/sim/blocks/blocks/elevenlabs.ts @@ -73,5 +73,6 @@ export const ElevenLabsBlock: BlockConfig = { outputs: { audioUrl: { type: 'string', description: 'Generated audio URL' }, + audioFile: { type: 'file', description: 'Generated audio file' }, }, } diff --git a/apps/sim/blocks/blocks/file.ts b/apps/sim/blocks/blocks/file.ts index 7e478f42a..3db0c2d47 100644 --- a/apps/sim/blocks/blocks/file.ts +++ b/apps/sim/blocks/blocks/file.ts @@ -1,11 +1,48 @@ import { createLogger } from '@sim/logger' import { DocumentIcon } from '@/components/icons' +import { inferContextFromKey } from '@/lib/uploads/utils/file-utils' import type { BlockConfig, SubBlockType } from '@/blocks/types' -import { createVersionedToolSelector } from '@/blocks/utils' -import type { FileParserOutput } from '@/tools/file/types' +import { createVersionedToolSelector, normalizeFileInput } from '@/blocks/utils' +import type { FileParserOutput, FileParserV3Output } from '@/tools/file/types' const logger = createLogger('FileBlock') +const resolveFilePathFromInput = (fileInput: unknown): string | null => { + if (!fileInput || typeof fileInput !== 'object') { + return null + } + + const record = fileInput as Record + if (typeof record.path === 'string' && record.path.trim() !== '') { + return record.path + } + if (typeof record.url === 'string' && record.url.trim() !== '') { + return record.url + } + if (typeof record.key === 'string' && record.key.trim() !== '') { + const key = record.key.trim() + const context = typeof record.context === 'string' ? record.context : inferContextFromKey(key) + return `/api/files/serve/${encodeURIComponent(key)}?context=${context}` + } + + return null +} + +const resolveFilePathsFromInput = (fileInput: unknown): string[] => { + if (!fileInput) { + return [] + } + + if (Array.isArray(fileInput)) { + return fileInput + .map((file) => resolveFilePathFromInput(file)) + .filter((path): path is string => Boolean(path)) + } + + const resolved = resolveFilePathFromInput(fileInput) + return resolved ? [resolved] : [] +} + export const FileBlock: BlockConfig = { type: 'file', name: 'File (Legacy)', @@ -79,24 +116,14 @@ export const FileBlock: BlockConfig = { // Handle file upload input if (inputMethod === 'upload') { - // Handle case where 'file' is an array (multiple files) - if (params.file && Array.isArray(params.file) && params.file.length > 0) { - const filePaths = params.file.map((file) => file.path) - + const filePaths = resolveFilePathsFromInput(params.file) + if (filePaths.length > 0) { return { filePath: filePaths.length === 1 ? filePaths[0] : filePaths, fileType: params.fileType || 'auto', } } - // Handle case where 'file' is a single file object - if (params.file?.path) { - return { - filePath: params.file.path, - fileType: params.fileType || 'auto', - } - } - // If no files, return error logger.error('No files provided for upload method') throw new Error('Please upload a file') @@ -116,7 +143,7 @@ export const FileBlock: BlockConfig = { }, outputs: { files: { - type: 'json', + type: 'file[]', description: 'Array of parsed file objects with content, metadata, and file properties', }, combinedContent: { @@ -124,7 +151,7 @@ export const FileBlock: BlockConfig = { description: 'All file contents merged into a single text string', }, processedFiles: { - type: 'files', + type: 'file[]', description: 'Array of UserFile objects for downstream use (attachments, uploads, etc.)', }, }, @@ -133,9 +160,9 @@ export const FileBlock: BlockConfig = { export const FileV2Block: BlockConfig = { ...FileBlock, type: 'file_v2', - name: 'File', + name: 'File (Legacy)', description: 'Read and parse multiple files', - hideFromToolbar: false, + hideFromToolbar: true, subBlocks: [ { id: 'file', @@ -173,7 +200,21 @@ export const FileV2Block: BlockConfig = { throw new Error('File is required') } - if (typeof fileInput === 'string') { + // First, try to normalize as file objects (handles JSON strings from advanced mode) + const normalizedFiles = normalizeFileInput(fileInput) + if (normalizedFiles) { + const filePaths = resolveFilePathsFromInput(normalizedFiles) + if (filePaths.length > 0) { + return { + filePath: filePaths.length === 1 ? filePaths[0] : filePaths, + fileType: params.fileType || 'auto', + workspaceId: params._context?.workspaceId, + } + } + } + + // If normalization fails, treat as direct URL string + if (typeof fileInput === 'string' && fileInput.trim()) { return { filePath: fileInput.trim(), fileType: params.fileType || 'auto', @@ -181,21 +222,6 @@ export const FileV2Block: BlockConfig = { } } - if (Array.isArray(fileInput) && fileInput.length > 0) { - const filePaths = fileInput.map((file) => file.path) - return { - filePath: filePaths.length === 1 ? filePaths[0] : filePaths, - fileType: params.fileType || 'auto', - } - } - - if (fileInput?.path) { - return { - filePath: fileInput.path, - fileType: params.fileType || 'auto', - } - } - logger.error('Invalid file input format') throw new Error('Invalid file input') }, @@ -209,7 +235,7 @@ export const FileV2Block: BlockConfig = { }, outputs: { files: { - type: 'json', + type: 'file[]', description: 'Array of parsed file objects with content, metadata, and file properties', }, combinedContent: { @@ -218,3 +244,96 @@ export const FileV2Block: BlockConfig = { }, }, } + +export const FileV3Block: BlockConfig = { + type: 'file_v3', + name: 'File', + description: 'Read and parse multiple files', + longDescription: + 'Upload files directly or import from external URLs to get UserFile objects for use in other blocks.', + docsLink: 'https://docs.sim.ai/tools/file', + category: 'tools', + bgColor: '#40916C', + icon: DocumentIcon, + subBlocks: [ + { + id: 'file', + title: 'Files', + type: 'file-upload' as SubBlockType, + canonicalParamId: 'fileInput', + acceptedTypes: '*', + placeholder: 'Upload files to process', + multiple: true, + mode: 'basic', + maxSize: 100, + required: true, + }, + { + id: 'fileUrl', + title: 'File URL', + type: 'short-input' as SubBlockType, + canonicalParamId: 'fileInput', + placeholder: 'https://example.com/document.pdf', + mode: 'advanced', + required: true, + }, + ], + tools: { + access: ['file_parser_v3'], + config: { + tool: () => 'file_parser_v3', + params: (params) => { + const fileInput = params.fileInput ?? params.file ?? params.fileUrl ?? params.filePath + if (!fileInput) { + logger.error('No file input provided') + throw new Error('File input is required') + } + + // First, try to normalize as file objects (handles JSON strings from advanced mode) + const normalizedFiles = normalizeFileInput(fileInput) + if (normalizedFiles) { + const filePaths = resolveFilePathsFromInput(normalizedFiles) + if (filePaths.length > 0) { + return { + filePath: filePaths.length === 1 ? filePaths[0] : filePaths, + fileType: params.fileType || 'auto', + workspaceId: params._context?.workspaceId, + workflowId: params._context?.workflowId, + executionId: params._context?.executionId, + } + } + } + + // If normalization fails, treat as direct URL string + if (typeof fileInput === 'string' && fileInput.trim()) { + return { + filePath: fileInput.trim(), + fileType: params.fileType || 'auto', + workspaceId: params._context?.workspaceId, + workflowId: params._context?.workflowId, + executionId: params._context?.executionId, + } + } + + logger.error('Invalid file input format') + throw new Error('File input is required') + }, + }, + }, + inputs: { + fileInput: { type: 'json', description: 'File input (upload or URL)' }, + fileUrl: { type: 'string', description: 'External file URL (advanced mode)' }, + file: { type: 'json', description: 'Uploaded file data (basic mode)' }, + fileType: { type: 'string', description: 'File type' }, + }, + outputs: { + files: { + type: 'file[]', + description: 'Parsed files as UserFile objects', + }, + combinedContent: { + type: 'string', + description: 'All file contents merged into a single text string', + }, + }, +} diff --git a/apps/sim/blocks/blocks/fireflies.ts b/apps/sim/blocks/blocks/fireflies.ts index b09247190..568cda788 100644 --- a/apps/sim/blocks/blocks/fireflies.ts +++ b/apps/sim/blocks/blocks/fireflies.ts @@ -1,13 +1,16 @@ import { FirefliesIcon } from '@/components/icons' +import { resolveHttpsUrlFromFileInput } from '@/lib/uploads/utils/file-utils' import type { BlockConfig } from '@/blocks/types' import { AuthMode } from '@/blocks/types' +import { normalizeFileInput } from '@/blocks/utils' import type { FirefliesResponse } from '@/tools/fireflies/types' import { getTrigger } from '@/triggers' export const FirefliesBlock: BlockConfig = { type: 'fireflies', - name: 'Fireflies', + name: 'Fireflies (Legacy)', description: 'Interact with Fireflies.ai meeting transcripts and recordings', + hideFromToolbar: true, authMode: AuthMode.ApiKey, triggerAllowed: true, longDescription: @@ -587,3 +590,61 @@ Return ONLY the summary text - no quotes, no labels.`, available: ['fireflies_transcription_complete'], }, } + +const firefliesV2SubBlocks = (FirefliesBlock.subBlocks || []).filter( + (subBlock) => subBlock.id !== 'audioUrl' +) +const firefliesV2Inputs = FirefliesBlock.inputs + ? Object.fromEntries(Object.entries(FirefliesBlock.inputs).filter(([key]) => key !== 'audioUrl')) + : {} + +export const FirefliesV2Block: BlockConfig = { + ...FirefliesBlock, + type: 'fireflies_v2', + name: 'Fireflies', + description: 'Interact with Fireflies.ai meeting transcripts and recordings', + hideFromToolbar: false, + subBlocks: firefliesV2SubBlocks, + tools: { + ...FirefliesBlock.tools, + config: { + ...FirefliesBlock.tools?.config, + tool: (params) => + FirefliesBlock.tools?.config?.tool + ? FirefliesBlock.tools.config.tool(params) + : params.operation || 'fireflies_list_transcripts', + params: (params) => { + const baseParams = FirefliesBlock.tools?.config?.params + if (!baseParams) { + return params + } + + if (params.operation === 'fireflies_upload_audio') { + const audioFile = normalizeFileInput(params.audioFile || params.audioFileReference, { + single: true, + }) + if (!audioFile) { + throw new Error('Audio file is required.') + } + const audioUrl = resolveHttpsUrlFromFileInput(audioFile) + if (!audioUrl) { + throw new Error('Audio file must include a https URL.') + } + + return baseParams({ + ...params, + audioUrl, + audioFile: undefined, + audioFileReference: undefined, + }) + } + + return baseParams(params) + }, + }, + }, + inputs: { + ...firefliesV2Inputs, + audioFileReference: { type: 'json', description: 'Audio/video file reference' }, + }, +} diff --git a/apps/sim/blocks/blocks/gmail.ts b/apps/sim/blocks/blocks/gmail.ts index 223b69ec7..5f8ac25e1 100644 --- a/apps/sim/blocks/blocks/gmail.ts +++ b/apps/sim/blocks/blocks/gmail.ts @@ -1,7 +1,7 @@ import { GmailIcon } from '@/components/icons' import type { BlockConfig } from '@/blocks/types' import { AuthMode } from '@/blocks/types' -import { createVersionedToolSelector } from '@/blocks/utils' +import { createVersionedToolSelector, normalizeFileInput } from '@/blocks/utils' import type { GmailToolResponse } from '@/tools/gmail/types' import { getTrigger } from '@/triggers' @@ -418,6 +418,8 @@ Return ONLY the search query - no explanations, no extra text.`, labelActionMessageId, labelManagement, manualLabelManagement, + attachmentFiles, + attachments, ...rest } = params @@ -465,9 +467,13 @@ Return ONLY the search query - no explanations, no extra text.`, } } + // Normalize attachments for send/draft operations + const normalizedAttachments = normalizeFileInput(attachmentFiles || attachments) + return { ...rest, credential, + ...(normalizedAttachments && { attachments: normalizedAttachments }), } }, }, @@ -516,7 +522,7 @@ Return ONLY the search query - no explanations, no extra text.`, // Tool outputs content: { type: 'string', description: 'Response content' }, metadata: { type: 'json', description: 'Email metadata' }, - attachments: { type: 'json', description: 'Email attachments array' }, + attachments: { type: 'file[]', description: 'Email attachments array' }, // Trigger outputs email_id: { type: 'string', description: 'Gmail message ID' }, thread_id: { type: 'string', description: 'Gmail thread ID' }, @@ -579,7 +585,7 @@ export const GmailV2Block: BlockConfig = { date: { type: 'string', description: 'Date' }, body: { type: 'string', description: 'Email body text (best-effort)' }, results: { type: 'json', description: 'Search/read summary results' }, - attachments: { type: 'json', description: 'Downloaded attachments (if enabled)' }, + attachments: { type: 'file[]', description: 'Downloaded attachments (if enabled)' }, // Draft-specific outputs draftId: { diff --git a/apps/sim/blocks/blocks/google_drive.ts b/apps/sim/blocks/blocks/google_drive.ts index 209bd12f9..d14168d5a 100644 --- a/apps/sim/blocks/blocks/google_drive.ts +++ b/apps/sim/blocks/blocks/google_drive.ts @@ -1,6 +1,7 @@ import { GoogleDriveIcon } from '@/components/icons' import type { BlockConfig } from '@/blocks/types' import { AuthMode } from '@/blocks/types' +import { normalizeFileInput } from '@/blocks/utils' import type { GoogleDriveResponse } from '@/tools/google_drive/types' export const GoogleDriveBlock: BlockConfig = { @@ -782,6 +783,8 @@ Return ONLY the message text - no subject line, no greetings/signatures, no extr manualDestinationFolderId, fileSelector, manualFileId, + file, + fileUpload, mimeType, shareType, starred, @@ -789,6 +792,9 @@ Return ONLY the message text - no subject line, no greetings/signatures, no extr ...rest } = params + // Normalize file input - handles both basic (file-upload) and advanced (short-input) modes + const normalizedFile = normalizeFileInput(file ?? fileUpload, { single: true }) + // Use folderSelector if provided, otherwise use manualFolderId const effectiveFolderId = (folderSelector || manualFolderId || '').trim() @@ -813,6 +819,7 @@ Return ONLY the message text - no subject line, no greetings/signatures, no extr folderId: effectiveFolderId || undefined, fileId: effectiveFileId || undefined, destinationFolderId: effectiveDestinationFolderId || undefined, + file: normalizedFile, pageSize: rest.pageSize ? Number.parseInt(rest.pageSize as string, 10) : undefined, mimeType: mimeType, type: shareType, // Map shareType to type for share tool @@ -861,7 +868,7 @@ Return ONLY the message text - no subject line, no greetings/signatures, no extr permissionId: { type: 'string', description: 'Permission ID to remove' }, }, outputs: { - file: { type: 'json', description: 'File metadata or downloaded file data' }, + file: { type: 'file', description: 'Downloaded file stored in execution files' }, files: { type: 'json', description: 'List of files' }, metadata: { type: 'json', description: 'Complete file metadata (from download)' }, content: { type: 'string', description: 'File content as text' }, diff --git a/apps/sim/blocks/blocks/google_sheets.ts b/apps/sim/blocks/blocks/google_sheets.ts index 259842584..a849b718c 100644 --- a/apps/sim/blocks/blocks/google_sheets.ts +++ b/apps/sim/blocks/blocks/google_sheets.ts @@ -1,6 +1,7 @@ import { GoogleSheetsIcon } from '@/components/icons' import type { BlockConfig } from '@/blocks/types' import { AuthMode } from '@/blocks/types' +import { createVersionedToolSelector } from '@/blocks/utils' import type { GoogleSheetsResponse, GoogleSheetsV2Response } from '@/tools/google_sheets/types' // Legacy block - hidden from toolbar @@ -681,34 +682,38 @@ Return ONLY the JSON array - no explanations, no markdown, no extra text.`, 'google_sheets_copy_sheet_v2', ], config: { - tool: (params) => { - switch (params.operation) { - case 'read': - return 'google_sheets_read_v2' - case 'write': - return 'google_sheets_write_v2' - case 'update': - return 'google_sheets_update_v2' - case 'append': - return 'google_sheets_append_v2' - case 'clear': - return 'google_sheets_clear_v2' - case 'get_info': - return 'google_sheets_get_spreadsheet_v2' - case 'create': - return 'google_sheets_create_spreadsheet_v2' - case 'batch_get': - return 'google_sheets_batch_get_v2' - case 'batch_update': - return 'google_sheets_batch_update_v2' - case 'batch_clear': - return 'google_sheets_batch_clear_v2' - case 'copy_sheet': - return 'google_sheets_copy_sheet_v2' - default: - throw new Error(`Invalid Google Sheets V2 operation: ${params.operation}`) - } - }, + tool: createVersionedToolSelector({ + baseToolSelector: (params) => { + switch (params.operation) { + case 'read': + return 'google_sheets_read' + case 'write': + return 'google_sheets_write' + case 'update': + return 'google_sheets_update' + case 'append': + return 'google_sheets_append' + case 'clear': + return 'google_sheets_clear' + case 'get_info': + return 'google_sheets_get_spreadsheet' + case 'create': + return 'google_sheets_create_spreadsheet' + case 'batch_get': + return 'google_sheets_batch_get' + case 'batch_update': + return 'google_sheets_batch_update' + case 'batch_clear': + return 'google_sheets_batch_clear' + case 'copy_sheet': + return 'google_sheets_copy_sheet' + default: + throw new Error(`Invalid Google Sheets operation: ${params.operation}`) + } + }, + suffix: '_v2', + fallbackToolId: 'google_sheets_read_v2', + }), params: (params) => { const { credential, diff --git a/apps/sim/blocks/blocks/google_slides.ts b/apps/sim/blocks/blocks/google_slides.ts index 016d04a74..784fc73fc 100644 --- a/apps/sim/blocks/blocks/google_slides.ts +++ b/apps/sim/blocks/blocks/google_slides.ts @@ -1,12 +1,15 @@ import { GoogleSlidesIcon } from '@/components/icons' +import { resolveHttpsUrlFromFileInput } from '@/lib/uploads/utils/file-utils' import type { BlockConfig } from '@/blocks/types' import { AuthMode } from '@/blocks/types' +import { normalizeFileInput } from '@/blocks/utils' import type { GoogleSlidesResponse } from '@/tools/google_slides/types' export const GoogleSlidesBlock: BlockConfig = { type: 'google_slides', - name: 'Google Slides', + name: 'Google Slides (Legacy)', description: 'Read, write, and create presentations', + hideFromToolbar: true, authMode: AuthMode.OAuth, longDescription: 'Integrate Google Slides into the workflow. Can read, write, create presentations, replace text, add slides, add images, get thumbnails, get page details, delete objects, duplicate objects, reorder slides, create tables, create shapes, and insert text.', @@ -315,12 +318,26 @@ Return ONLY the JSON array - no explanations, no markdown, no extra text.`, required: true, }, { - id: 'imageUrl', - title: 'Image URL', - type: 'short-input', - placeholder: 'Public URL of the image (PNG, JPEG, or GIF)', - condition: { field: 'operation', value: 'add_image' }, + id: 'imageFile', + title: 'Image', + type: 'file-upload', + canonicalParamId: 'imageSource', + placeholder: 'Upload image (PNG, JPEG, or GIF)', + mode: 'basic', + multiple: false, required: true, + acceptedTypes: '.png,.jpg,.jpeg,.gif', + condition: { field: 'operation', value: 'add_image' }, + }, + { + id: 'imageUrl', + title: 'Image', + type: 'short-input', + canonicalParamId: 'imageSource', + placeholder: 'Reference image from previous blocks or enter URL', + mode: 'advanced', + required: true, + condition: { field: 'operation', value: 'add_image' }, }, { id: 'imageWidth', @@ -809,7 +826,9 @@ Return ONLY the text content - no explanations, no markdown formatting markers, placeholderIdMappings: { type: 'string', description: 'JSON array of placeholder ID mappings' }, // Add image operation pageObjectId: { type: 'string', description: 'Slide object ID for image' }, - imageUrl: { type: 'string', description: 'Image URL' }, + imageFile: { type: 'json', description: 'Uploaded image (UserFile)' }, + imageUrl: { type: 'string', description: 'Image URL or reference' }, + imageSource: { type: 'json', description: 'Image source (file or URL)' }, imageWidth: { type: 'number', description: 'Image width in points' }, imageHeight: { type: 'number', description: 'Image height in points' }, positionX: { type: 'number', description: 'X position in points' }, @@ -887,3 +906,85 @@ Return ONLY the text content - no explanations, no markdown formatting markers, text: { type: 'string', description: 'Text that was inserted' }, }, } + +const googleSlidesV2SubBlocks = (GoogleSlidesBlock.subBlocks || []).flatMap((subBlock) => { + if (subBlock.id === 'imageFile') { + return [ + { + ...subBlock, + canonicalParamId: 'imageFile', + }, + ] + } + + if (subBlock.id !== 'imageUrl') { + return [subBlock] + } + + return [ + { + id: 'imageFileReference', + title: 'Image', + type: 'short-input' as const, + canonicalParamId: 'imageFile', + placeholder: 'Reference image from previous blocks', + mode: 'advanced' as const, + required: true, + condition: { field: 'operation', value: 'add_image' }, + }, + ] +}) + +const googleSlidesV2Inputs = GoogleSlidesBlock.inputs + ? Object.fromEntries( + Object.entries(GoogleSlidesBlock.inputs).filter( + ([key]) => key !== 'imageUrl' && key !== 'imageSource' + ) + ) + : {} + +export const GoogleSlidesV2Block: BlockConfig = { + ...GoogleSlidesBlock, + type: 'google_slides_v2', + name: 'Google Slides', + description: 'Read, write, and create presentations', + hideFromToolbar: false, + subBlocks: googleSlidesV2SubBlocks, + tools: { + access: GoogleSlidesBlock.tools!.access, + config: { + tool: GoogleSlidesBlock.tools!.config!.tool, + params: (params) => { + const baseParams = GoogleSlidesBlock.tools?.config?.params + if (!baseParams) { + return params + } + + if (params.operation === 'add_image') { + const imageInput = params.imageFile || params.imageFileReference || params.imageSource + const fileObject = normalizeFileInput(imageInput, { single: true }) + if (!fileObject) { + throw new Error('Image file is required.') + } + const imageUrl = resolveHttpsUrlFromFileInput(fileObject) + if (!imageUrl) { + throw new Error('Image file must include a https URL.') + } + + return baseParams({ + ...params, + imageUrl, + imageFileReference: undefined, + imageSource: undefined, + }) + } + + return baseParams(params) + }, + }, + }, + inputs: { + ...googleSlidesV2Inputs, + imageFileReference: { type: 'json', description: 'Image file reference' }, + }, +} diff --git a/apps/sim/blocks/blocks/google_vault.ts b/apps/sim/blocks/blocks/google_vault.ts index 25a6a9fb9..47e53d56d 100644 --- a/apps/sim/blocks/blocks/google_vault.ts +++ b/apps/sim/blocks/blocks/google_vault.ts @@ -526,7 +526,7 @@ Return ONLY the description text - no explanations, no quotes, no extra text.`, description: 'Single hold object (for create_matters_holds or list_matters_holds with holdId)', }, - file: { type: 'json', description: 'Downloaded export file (UserFile) from execution files' }, + file: { type: 'file', description: 'Downloaded export file (UserFile) from execution files' }, nextPageToken: { type: 'string', description: 'Token for fetching next page of results (for list operations)', diff --git a/apps/sim/blocks/blocks/image_generator.ts b/apps/sim/blocks/blocks/image_generator.ts index befe3ecb4..e2efad69d 100644 --- a/apps/sim/blocks/blocks/image_generator.ts +++ b/apps/sim/blocks/blocks/image_generator.ts @@ -149,7 +149,7 @@ export const ImageGeneratorBlock: BlockConfig = { }, outputs: { content: { type: 'string', description: 'Generation response' }, - image: { type: 'string', description: 'Generated image URL' }, + image: { type: 'file', description: 'Generated image file (UserFile)' }, metadata: { type: 'json', description: 'Generation metadata' }, }, } diff --git a/apps/sim/blocks/blocks/imap.ts b/apps/sim/blocks/blocks/imap.ts index 33cc6e0ec..e23727b20 100644 --- a/apps/sim/blocks/blocks/imap.ts +++ b/apps/sim/blocks/blocks/imap.ts @@ -44,7 +44,7 @@ export const ImapBlock: BlockConfig = { bodyHtml: { type: 'string', description: 'HTML email body' }, mailbox: { type: 'string', description: 'Mailbox/folder where email was received' }, hasAttachments: { type: 'boolean', description: 'Whether email has attachments' }, - attachments: { type: 'json', description: 'Array of email attachments' }, + attachments: { type: 'file[]', description: 'Array of email attachments' }, timestamp: { type: 'string', description: 'Event timestamp' }, }, triggers: { diff --git a/apps/sim/blocks/blocks/jira.ts b/apps/sim/blocks/blocks/jira.ts index d2d61b77e..3d67b3902 100644 --- a/apps/sim/blocks/blocks/jira.ts +++ b/apps/sim/blocks/blocks/jira.ts @@ -1,6 +1,7 @@ import { JiraIcon } from '@/components/icons' import type { BlockConfig } from '@/blocks/types' import { AuthMode } from '@/blocks/types' +import { normalizeFileInput } from '@/blocks/utils' import type { JiraResponse } from '@/tools/jira/types' import { getTrigger } from '@/triggers' @@ -34,6 +35,7 @@ export const JiraBlock: BlockConfig = { { label: 'Update Comment', id: 'update_comment' }, { label: 'Delete Comment', id: 'delete_comment' }, { label: 'Get Attachments', id: 'get_attachments' }, + { label: 'Add Attachment', id: 'add_attachment' }, { label: 'Delete Attachment', id: 'delete_attachment' }, { label: 'Add Worklog', id: 'add_worklog' }, { label: 'Get Worklogs', id: 'get_worklogs' }, @@ -137,6 +139,7 @@ export const JiraBlock: BlockConfig = { 'update_comment', 'delete_comment', 'get_attachments', + 'add_attachment', 'add_worklog', 'get_worklogs', 'update_worklog', @@ -168,6 +171,7 @@ export const JiraBlock: BlockConfig = { 'update_comment', 'delete_comment', 'get_attachments', + 'add_attachment', 'add_worklog', 'get_worklogs', 'update_worklog', @@ -407,6 +411,27 @@ Return ONLY the comment text - no explanations.`, condition: { field: 'operation', value: ['update_comment', 'delete_comment'] }, }, // Attachment fields + { + id: 'attachmentFiles', + title: 'Attachments', + type: 'file-upload', + canonicalParamId: 'files', + placeholder: 'Upload files', + condition: { field: 'operation', value: 'add_attachment' }, + mode: 'basic', + multiple: true, + required: true, + }, + { + id: 'files', + title: 'File References', + type: 'short-input', + canonicalParamId: 'files', + placeholder: 'File reference from previous block', + condition: { field: 'operation', value: 'add_attachment' }, + mode: 'advanced', + required: true, + }, { id: 'attachmentId', title: 'Attachment ID', @@ -576,6 +601,7 @@ Return ONLY the comment text - no explanations.`, 'jira_update_comment', 'jira_delete_comment', 'jira_get_attachments', + 'jira_add_attachment', 'jira_delete_attachment', 'jira_add_worklog', 'jira_get_worklogs', @@ -623,6 +649,8 @@ Return ONLY the comment text - no explanations.`, return 'jira_delete_comment' case 'get_attachments': return 'jira_get_attachments' + case 'add_attachment': + return 'jira_add_attachment' case 'delete_attachment': return 'jira_delete_attachment' case 'add_worklog': @@ -838,6 +866,20 @@ Return ONLY the comment text - no explanations.`, issueKey: effectiveIssueKey, } } + case 'add_attachment': { + if (!effectiveIssueKey) { + throw new Error('Issue Key is required to add attachments.') + } + const normalizedFiles = normalizeFileInput(params.attachmentFiles || params.files) + if (!normalizedFiles || normalizedFiles.length === 0) { + throw new Error('At least one attachment file is required.') + } + return { + ...baseParams, + issueKey: effectiveIssueKey, + files: normalizedFiles, + } + } case 'delete_attachment': { return { ...baseParams, @@ -982,6 +1024,8 @@ Return ONLY the comment text - no explanations.`, commentBody: { type: 'string', description: 'Text content for comment operations' }, commentId: { type: 'string', description: 'Comment ID for update/delete operations' }, // Attachment operation inputs + attachmentFiles: { type: 'json', description: 'Files to attach (UI upload)' }, + files: { type: 'array', description: 'Files to attach (UserFile array)' }, attachmentId: { type: 'string', description: 'Attachment ID for delete operation' }, // Worklog operation inputs timeSpentSeconds: { @@ -1052,6 +1096,8 @@ Return ONLY the comment text - no explanations.`, type: 'json', description: 'Array of attachments with id, filename, size, mimeType, created, author', }, + files: { type: 'file[]', description: 'Uploaded attachment files' }, + attachmentIds: { type: 'json', description: 'Uploaded attachment IDs' }, // jira_delete_attachment, jira_delete_comment, jira_delete_issue, jira_delete_worklog, jira_delete_issue_link outputs attachmentId: { type: 'string', description: 'Deleted attachment ID' }, diff --git a/apps/sim/blocks/blocks/linear.ts b/apps/sim/blocks/blocks/linear.ts index f57682932..2b8e43587 100644 --- a/apps/sim/blocks/blocks/linear.ts +++ b/apps/sim/blocks/blocks/linear.ts @@ -1,6 +1,7 @@ import { LinearIcon } from '@/components/icons' import type { BlockConfig } from '@/blocks/types' import { AuthMode } from '@/blocks/types' +import { normalizeFileInput } from '@/blocks/utils' import type { LinearResponse } from '@/tools/linear/types' import { getTrigger } from '@/triggers' @@ -668,17 +669,44 @@ Return ONLY the date string in YYYY-MM-DD format - no explanations, no quotes, n generationType: 'timestamp', }, }, + // Attachment file + { + id: 'attachmentFileUpload', + title: 'Attachment', + type: 'file-upload', + canonicalParamId: 'file', + placeholder: 'Upload attachment', + condition: { + field: 'operation', + value: ['linear_create_attachment'], + }, + mode: 'basic', + multiple: false, + }, + { + id: 'file', + title: 'File Reference', + type: 'short-input', + canonicalParamId: 'file', + placeholder: 'File reference from previous block', + condition: { + field: 'operation', + value: ['linear_create_attachment'], + }, + mode: 'advanced', + }, // Attachment URL { id: 'url', title: 'URL', type: 'short-input', placeholder: 'Enter URL', - required: true, + required: false, condition: { field: 'operation', value: ['linear_create_attachment'], }, + mode: 'advanced', }, // Attachment title { @@ -1742,16 +1770,34 @@ Return ONLY the date string in YYYY-MM-DD format - no explanations, no quotes, n teamId: effectiveTeamId, } - case 'linear_create_attachment': - if (!params.issueId?.trim() || !params.url?.trim()) { - throw new Error('Issue ID and URL are required.') + case 'linear_create_attachment': { + if (!params.issueId?.trim()) { + throw new Error('Issue ID is required.') + } + // Normalize file inputs - handles JSON stringified values from advanced mode + const attachmentFile = + normalizeFileInput(params.attachmentFileUpload, { + single: true, + errorMessage: 'Attachment file must be a single file.', + }) || + normalizeFileInput(params.file, { + single: true, + errorMessage: 'Attachment file must be a single file.', + }) + const attachmentUrl = + params.url?.trim() || + (attachmentFile ? (attachmentFile as { url?: string }).url : undefined) + if (!attachmentUrl) { + throw new Error('URL or file is required.') } return { ...baseParams, issueId: params.issueId.trim(), - url: params.url.trim(), + url: attachmentUrl, + file: attachmentFile, title: params.attachmentTitle, } + } case 'linear_list_attachments': if (!params.issueId?.trim()) { @@ -2248,6 +2294,8 @@ Return ONLY the date string in YYYY-MM-DD format - no explanations, no quotes, n endDate: { type: 'string', description: 'End date' }, targetDate: { type: 'string', description: 'Target date' }, url: { type: 'string', description: 'URL' }, + attachmentFileUpload: { type: 'json', description: 'File to attach (UI upload)' }, + file: { type: 'json', description: 'File to attach (UserFile)' }, attachmentTitle: { type: 'string', description: 'Attachment title' }, attachmentId: { type: 'string', description: 'Attachment identifier' }, relationType: { type: 'string', description: 'Relation type' }, diff --git a/apps/sim/blocks/blocks/microsoft_excel.ts b/apps/sim/blocks/blocks/microsoft_excel.ts index adb4c5fa1..3438c5bdc 100644 --- a/apps/sim/blocks/blocks/microsoft_excel.ts +++ b/apps/sim/blocks/blocks/microsoft_excel.ts @@ -1,6 +1,7 @@ import { MicrosoftExcelIcon } from '@/components/icons' import type { BlockConfig } from '@/blocks/types' import { AuthMode } from '@/blocks/types' +import { createVersionedToolSelector } from '@/blocks/utils' import type { MicrosoftExcelResponse, MicrosoftExcelV2Response, @@ -489,16 +490,20 @@ Return ONLY the JSON array - no explanations, no markdown, no extra text.`, tools: { access: ['microsoft_excel_read_v2', 'microsoft_excel_write_v2'], config: { - tool: (params) => { - switch (params.operation) { - case 'read': - return 'microsoft_excel_read_v2' - case 'write': - return 'microsoft_excel_write_v2' - default: - throw new Error(`Invalid Microsoft Excel V2 operation: ${params.operation}`) - } - }, + tool: createVersionedToolSelector({ + baseToolSelector: (params) => { + switch (params.operation) { + case 'read': + return 'microsoft_excel_read' + case 'write': + return 'microsoft_excel_write' + default: + throw new Error(`Invalid Microsoft Excel operation: ${params.operation}`) + } + }, + suffix: '_v2', + fallbackToolId: 'microsoft_excel_read_v2', + }), params: (params) => { const { credential, diff --git a/apps/sim/blocks/blocks/microsoft_teams.ts b/apps/sim/blocks/blocks/microsoft_teams.ts index 498d2ae6c..44324e426 100644 --- a/apps/sim/blocks/blocks/microsoft_teams.ts +++ b/apps/sim/blocks/blocks/microsoft_teams.ts @@ -1,6 +1,7 @@ import { MicrosoftTeamsIcon } from '@/components/icons' import type { BlockConfig } from '@/blocks/types' import { AuthMode } from '@/blocks/types' +import { normalizeFileInput } from '@/blocks/utils' import type { MicrosoftTeamsResponse } from '@/tools/microsoft_teams/types' import { getTrigger } from '@/triggers' @@ -344,9 +345,11 @@ export const MicrosoftTeamsBlock: BlockConfig = { } // Add files if provided - const fileParam = attachmentFiles || files - if (fileParam && (operation === 'write_chat' || operation === 'write_channel')) { - baseParams.files = fileParam + if (operation === 'write_chat' || operation === 'write_channel') { + const normalizedFiles = normalizeFileInput(attachmentFiles || files) + if (normalizedFiles) { + baseParams.files = normalizedFiles + } } // Add messageId if provided @@ -462,7 +465,8 @@ export const MicrosoftTeamsBlock: BlockConfig = { messages: { type: 'json', description: 'Array of message objects' }, totalAttachments: { type: 'number', description: 'Total number of attachments' }, attachmentTypes: { type: 'json', description: 'Array of attachment content types' }, - attachments: { type: 'array', description: 'Downloaded message attachments' }, + attachments: { type: 'file[]', description: 'Downloaded message attachments' }, + files: { type: 'file[]', description: 'Files attached to the message' }, updatedContent: { type: 'boolean', description: 'Whether content was successfully updated/sent', diff --git a/apps/sim/blocks/blocks/mistral_parse.ts b/apps/sim/blocks/blocks/mistral_parse.ts index 62773a71c..424f65b8a 100644 --- a/apps/sim/blocks/blocks/mistral_parse.ts +++ b/apps/sim/blocks/blocks/mistral_parse.ts @@ -1,6 +1,6 @@ import { MistralIcon } from '@/components/icons' import { AuthMode, type BlockConfig, type SubBlockType } from '@/blocks/types' -import { createVersionedToolSelector } from '@/blocks/utils' +import { createVersionedToolSelector, normalizeFileInput } from '@/blocks/utils' import type { MistralParserOutput } from '@/tools/mistral/types' export const MistralParseBlock: BlockConfig = { @@ -94,7 +94,7 @@ export const MistralParseBlock: BlockConfig = { if (!params.fileUpload) { throw new Error('Please upload a PDF document') } - parameters.fileUpload = params.fileUpload + parameters.file = params.fileUpload } let pagesArray: number[] | undefined @@ -159,14 +159,16 @@ export const MistralParseV2Block: BlockConfig = { placeholder: 'Upload a PDF document', mode: 'basic', maxSize: 50, + required: true, }, { - id: 'filePath', - title: 'PDF Document', + id: 'fileReference', + title: 'File Reference', type: 'short-input' as SubBlockType, canonicalParamId: 'document', - placeholder: 'Document URL', + placeholder: 'File reference from previous block', mode: 'advanced', + required: true, }, { id: 'resultType', @@ -211,15 +213,14 @@ export const MistralParseV2Block: BlockConfig = { resultType: params.resultType || 'markdown', } - const documentInput = params.fileUpload || params.filePath || params.document + const documentInput = normalizeFileInput( + params.fileUpload || params.fileReference || params.document, + { single: true } + ) if (!documentInput) { throw new Error('PDF document is required') } - if (typeof documentInput === 'object') { - parameters.fileUpload = documentInput - } else if (typeof documentInput === 'string') { - parameters.filePath = documentInput.trim() - } + parameters.file = documentInput let pagesArray: number[] | undefined if (params.pages && params.pages.trim() !== '') { @@ -254,8 +255,8 @@ export const MistralParseV2Block: BlockConfig = { }, }, inputs: { - document: { type: 'json', description: 'Document input (file upload or URL reference)' }, - filePath: { type: 'string', description: 'PDF document URL (advanced mode)' }, + document: { type: 'json', description: 'Document input (file upload or file reference)' }, + fileReference: { type: 'json', description: 'File reference (advanced mode)' }, fileUpload: { type: 'json', description: 'Uploaded PDF file (basic mode)' }, apiKey: { type: 'string', description: 'Mistral API key' }, resultType: { type: 'string', description: 'Output format type' }, diff --git a/apps/sim/blocks/blocks/notion.ts b/apps/sim/blocks/blocks/notion.ts index 9bc6e6bf3..a970de73f 100644 --- a/apps/sim/blocks/blocks/notion.ts +++ b/apps/sim/blocks/blocks/notion.ts @@ -412,6 +412,7 @@ export const NotionV2Block: BlockConfig = { 'notion_read_database_v2', 'notion_write_v2', 'notion_create_page_v2', + 'notion_update_page_v2', 'notion_query_database_v2', 'notion_search_v2', 'notion_create_database_v2', diff --git a/apps/sim/blocks/blocks/onedrive.ts b/apps/sim/blocks/blocks/onedrive.ts index d9753cced..e2e3545fb 100644 --- a/apps/sim/blocks/blocks/onedrive.ts +++ b/apps/sim/blocks/blocks/onedrive.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { MicrosoftOneDriveIcon } from '@/components/icons' import type { BlockConfig } from '@/blocks/types' import { AuthMode } from '@/blocks/types' +import { normalizeFileInput } from '@/blocks/utils' import type { OneDriveResponse } from '@/tools/onedrive/types' import { normalizeExcelValuesForToolParams } from '@/tools/onedrive/utils' @@ -352,17 +353,31 @@ export const OneDriveBlock: BlockConfig = { } }, params: (params) => { - const { credential, folderId, fileId, mimeType, values, downloadFileName, ...rest } = params + const { + credential, + folderId, + fileId, + mimeType, + values, + downloadFileName, + file, + fileReference, + ...rest + } = params let normalizedValues: ReturnType if (values !== undefined) { normalizedValues = normalizeExcelValuesForToolParams(values) } + // Normalize file input from both basic (file-upload) and advanced (short-input) modes + const normalizedFile = normalizeFileInput(file || fileReference, { single: true }) + return { credential, ...rest, values: normalizedValues, + file: normalizedFile, folderId: folderId || undefined, fileId: fileId || undefined, pageSize: rest.pageSize ? Number.parseInt(rest.pageSize as string, 10) : undefined, @@ -393,7 +408,7 @@ export const OneDriveBlock: BlockConfig = { deleted: { type: 'boolean', description: 'Whether the file was deleted' }, fileId: { type: 'string', description: 'The ID of the deleted file' }, file: { - type: 'json', + type: 'file', description: 'The OneDrive file object, including details such as id, name, size, and more.', }, files: { diff --git a/apps/sim/blocks/blocks/outlook.ts b/apps/sim/blocks/blocks/outlook.ts index 9bf8c312e..b626c20a4 100644 --- a/apps/sim/blocks/blocks/outlook.ts +++ b/apps/sim/blocks/blocks/outlook.ts @@ -1,6 +1,7 @@ import { OutlookIcon } from '@/components/icons' import type { BlockConfig } from '@/blocks/types' import { AuthMode } from '@/blocks/types' +import { normalizeFileInput } from '@/blocks/utils' import type { OutlookResponse } from '@/tools/outlook/types' import { getTrigger } from '@/triggers' @@ -335,12 +336,20 @@ export const OutlookBlock: BlockConfig = { copyMessageId, copyDestinationFolder, manualCopyDestinationFolder, + attachmentFiles, + attachments, ...rest } = params // Handle both selector and manual folder input const effectiveFolder = (folder || manualFolder || '').trim() + // Normalize file attachments from either basic (file-upload) or advanced (short-input) mode + const normalizedAttachments = normalizeFileInput(attachmentFiles || attachments) + if (normalizedAttachments) { + rest.attachments = normalizedAttachments + } + if (rest.operation === 'read_outlook') { rest.folder = effectiveFolder || 'INBOX' } @@ -440,7 +449,7 @@ export const OutlookBlock: BlockConfig = { sentDateTime: { type: 'string', description: 'Email sent timestamp' }, hasAttachments: { type: 'boolean', description: 'Whether email has attachments' }, attachments: { - type: 'json', + type: 'file[]', description: 'Email attachments (if includeAttachments is enabled)', }, isRead: { type: 'boolean', description: 'Whether email is read' }, diff --git a/apps/sim/blocks/blocks/pipedrive.ts b/apps/sim/blocks/blocks/pipedrive.ts index b6bd6fb8e..22d81d782 100644 --- a/apps/sim/blocks/blocks/pipedrive.ts +++ b/apps/sim/blocks/blocks/pipedrive.ts @@ -804,6 +804,7 @@ Return ONLY the date string in YYYY-MM-DD format - no explanations, no quotes, n deals: { type: 'json', description: 'Array of deal objects' }, deal: { type: 'json', description: 'Single deal object' }, files: { type: 'json', description: 'Array of file objects' }, + downloadedFiles: { type: 'file[]', description: 'Downloaded files from Pipedrive' }, messages: { type: 'json', description: 'Array of mail message objects' }, pipelines: { type: 'json', description: 'Array of pipeline objects' }, projects: { type: 'json', description: 'Array of project objects' }, diff --git a/apps/sim/blocks/blocks/pulse.ts b/apps/sim/blocks/blocks/pulse.ts index 0e2f5658f..c61f11070 100644 --- a/apps/sim/blocks/blocks/pulse.ts +++ b/apps/sim/blocks/blocks/pulse.ts @@ -1,11 +1,13 @@ import { PulseIcon } from '@/components/icons' import { AuthMode, type BlockConfig, type SubBlockType } from '@/blocks/types' +import { createVersionedToolSelector, normalizeFileInput } from '@/blocks/utils' import type { PulseParserOutput } from '@/tools/pulse/types' export const PulseBlock: BlockConfig = { type: 'pulse', name: 'Pulse', description: 'Extract text from documents using Pulse OCR', + hideFromToolbar: true, authMode: AuthMode.ApiKey, longDescription: 'Integrate Pulse into the workflow. Extract text from PDF documents, images, and Office files via URL or upload.', @@ -77,7 +79,7 @@ export const PulseBlock: BlockConfig = { throw new Error('Document is required') } if (typeof documentInput === 'object') { - parameters.fileUpload = documentInput + parameters.file = documentInput } else if (typeof documentInput === 'string') { parameters.filePath = documentInput.trim() } @@ -126,3 +128,88 @@ export const PulseBlock: BlockConfig = { figures: { type: 'json', description: 'Extracted figures if figure extraction was enabled' }, }, } + +const pulseV2Inputs = PulseBlock.inputs + ? { + ...Object.fromEntries( + Object.entries(PulseBlock.inputs).filter(([key]) => key !== 'filePath') + ), + fileReference: { type: 'json', description: 'File reference (advanced mode)' }, + } + : {} +const pulseV2SubBlocks = (PulseBlock.subBlocks || []).flatMap((subBlock) => { + if (subBlock.id === 'filePath') { + return [] // Remove the old filePath subblock + } + if (subBlock.id === 'fileUpload') { + // Insert fileReference right after fileUpload + return [ + subBlock, + { + id: 'fileReference', + title: 'Document', + type: 'short-input' as SubBlockType, + canonicalParamId: 'document', + placeholder: 'File reference', + mode: 'advanced' as const, + }, + ] + } + return [subBlock] +}) + +export const PulseV2Block: BlockConfig = { + ...PulseBlock, + type: 'pulse_v2', + name: 'Pulse', + hideFromToolbar: false, + longDescription: + 'Integrate Pulse into the workflow. Extract text from PDF documents, images, and Office files via upload or file references.', + subBlocks: pulseV2SubBlocks, + tools: { + access: ['pulse_parser_v2'], + config: { + tool: createVersionedToolSelector({ + baseToolSelector: () => 'pulse_parser', + suffix: '_v2', + fallbackToolId: 'pulse_parser_v2', + }), + params: (params) => { + if (!params || !params.apiKey || params.apiKey.trim() === '') { + throw new Error('Pulse API key is required') + } + + const parameters: Record = { + apiKey: params.apiKey.trim(), + } + + const normalizedFile = normalizeFileInput( + params.fileUpload || params.fileReference || params.document, + { single: true } + ) + if (!normalizedFile) { + throw new Error('Document file is required') + } + parameters.file = normalizedFile + + if (params.pages && params.pages.trim() !== '') { + parameters.pages = params.pages.trim() + } + + if (params.chunking && params.chunking.trim() !== '') { + parameters.chunking = params.chunking.trim() + } + + if (params.chunkSize && params.chunkSize.trim() !== '') { + const size = Number.parseInt(params.chunkSize.trim(), 10) + if (!Number.isNaN(size) && size > 0) { + parameters.chunkSize = size + } + } + + return parameters + }, + }, + }, + inputs: pulseV2Inputs, +} diff --git a/apps/sim/blocks/blocks/reducto.ts b/apps/sim/blocks/blocks/reducto.ts index 681c2aa20..fb9d39370 100644 --- a/apps/sim/blocks/blocks/reducto.ts +++ b/apps/sim/blocks/blocks/reducto.ts @@ -1,11 +1,13 @@ import { ReductoIcon } from '@/components/icons' import { AuthMode, type BlockConfig, type SubBlockType } from '@/blocks/types' +import { createVersionedToolSelector, normalizeFileInput } from '@/blocks/utils' import type { ReductoParserOutput } from '@/tools/reducto/types' export const ReductoBlock: BlockConfig = { type: 'reducto', name: 'Reducto', description: 'Extract text from PDF documents', + hideFromToolbar: true, authMode: AuthMode.ApiKey, longDescription: `Integrate Reducto Parse into the workflow. Can extract text from uploaded PDF documents, or from a URL.`, docsLink: 'https://docs.sim.ai/tools/reducto', @@ -74,7 +76,7 @@ export const ReductoBlock: BlockConfig = { } if (typeof documentInput === 'object') { - parameters.fileUpload = documentInput + parameters.file = documentInput } else if (typeof documentInput === 'string') { parameters.filePath = documentInput.trim() } @@ -132,3 +134,103 @@ export const ReductoBlock: BlockConfig = { studio_link: { type: 'string', description: 'Link to Reducto studio interface' }, }, } + +const reductoV2Inputs = ReductoBlock.inputs + ? { + ...Object.fromEntries( + Object.entries(ReductoBlock.inputs).filter(([key]) => key !== 'filePath') + ), + fileReference: { type: 'json', description: 'File reference (advanced mode)' }, + } + : {} +const reductoV2SubBlocks = (ReductoBlock.subBlocks || []).flatMap((subBlock) => { + if (subBlock.id === 'filePath') { + return [] + } + if (subBlock.id === 'fileUpload') { + return [ + subBlock, + { + id: 'fileReference', + title: 'PDF Document', + type: 'short-input' as SubBlockType, + canonicalParamId: 'document', + placeholder: 'File reference', + mode: 'advanced' as const, + }, + ] + } + return [subBlock] +}) + +export const ReductoV2Block: BlockConfig = { + ...ReductoBlock, + type: 'reducto_v2', + name: 'Reducto', + hideFromToolbar: false, + longDescription: `Integrate Reducto Parse into the workflow. Can extract text from uploaded PDF documents or file references.`, + subBlocks: reductoV2SubBlocks, + tools: { + access: ['reducto_parser_v2'], + config: { + tool: createVersionedToolSelector({ + baseToolSelector: () => 'reducto_parser', + suffix: '_v2', + fallbackToolId: 'reducto_parser_v2', + }), + params: (params) => { + if (!params || !params.apiKey || params.apiKey.trim() === '') { + throw new Error('Reducto API key is required') + } + + const parameters: Record = { + apiKey: params.apiKey.trim(), + } + + const documentInput = normalizeFileInput( + params.fileUpload || params.fileReference || params.document, + { single: true } + ) + if (!documentInput) { + throw new Error('PDF document file is required') + } + parameters.file = documentInput + + let pagesArray: number[] | undefined + if (params.pages && params.pages.trim() !== '') { + try { + pagesArray = params.pages + .split(',') + .map((p: string) => p.trim()) + .filter((p: string) => p.length > 0) + .map((p: string) => { + const num = Number.parseInt(p, 10) + if (Number.isNaN(num) || num < 0) { + throw new Error(`Invalid page number: ${p}`) + } + return num + }) + + if (pagesArray && pagesArray.length === 0) { + pagesArray = undefined + } + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error) + throw new Error(`Page number format error: ${errorMessage}`) + } + } + + if (pagesArray && pagesArray.length > 0) { + parameters.pages = pagesArray + } + + if (params.tableOutputFormat) { + parameters.tableOutputFormat = params.tableOutputFormat + } + + return parameters + }, + }, + }, + inputs: reductoV2Inputs, +} diff --git a/apps/sim/blocks/blocks/s3.ts b/apps/sim/blocks/blocks/s3.ts index 6dba63175..9c8c537a1 100644 --- a/apps/sim/blocks/blocks/s3.ts +++ b/apps/sim/blocks/blocks/s3.ts @@ -1,6 +1,7 @@ import { S3Icon } from '@/components/icons' import type { BlockConfig } from '@/blocks/types' import { AuthMode } from '@/blocks/types' +import { normalizeFileInput } from '@/blocks/utils' import type { S3Response } from '@/tools/s3/types' export const S3Block: BlockConfig = { @@ -271,7 +272,8 @@ export const S3Block: BlockConfig = { throw new Error('Object Key is required for upload') } // Use file from uploadFile if in basic mode, otherwise use file reference - const fileParam = params.uploadFile || params.file + // normalizeFileInput handles JSON stringified values from advanced mode + const fileParam = normalizeFileInput(params.uploadFile || params.file, { single: true }) return { accessKeyId: params.accessKeyId, @@ -418,6 +420,7 @@ export const S3Block: BlockConfig = { type: 'string', description: 'S3 URI (s3://bucket/key) for use with other AWS services', }, + file: { type: 'file', description: 'Downloaded file stored in execution files' }, objects: { type: 'json', description: 'List of objects (for list operation)' }, deleted: { type: 'boolean', description: 'Deletion status' }, metadata: { type: 'json', description: 'Operation metadata' }, diff --git a/apps/sim/blocks/blocks/sendgrid.ts b/apps/sim/blocks/blocks/sendgrid.ts index 422f9b57f..c55513026 100644 --- a/apps/sim/blocks/blocks/sendgrid.ts +++ b/apps/sim/blocks/blocks/sendgrid.ts @@ -1,5 +1,6 @@ import { SendgridIcon } from '@/components/icons' import type { BlockConfig } from '@/blocks/types' +import { normalizeFileInput } from '@/blocks/utils' import type { SendMailResult } from '@/tools/sendgrid/types' export const SendGridBlock: BlockConfig = { @@ -561,9 +562,14 @@ Return ONLY the HTML content.`, templateGenerations, listPageSize, templatePageSize, + attachmentFiles, + attachments, ...rest } = params + // Normalize attachments for send_mail operation + const normalizedAttachments = normalizeFileInput(attachmentFiles || attachments) + // Map renamed fields back to tool parameter names return { ...rest, @@ -577,6 +583,7 @@ Return ONLY the HTML content.`, ...(templateGenerations && { generations: templateGenerations }), ...(listPageSize && { pageSize: listPageSize }), ...(templatePageSize && { pageSize: templatePageSize }), + ...(normalizedAttachments && { attachments: normalizedAttachments }), } }, }, diff --git a/apps/sim/blocks/blocks/sftp.ts b/apps/sim/blocks/blocks/sftp.ts index f459c1a03..c7afdb534 100644 --- a/apps/sim/blocks/blocks/sftp.ts +++ b/apps/sim/blocks/blocks/sftp.ts @@ -1,6 +1,7 @@ import { SftpIcon } from '@/components/icons' import type { BlockConfig } from '@/blocks/types' import { AuthMode } from '@/blocks/types' +import { normalizeFileInput } from '@/blocks/utils' import type { SftpUploadResult } from '@/tools/sftp/types' export const SftpBlock: BlockConfig = { @@ -222,7 +223,7 @@ export const SftpBlock: BlockConfig = { return { ...connectionConfig, remotePath: params.remotePath, - files: params.files, + files: normalizeFileInput(params.uploadFiles || params.files), overwrite: params.overwrite !== false, permissions: params.permissions, } @@ -293,6 +294,7 @@ export const SftpBlock: BlockConfig = { outputs: { success: { type: 'boolean', description: 'Whether the operation was successful' }, uploadedFiles: { type: 'json', description: 'Array of uploaded file details' }, + file: { type: 'file', description: 'Downloaded file stored in execution files' }, fileName: { type: 'string', description: 'Downloaded file name' }, content: { type: 'string', description: 'Downloaded file content' }, size: { type: 'number', description: 'File size in bytes' }, diff --git a/apps/sim/blocks/blocks/sharepoint.ts b/apps/sim/blocks/blocks/sharepoint.ts index 5fe1dfb6d..e1a6aac2a 100644 --- a/apps/sim/blocks/blocks/sharepoint.ts +++ b/apps/sim/blocks/blocks/sharepoint.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { MicrosoftSharepointIcon } from '@/components/icons' import type { BlockConfig } from '@/blocks/types' import { AuthMode } from '@/blocks/types' +import { normalizeFileInput } from '@/blocks/utils' import type { SharepointResponse } from '@/tools/sharepoint/types' const logger = createLogger('SharepointBlock') @@ -449,7 +450,7 @@ Return ONLY the JSON object - no explanations, no markdown, no extra text.`, } // Handle file upload files parameter - const fileParam = uploadFiles || files + const normalizedFiles = normalizeFileInput(uploadFiles || files) const baseParams: Record = { credential, siteId: effectiveSiteId || undefined, @@ -463,8 +464,8 @@ Return ONLY the JSON object - no explanations, no markdown, no extra text.`, } // Add files if provided - if (fileParam) { - baseParams.files = fileParam + if (normalizedFiles) { + baseParams.files = normalizedFiles } if (columnDefinitions) { diff --git a/apps/sim/blocks/blocks/slack.ts b/apps/sim/blocks/blocks/slack.ts index 7e432f27f..68e0a7a27 100644 --- a/apps/sim/blocks/blocks/slack.ts +++ b/apps/sim/blocks/blocks/slack.ts @@ -1,6 +1,7 @@ import { SlackIcon } from '@/components/icons' import type { BlockConfig } from '@/blocks/types' import { AuthMode } from '@/blocks/types' +import { normalizeFileInput } from '@/blocks/utils' import type { SlackResponse } from '@/tools/slack/types' import { getTrigger } from '@/triggers' @@ -620,9 +621,9 @@ Return ONLY the timestamp string - no explanations, no quotes, no extra text.`, if (threadTs) { baseParams.thread_ts = threadTs } - const fileParam = attachmentFiles || files - if (fileParam) { - baseParams.files = fileParam + const normalizedFiles = normalizeFileInput(attachmentFiles || files) + if (normalizedFiles) { + baseParams.files = normalizedFiles } break } @@ -796,6 +797,7 @@ Return ONLY the timestamp string - no explanations, no quotes, no extra text.`, type: 'number', description: 'Number of files uploaded (when files are attached)', }, + files: { type: 'file[]', description: 'Files attached to the message' }, // slack_canvas outputs canvas_id: { type: 'string', description: 'Canvas identifier for created canvases' }, @@ -859,7 +861,7 @@ Return ONLY the timestamp string - no explanations, no quotes, no extra text.`, // slack_download outputs file: { - type: 'json', + type: 'file', description: 'Downloaded file stored in execution files', }, diff --git a/apps/sim/blocks/blocks/smtp.ts b/apps/sim/blocks/blocks/smtp.ts index c292281b6..640cdd680 100644 --- a/apps/sim/blocks/blocks/smtp.ts +++ b/apps/sim/blocks/blocks/smtp.ts @@ -1,6 +1,7 @@ import { SmtpIcon } from '@/components/icons' import type { BlockConfig } from '@/blocks/types' import { AuthMode } from '@/blocks/types' +import { normalizeFileInput } from '@/blocks/utils' import type { SmtpSendMailResult } from '@/tools/smtp/types' export const SmtpBlock: BlockConfig = { @@ -176,7 +177,7 @@ export const SmtpBlock: BlockConfig = { cc: params.cc, bcc: params.bcc, replyTo: params.replyTo, - attachments: params.attachments, + attachments: normalizeFileInput(params.attachmentFiles || params.attachments), }), }, }, diff --git a/apps/sim/blocks/blocks/spotify.ts b/apps/sim/blocks/blocks/spotify.ts index c4341e6af..c152b3a56 100644 --- a/apps/sim/blocks/blocks/spotify.ts +++ b/apps/sim/blocks/blocks/spotify.ts @@ -1,6 +1,7 @@ import { SpotifyIcon } from '@/components/icons' import type { BlockConfig } from '@/blocks/types' import { AuthMode } from '@/blocks/types' +import { normalizeFileInput } from '@/blocks/utils' import type { ToolResponse } from '@/tools/types' export const SpotifyBlock: BlockConfig = { @@ -450,10 +451,24 @@ export const SpotifyBlock: BlockConfig = { // === PLAYLIST COVER === { - id: 'imageBase64', - title: 'Image (Base64)', - type: 'long-input', - placeholder: 'Base64-encoded JPEG image (max 256KB)', + id: 'coverImageFile', + title: 'Cover Image', + type: 'file-upload', + canonicalParamId: 'coverImage', + placeholder: 'Upload cover image (JPEG, max 256KB)', + mode: 'basic', + multiple: false, + required: true, + acceptedTypes: '.jpg,.jpeg', + condition: { field: 'operation', value: 'spotify_add_playlist_cover' }, + }, + { + id: 'coverImageRef', + title: 'Cover Image', + type: 'short-input', + canonicalParamId: 'coverImage', + placeholder: 'Reference image from previous blocks', + mode: 'advanced', required: true, condition: { field: 'operation', value: 'spotify_add_playlist_cover' }, }, @@ -771,6 +786,10 @@ export const SpotifyBlock: BlockConfig = { if (params.playUris) { params.uris = params.playUris } + // Normalize file input for cover image + if (params.coverImage !== undefined) { + params.coverImage = normalizeFileInput(params.coverImage, { single: true }) + } return params.operation || 'spotify_search' }, }, @@ -804,7 +823,9 @@ export const SpotifyBlock: BlockConfig = { newName: { type: 'string', description: 'New playlist name' }, description: { type: 'string', description: 'Playlist description' }, public: { type: 'boolean', description: 'Whether playlist is public' }, - imageBase64: { type: 'string', description: 'Base64-encoded JPEG image' }, + coverImage: { type: 'json', description: 'Cover image (UserFile)' }, + coverImageFile: { type: 'json', description: 'Cover image upload (basic mode)' }, + coverImageRef: { type: 'json', description: 'Cover image reference (advanced mode)' }, range_start: { type: 'number', description: 'Start index for reorder' }, insert_before: { type: 'number', description: 'Insert before index' }, range_length: { type: 'number', description: 'Number of items to move' }, diff --git a/apps/sim/blocks/blocks/ssh.ts b/apps/sim/blocks/blocks/ssh.ts index 924b26c45..eb7e975d2 100644 --- a/apps/sim/blocks/blocks/ssh.ts +++ b/apps/sim/blocks/blocks/ssh.ts @@ -108,6 +108,28 @@ export const SSHBlock: BlockConfig = { placeholder: 'ls -la /var/www', required: true, condition: { field: 'operation', value: 'ssh_execute_command' }, + wandConfig: { + enabled: true, + prompt: `You are an expert Linux/Unix system administrator. +Generate a shell command or commands based on the user's request for SSH execution on a remote server. + +Current command: {context} + +RULES: +1. Generate ONLY the raw shell command(s) - no markdown, no explanations, no code blocks +2. Use standard Unix/Linux commands that work on most systems +3. For multiple commands, separate with && or ; as appropriate +4. Prefer safe, non-destructive commands when possible +5. Use proper quoting for paths with spaces +6. Consider common shell utilities: ls, cat, grep, find, awk, sed, tar, curl, wget, systemctl, etc. + +Examples: +- "list files" → ls -la +- "find large files" → find . -type f -size +100M +- "check disk space" → df -h +- "show running processes" → ps aux +- "restart nginx" → sudo systemctl restart nginx`, + }, }, { id: 'workingDirectory', @@ -125,6 +147,26 @@ export const SSHBlock: BlockConfig = { placeholder: '#!/bin/bash\necho "Hello World"', required: true, condition: { field: 'operation', value: 'ssh_execute_script' }, + wandConfig: { + enabled: true, + prompt: `You are an expert shell script writer. +Generate a complete shell script based on the user's request for SSH execution on a remote server. + +Current script: {context} + +RULES: +1. Generate ONLY the raw script content - no markdown, no explanations, no code blocks +2. Include appropriate shebang (#!/bin/bash) at the start +3. Use proper error handling where appropriate (set -e, set -o pipefail) +4. Add comments for complex logic +5. Use variables for repeated values +6. Handle edge cases gracefully +7. Make scripts portable across common Linux distributions + +Examples: +- "backup script" → #!/bin/bash\\nset -e\\ntar -czf backup-$(date +%Y%m%d).tar.gz /var/www +- "deploy script" → #!/bin/bash\\nset -e\\ngit pull origin main\\nnpm install\\npm run build\\nsystemctl restart app`, + }, }, { id: 'interpreter', @@ -159,6 +201,25 @@ export const SSHBlock: BlockConfig = { placeholder: 'Content to upload...', required: true, condition: { field: 'operation', value: 'ssh_upload_file' }, + wandConfig: { + enabled: true, + prompt: `You are an expert at generating configuration files and file content for server deployment. +Generate file content based on the user's request for uploading to a remote server via SSH. + +Current content: {context} + +RULES: +1. Generate ONLY the raw file content - no markdown, no explanations, no code blocks +2. Use proper formatting for the file type (JSON, YAML, INI, etc.) +3. Include helpful comments where appropriate for config files +4. Use sensible defaults and best practices +5. Ensure valid syntax for the file format + +Examples: +- "nginx config" → server { listen 80; server_name example.com; ... } +- "json config" → { "key": "value", "port": 3000 } +- "env file" → NODE_ENV=production\\nPORT=3000\\nDATABASE_URL=...`, + }, }, { id: 'fileName', @@ -335,6 +396,25 @@ export const SSHBlock: BlockConfig = { placeholder: 'Content to write...', required: true, condition: { field: 'operation', value: 'ssh_write_file_content' }, + wandConfig: { + enabled: true, + prompt: `You are an expert at generating configuration files and file content for server deployment. +Generate file content based on the user's request for writing to a remote server via SSH. + +Current content: {context} + +RULES: +1. Generate ONLY the raw file content - no markdown, no explanations, no code blocks +2. Use proper formatting for the file type (JSON, YAML, INI, etc.) +3. Include helpful comments where appropriate for config files +4. Use sensible defaults and best practices +5. Ensure valid syntax for the file format + +Examples: +- "nginx config" → server { listen 80; server_name example.com; ... } +- "json config" → { "key": "value", "port": 3000 } +- "env file" → NODE_ENV=production\\nPORT=3000\\nDATABASE_URL=...`, + }, }, { id: 'writeMode', @@ -507,6 +587,7 @@ export const SSHBlock: BlockConfig = { stderr: { type: 'string', description: 'Command standard error' }, exitCode: { type: 'number', description: 'Command exit code' }, success: { type: 'boolean', description: 'Operation success status' }, + file: { type: 'file', description: 'Downloaded file stored in execution files' }, fileContent: { type: 'string', description: 'Downloaded/read file content' }, entries: { type: 'json', description: 'Directory entries' }, exists: { type: 'boolean', description: 'File/directory existence' }, diff --git a/apps/sim/blocks/blocks/stt.ts b/apps/sim/blocks/blocks/stt.ts index 99611d732..66adef7a9 100644 --- a/apps/sim/blocks/blocks/stt.ts +++ b/apps/sim/blocks/blocks/stt.ts @@ -1,11 +1,13 @@ import { STTIcon } from '@/components/icons' import { AuthMode, type BlockConfig } from '@/blocks/types' +import { createVersionedToolSelector, normalizeFileInput } from '@/blocks/utils' import type { SttBlockResponse } from '@/tools/stt/types' export const SttBlock: BlockConfig = { type: 'stt', name: 'Speech-to-Text', description: 'Convert speech to text using AI', + hideFromToolbar: true, authMode: AuthMode.ApiKey, longDescription: 'Transcribe audio and video files to text using leading AI providers. Supports multiple languages, timestamps, and speaker diarization.', @@ -80,10 +82,7 @@ export const SttBlock: BlockConfig = { title: 'Model', type: 'dropdown', condition: { field: 'provider', value: 'assemblyai' }, - options: [ - { label: 'Best', id: 'best' }, - { label: 'Nano', id: 'nano' }, - ], + options: [{ label: 'Best', id: 'best' }], value: () => 'best', required: true, }, @@ -259,22 +258,29 @@ export const SttBlock: BlockConfig = { return 'stt_whisper' } }, - params: (params) => ({ - provider: params.provider, - apiKey: params.apiKey, - model: params.model, - audioFile: params.audioFile, - audioFileReference: params.audioFileReference, - audioUrl: params.audioUrl, - language: params.language, - timestamps: params.timestamps, - diarization: params.diarization, - translateToEnglish: params.translateToEnglish, - sentiment: params.sentiment, - entityDetection: params.entityDetection, - piiRedaction: params.piiRedaction, - summarization: params.summarization, - }), + params: (params) => { + // Normalize file input from basic (file-upload) or advanced (short-input) mode + const audioFile = normalizeFileInput(params.audioFile || params.audioFileReference, { + single: true, + }) + + return { + provider: params.provider, + apiKey: params.apiKey, + model: params.model, + audioFile, + audioFileReference: undefined, + audioUrl: params.audioUrl, + language: params.language, + timestamps: params.timestamps, + diarization: params.diarization, + translateToEnglish: params.translateToEnglish, + sentiment: params.sentiment, + entityDetection: params.entityDetection, + piiRedaction: params.piiRedaction, + summarization: params.summarization, + } + }, }, }, @@ -345,3 +351,70 @@ export const SttBlock: BlockConfig = { }, }, } + +const sttV2Inputs = SttBlock.inputs + ? Object.fromEntries(Object.entries(SttBlock.inputs).filter(([key]) => key !== 'audioUrl')) + : {} +const sttV2SubBlocks = (SttBlock.subBlocks || []).filter((subBlock) => subBlock.id !== 'audioUrl') + +export const SttV2Block: BlockConfig = { + ...SttBlock, + type: 'stt_v2', + name: 'Speech-to-Text', + hideFromToolbar: false, + subBlocks: sttV2SubBlocks, + tools: { + access: [ + 'stt_whisper_v2', + 'stt_deepgram_v2', + 'stt_elevenlabs_v2', + 'stt_assemblyai_v2', + 'stt_gemini_v2', + ], + config: { + tool: createVersionedToolSelector({ + baseToolSelector: (params) => { + switch (params.provider) { + case 'whisper': + return 'stt_whisper' + case 'deepgram': + return 'stt_deepgram' + case 'elevenlabs': + return 'stt_elevenlabs' + case 'assemblyai': + return 'stt_assemblyai' + case 'gemini': + return 'stt_gemini' + default: + return 'stt_whisper' + } + }, + suffix: '_v2', + fallbackToolId: 'stt_whisper_v2', + }), + params: (params) => { + // Normalize file input from basic (file-upload) or advanced (short-input) mode + const audioFile = normalizeFileInput(params.audioFile || params.audioFileReference, { + single: true, + }) + + return { + provider: params.provider, + apiKey: params.apiKey, + model: params.model, + audioFile, + audioFileReference: undefined, + language: params.language, + timestamps: params.timestamps, + diarization: params.diarization, + translateToEnglish: params.translateToEnglish, + sentiment: params.sentiment, + entityDetection: params.entityDetection, + piiRedaction: params.piiRedaction, + summarization: params.summarization, + } + }, + }, + }, + inputs: sttV2Inputs, +} diff --git a/apps/sim/blocks/blocks/supabase.ts b/apps/sim/blocks/blocks/supabase.ts index 8b5fc75f7..78256c5be 100644 --- a/apps/sim/blocks/blocks/supabase.ts +++ b/apps/sim/blocks/blocks/supabase.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { SupabaseIcon } from '@/components/icons' import { AuthMode, type BlockConfig } from '@/blocks/types' +import { normalizeFileInput } from '@/blocks/utils' import type { SupabaseResponse } from '@/tools/supabase/types' const logger = createLogger('SupabaseBlock') @@ -675,9 +676,9 @@ Return ONLY the PostgREST filter expression - no explanations, no markdown, no e { id: 'fileContent', title: 'File Content', - type: 'code', + type: 'short-input', canonicalParamId: 'fileData', - placeholder: 'Base64 encoded for binary files, or plain text', + placeholder: 'File reference from previous block', condition: { field: 'operation', value: 'storage_upload' }, mode: 'advanced', required: true, @@ -973,9 +974,18 @@ Return ONLY the PostgREST filter expression - no explanations, no markdown, no e allowedMimeTypes, upsert, download, + file, + fileContent, + fileData, ...rest } = params + // Normalize file input for storage_upload operation + // normalizeFileInput handles JSON stringified values from advanced mode + const normalizedFileData = normalizeFileInput(file || fileContent || fileData, { + single: true, + }) + // Parse JSON data if it's a string let parsedData if (data && typeof data === 'string' && data.trim()) { @@ -1102,6 +1112,10 @@ Return ONLY the PostgREST filter expression - no explanations, no markdown, no e result.isPublic = parsedIsPublic } + if (normalizedFileData !== undefined) { + result.fileData = normalizedFileData + } + return result }, }, @@ -1173,7 +1187,7 @@ Return ONLY the PostgREST filter expression - no explanations, no markdown, no e description: 'Row count for count operations', }, file: { - type: 'files', + type: 'file', description: 'Downloaded file stored in execution files', }, publicUrl: { diff --git a/apps/sim/blocks/blocks/telegram.ts b/apps/sim/blocks/blocks/telegram.ts index 6c40812a4..2be6eb546 100644 --- a/apps/sim/blocks/blocks/telegram.ts +++ b/apps/sim/blocks/blocks/telegram.ts @@ -1,6 +1,7 @@ import { TelegramIcon } from '@/components/icons' import type { BlockConfig } from '@/blocks/types' import { AuthMode } from '@/blocks/types' +import { normalizeFileInput } from '@/blocks/utils' import type { TelegramResponse } from '@/tools/telegram/types' import { getTrigger } from '@/triggers' @@ -65,39 +66,91 @@ export const TelegramBlock: BlockConfig = { required: true, condition: { field: 'operation', value: 'telegram_message' }, }, + { + id: 'photoFile', + title: 'Photo', + type: 'file-upload', + canonicalParamId: 'photo', + placeholder: 'Upload photo', + mode: 'basic', + multiple: false, + required: true, + acceptedTypes: '.jpg,.jpeg,.png,.gif,.webp', + condition: { field: 'operation', value: 'telegram_send_photo' }, + }, { id: 'photo', title: 'Photo', type: 'short-input', - placeholder: 'Enter photo URL or file_id', - description: 'Photo to send. Pass a file_id or HTTP URL', + canonicalParamId: 'photo', + placeholder: 'Reference photo from previous blocks or enter URL/file_id', + mode: 'advanced', required: true, condition: { field: 'operation', value: 'telegram_send_photo' }, }, + { + id: 'videoFile', + title: 'Video', + type: 'file-upload', + canonicalParamId: 'video', + placeholder: 'Upload video', + mode: 'basic', + multiple: false, + required: true, + acceptedTypes: '.mp4,.mov,.avi,.mkv,.webm', + condition: { field: 'operation', value: 'telegram_send_video' }, + }, { id: 'video', title: 'Video', type: 'short-input', - placeholder: 'Enter video URL or file_id', - description: 'Video to send. Pass a file_id or HTTP URL', + canonicalParamId: 'video', + placeholder: 'Reference video from previous blocks or enter URL/file_id', + mode: 'advanced', required: true, condition: { field: 'operation', value: 'telegram_send_video' }, }, + { + id: 'audioFile', + title: 'Audio', + type: 'file-upload', + canonicalParamId: 'audio', + placeholder: 'Upload audio', + mode: 'basic', + multiple: false, + required: true, + acceptedTypes: '.mp3,.m4a,.wav,.ogg,.flac', + condition: { field: 'operation', value: 'telegram_send_audio' }, + }, { id: 'audio', title: 'Audio', type: 'short-input', - placeholder: 'Enter audio URL or file_id', - description: 'Audio file to send. Pass a file_id or HTTP URL', + canonicalParamId: 'audio', + placeholder: 'Reference audio from previous blocks or enter URL/file_id', + mode: 'advanced', required: true, condition: { field: 'operation', value: 'telegram_send_audio' }, }, + { + id: 'animationFile', + title: 'Animation', + type: 'file-upload', + canonicalParamId: 'animation', + placeholder: 'Upload animation (GIF)', + mode: 'basic', + multiple: false, + required: true, + acceptedTypes: '.gif,.mp4', + condition: { field: 'operation', value: 'telegram_send_animation' }, + }, { id: 'animation', title: 'Animation', type: 'short-input', - placeholder: 'Enter animation URL or file_id', - description: 'Animation (GIF) to send. Pass a file_id or HTTP URL', + canonicalParamId: 'animation', + placeholder: 'Reference animation from previous blocks or enter URL/file_id', + mode: 'advanced', required: true, condition: { field: 'operation', value: 'telegram_send_animation' }, }, @@ -215,48 +268,62 @@ export const TelegramBlock: BlockConfig = { ...commonParams, messageId: params.messageId, } - case 'telegram_send_photo': - if (!params.photo) { - throw new Error('Photo URL or file_id is required.') + case 'telegram_send_photo': { + const photoSource = normalizeFileInput(params.photoFile || params.photo, { + single: true, + }) + if (!photoSource) { + throw new Error('Photo is required.') } return { ...commonParams, - photo: params.photo, + photo: photoSource, caption: params.caption, } - case 'telegram_send_video': - if (!params.video) { - throw new Error('Video URL or file_id is required.') + } + case 'telegram_send_video': { + const videoSource = normalizeFileInput(params.videoFile || params.video, { + single: true, + }) + if (!videoSource) { + throw new Error('Video is required.') } return { ...commonParams, - video: params.video, + video: videoSource, caption: params.caption, } - case 'telegram_send_audio': - if (!params.audio) { - throw new Error('Audio URL or file_id is required.') + } + case 'telegram_send_audio': { + const audioSource = normalizeFileInput(params.audioFile || params.audio, { + single: true, + }) + if (!audioSource) { + throw new Error('Audio is required.') } return { ...commonParams, - audio: params.audio, + audio: audioSource, caption: params.caption, } - case 'telegram_send_animation': - if (!params.animation) { - throw new Error('Animation URL or file_id is required.') + } + case 'telegram_send_animation': { + const animationSource = normalizeFileInput(params.animationFile || params.animation, { + single: true, + }) + if (!animationSource) { + throw new Error('Animation is required.') } return { ...commonParams, - animation: params.animation, + animation: animationSource, caption: params.caption, } + } case 'telegram_send_document': { - // Handle file upload - const fileParam = params.attachmentFiles || params.files return { ...commonParams, - files: fileParam, + files: normalizeFileInput(params.attachmentFiles || params.files), caption: params.caption, } } @@ -274,10 +341,14 @@ export const TelegramBlock: BlockConfig = { botToken: { type: 'string', description: 'Telegram bot token' }, chatId: { type: 'string', description: 'Chat identifier' }, text: { type: 'string', description: 'Message text' }, - photo: { type: 'string', description: 'Photo URL or file_id' }, - video: { type: 'string', description: 'Video URL or file_id' }, - audio: { type: 'string', description: 'Audio URL or file_id' }, - animation: { type: 'string', description: 'Animation URL or file_id' }, + photoFile: { type: 'json', description: 'Uploaded photo (UserFile)' }, + photo: { type: 'json', description: 'Photo reference or URL/file_id' }, + videoFile: { type: 'json', description: 'Uploaded video (UserFile)' }, + video: { type: 'json', description: 'Video reference or URL/file_id' }, + audioFile: { type: 'json', description: 'Uploaded audio (UserFile)' }, + audio: { type: 'json', description: 'Audio reference or URL/file_id' }, + animationFile: { type: 'json', description: 'Uploaded animation (UserFile)' }, + animation: { type: 'json', description: 'Animation reference or URL/file_id' }, attachmentFiles: { type: 'json', description: 'Files to attach (UI upload)', @@ -295,6 +366,7 @@ export const TelegramBlock: BlockConfig = { }, message: { type: 'string', description: 'Success or error message' }, data: { type: 'json', description: 'Response data' }, + files: { type: 'file[]', description: 'Files attached to the message' }, // Specific result fields messageId: { type: 'number', description: 'Sent message ID' }, chatId: { type: 'number', description: 'Chat ID where message was sent' }, diff --git a/apps/sim/blocks/blocks/textract.ts b/apps/sim/blocks/blocks/textract.ts index 2b8388708..10f5a1113 100644 --- a/apps/sim/blocks/blocks/textract.ts +++ b/apps/sim/blocks/blocks/textract.ts @@ -1,11 +1,13 @@ import { TextractIcon } from '@/components/icons' import { AuthMode, type BlockConfig, type SubBlockType } from '@/blocks/types' +import { createVersionedToolSelector, normalizeFileInput } from '@/blocks/utils' import type { TextractParserOutput } from '@/tools/textract/types' export const TextractBlock: BlockConfig = { type: 'textract', name: 'AWS Textract', description: 'Extract text, tables, and forms from documents', + hideFromToolbar: true, authMode: AuthMode.ApiKey, longDescription: `Integrate AWS Textract into your workflow to extract text, tables, forms, and key-value pairs from documents. Single-page mode supports JPEG, PNG, and single-page PDF. Multi-page mode supports multi-page PDF and TIFF.`, docsLink: 'https://docs.sim.ai/tools/textract', @@ -140,7 +142,7 @@ export const TextractBlock: BlockConfig = { throw new Error('Document is required') } if (typeof documentInput === 'object') { - parameters.fileUpload = documentInput + parameters.file = documentInput } else if (typeof documentInput === 'string') { parameters.filePath = documentInput.trim() } @@ -189,3 +191,103 @@ export const TextractBlock: BlockConfig = { }, }, } + +const textractV2Inputs = TextractBlock.inputs + ? { + ...Object.fromEntries( + Object.entries(TextractBlock.inputs).filter(([key]) => key !== 'filePath') + ), + fileReference: { type: 'json', description: 'File reference (advanced mode)' }, + } + : {} +const textractV2SubBlocks = (TextractBlock.subBlocks || []).flatMap((subBlock) => { + if (subBlock.id === 'filePath') { + return [] // Remove the old filePath subblock + } + if (subBlock.id === 'fileUpload') { + // Insert fileReference right after fileUpload + return [ + subBlock, + { + id: 'fileReference', + title: 'Document', + type: 'short-input' as SubBlockType, + canonicalParamId: 'document', + placeholder: 'File reference', + condition: { + field: 'processingMode', + value: 'async', + not: true, + }, + mode: 'advanced' as const, + }, + ] + } + return [subBlock] +}) + +export const TextractV2Block: BlockConfig = { + ...TextractBlock, + type: 'textract_v2', + name: 'AWS Textract', + hideFromToolbar: false, + subBlocks: textractV2SubBlocks, + tools: { + access: ['textract_parser_v2'], + config: { + tool: createVersionedToolSelector({ + baseToolSelector: () => 'textract_parser', + suffix: '_v2', + fallbackToolId: 'textract_parser_v2', + }), + params: (params) => { + if (!params.accessKeyId || params.accessKeyId.trim() === '') { + throw new Error('AWS Access Key ID is required') + } + if (!params.secretAccessKey || params.secretAccessKey.trim() === '') { + throw new Error('AWS Secret Access Key is required') + } + if (!params.region || params.region.trim() === '') { + throw new Error('AWS Region is required') + } + + const processingMode = params.processingMode || 'sync' + const parameters: Record = { + accessKeyId: params.accessKeyId.trim(), + secretAccessKey: params.secretAccessKey.trim(), + region: params.region.trim(), + processingMode, + } + + if (processingMode === 'async') { + if (!params.s3Uri || params.s3Uri.trim() === '') { + throw new Error('S3 URI is required for multi-page processing') + } + parameters.s3Uri = params.s3Uri.trim() + } else { + const file = normalizeFileInput( + params.fileUpload || params.fileReference || params.document, + { single: true } + ) + if (!file) { + throw new Error('Document file is required') + } + parameters.file = file + } + + const featureTypes: string[] = [] + if (params.extractTables) featureTypes.push('TABLES') + if (params.extractForms) featureTypes.push('FORMS') + if (params.detectSignatures) featureTypes.push('SIGNATURES') + if (params.analyzeLayout) featureTypes.push('LAYOUT') + + if (featureTypes.length > 0) { + parameters.featureTypes = featureTypes + } + + return parameters + }, + }, + }, + inputs: textractV2Inputs, +} diff --git a/apps/sim/blocks/blocks/tts.ts b/apps/sim/blocks/blocks/tts.ts index c3ab813fd..eebc8acd3 100644 --- a/apps/sim/blocks/blocks/tts.ts +++ b/apps/sim/blocks/blocks/tts.ts @@ -578,7 +578,7 @@ export const TtsBlock: BlockConfig = { outputs: { audioUrl: { type: 'string', description: 'URL to the generated audio file' }, - audioFile: { type: 'json', description: 'Generated audio file object (UserFile)' }, + audioFile: { type: 'file', description: 'Generated audio file object (UserFile)' }, duration: { type: 'number', description: 'Audio duration in seconds', diff --git a/apps/sim/blocks/blocks/video_generator.ts b/apps/sim/blocks/blocks/video_generator.ts index 88672a17b..ae31eb951 100644 --- a/apps/sim/blocks/blocks/video_generator.ts +++ b/apps/sim/blocks/blocks/video_generator.ts @@ -1,5 +1,6 @@ import { VideoIcon } from '@/components/icons' import { AuthMode, type BlockConfig } from '@/blocks/types' +import { normalizeFileInput } from '@/blocks/utils' import type { VideoBlockResponse } from '@/tools/video/types' export const VideoGeneratorBlock: BlockConfig = { @@ -420,7 +421,7 @@ export const VideoGeneratorBlock: BlockConfig = { outputs: { videoUrl: { type: 'string', description: 'Generated video URL' }, - videoFile: { type: 'json', description: 'Video file object with metadata' }, + videoFile: { type: 'file', description: 'Video file object with metadata' }, duration: { type: 'number', description: 'Video duration in seconds' }, width: { type: 'number', description: 'Video width in pixels' }, height: { type: 'number', description: 'Video height in pixels' }, @@ -745,7 +746,7 @@ export const VideoGeneratorV2Block: BlockConfig = { duration: params.duration ? Number(params.duration) : undefined, aspectRatio: params.aspectRatio, resolution: params.resolution, - visualReference: visualRef, + visualReference: normalizeFileInput(visualRef, { single: true }), consistencyMode: params.consistencyMode, stylePreset: params.stylePreset, promptOptimizer: params.promptOptimizer, diff --git a/apps/sim/blocks/blocks/vision.ts b/apps/sim/blocks/blocks/vision.ts index 8a94d2240..a367b0c58 100644 --- a/apps/sim/blocks/blocks/vision.ts +++ b/apps/sim/blocks/blocks/vision.ts @@ -1,12 +1,30 @@ import { EyeIcon } from '@/components/icons' import type { BlockConfig } from '@/blocks/types' import { AuthMode } from '@/blocks/types' +import { createVersionedToolSelector, normalizeFileInput } from '@/blocks/utils' import type { VisionResponse } from '@/tools/vision/types' +const VISION_MODEL_OPTIONS = [ + { label: 'GPT 5.2', id: 'gpt-5.2' }, + { label: 'GPT 5.1', id: 'gpt-5.1' }, + { label: 'GPT 5', id: 'gpt-5' }, + { label: 'GPT 5 Mini', id: 'gpt-5-mini' }, + { label: 'GPT 5 Nano', id: 'gpt-5-nano' }, + { label: 'Claude Opus 4.5', id: 'claude-opus-4-5' }, + { label: 'Claude Sonnet 4.5', id: 'claude-sonnet-4-5' }, + { label: 'Claude Haiku 4.5', id: 'claude-haiku-4-5' }, + { label: 'Gemini 3 Pro Preview', id: 'gemini-3-pro-preview' }, + { label: 'Gemini 3 Flash Preview', id: 'gemini-3-flash-preview' }, + { label: 'Gemini 2.5 Pro', id: 'gemini-2.5-pro' }, + { label: 'Gemini 2.5 Flash', id: 'gemini-2.5-flash' }, + { label: 'Gemini 2.5 Flash Lite', id: 'gemini-2.5-flash-lite' }, +] + export const VisionBlock: BlockConfig = { type: 'vision', - name: 'Vision', + name: 'Vision (Legacy)', description: 'Analyze images with vision models', + hideFromToolbar: true, authMode: AuthMode.ApiKey, longDescription: 'Integrate Vision into the workflow. Can analyze images with vision models.', docsLink: 'https://docs.sim.ai/tools/vision', @@ -47,12 +65,8 @@ export const VisionBlock: BlockConfig = { id: 'model', title: 'Vision Model', type: 'dropdown', - options: [ - { label: 'gpt-4o', id: 'gpt-4o' }, - { label: 'claude-3-opus', id: 'claude-3-opus-20240229' }, - { label: 'claude-3-sonnet', id: 'claude-3-sonnet-20240229' }, - ], - value: () => 'gpt-4o', + options: VISION_MODEL_OPTIONS, + value: () => 'gpt-5.2', }, { id: 'prompt', @@ -87,3 +101,84 @@ export const VisionBlock: BlockConfig = { tokens: { type: 'number', description: 'Token usage' }, }, } + +export const VisionV2Block: BlockConfig = { + ...VisionBlock, + type: 'vision_v2', + name: 'Vision', + description: 'Analyze images with vision models', + hideFromToolbar: false, + tools: { + access: ['vision_tool_v2'], + config: { + tool: createVersionedToolSelector({ + baseToolSelector: () => 'vision_tool', + suffix: '_v2', + fallbackToolId: 'vision_tool_v2', + }), + params: (params) => { + // normalizeFileInput handles JSON stringified values from advanced mode + // Vision expects a single file + const imageFile = normalizeFileInput(params.imageFile || params.imageFileReference, { + single: true, + }) + return { + ...params, + imageFile, + imageFileReference: undefined, + } + }, + }, + }, + subBlocks: [ + { + id: 'imageFile', + title: 'Image File', + type: 'file-upload', + canonicalParamId: 'imageFile', + placeholder: 'Upload an image file', + mode: 'basic', + multiple: false, + required: true, + acceptedTypes: '.jpg,.jpeg,.png,.gif,.webp', + }, + { + id: 'imageFileReference', + title: 'Image File Reference', + type: 'short-input', + canonicalParamId: 'imageFile', + placeholder: 'Reference an image from previous blocks', + mode: 'advanced', + required: true, + }, + { + id: 'model', + title: 'Vision Model', + type: 'dropdown', + options: VISION_MODEL_OPTIONS, + value: () => 'gpt-5.2', + }, + { + id: 'prompt', + title: 'Prompt', + type: 'long-input', + placeholder: 'Enter prompt for image analysis', + required: true, + }, + { + id: 'apiKey', + title: 'API Key', + type: 'short-input', + placeholder: 'Enter your API key', + password: true, + required: true, + }, + ], + inputs: { + apiKey: { type: 'string', description: 'Provider API key' }, + imageFile: { type: 'json', description: 'Image file (UserFile)' }, + imageFileReference: { type: 'json', description: 'Image file reference' }, + model: { type: 'string', description: 'Vision model' }, + prompt: { type: 'string', description: 'Analysis prompt' }, + }, +} diff --git a/apps/sim/blocks/blocks/wordpress.ts b/apps/sim/blocks/blocks/wordpress.ts index eb19e776a..e0b206ce5 100644 --- a/apps/sim/blocks/blocks/wordpress.ts +++ b/apps/sim/blocks/blocks/wordpress.ts @@ -1,6 +1,7 @@ import { WordpressIcon } from '@/components/icons' import type { BlockConfig } from '@/blocks/types' import { AuthMode } from '@/blocks/types' +import { normalizeFileInput } from '@/blocks/utils' import type { WordPressResponse } from '@/tools/wordpress/types' export const WordPressBlock: BlockConfig = { @@ -769,7 +770,7 @@ export const WordPressBlock: BlockConfig = { case 'wordpress_upload_media': return { ...baseParams, - file: params.fileUpload || params.file, + file: normalizeFileInput(params.fileUpload || params.file, { single: true }), filename: params.filename, title: params.mediaTitle, caption: params.caption, diff --git a/apps/sim/blocks/registry.ts b/apps/sim/blocks/registry.ts index 4fbaf2766..933456957 100644 --- a/apps/sim/blocks/registry.ts +++ b/apps/sim/blocks/registry.ts @@ -29,9 +29,9 @@ import { ElevenLabsBlock } from '@/blocks/blocks/elevenlabs' import { EnrichBlock } from '@/blocks/blocks/enrich' import { EvaluatorBlock } from '@/blocks/blocks/evaluator' import { ExaBlock } from '@/blocks/blocks/exa' -import { FileBlock, FileV2Block } from '@/blocks/blocks/file' +import { FileBlock, FileV2Block, FileV3Block } from '@/blocks/blocks/file' import { FirecrawlBlock } from '@/blocks/blocks/firecrawl' -import { FirefliesBlock } from '@/blocks/blocks/fireflies' +import { FirefliesBlock, FirefliesV2Block } from '@/blocks/blocks/fireflies' import { FunctionBlock } from '@/blocks/blocks/function' import { GenericWebhookBlock } from '@/blocks/blocks/generic_webhook' import { GitHubBlock, GitHubV2Block } from '@/blocks/blocks/github' @@ -45,7 +45,7 @@ import { GoogleFormsBlock } from '@/blocks/blocks/google_forms' import { GoogleGroupsBlock } from '@/blocks/blocks/google_groups' import { GoogleMapsBlock } from '@/blocks/blocks/google_maps' import { GoogleSheetsBlock, GoogleSheetsV2Block } from '@/blocks/blocks/google_sheets' -import { GoogleSlidesBlock } from '@/blocks/blocks/google_slides' +import { GoogleSlidesBlock, GoogleSlidesV2Block } from '@/blocks/blocks/google_slides' import { GoogleVaultBlock } from '@/blocks/blocks/google_vault' import { GrafanaBlock } from '@/blocks/blocks/grafana' import { GrainBlock } from '@/blocks/blocks/grain' @@ -95,11 +95,11 @@ import { PipedriveBlock } from '@/blocks/blocks/pipedrive' import { PolymarketBlock } from '@/blocks/blocks/polymarket' import { PostgreSQLBlock } from '@/blocks/blocks/postgresql' import { PostHogBlock } from '@/blocks/blocks/posthog' -import { PulseBlock } from '@/blocks/blocks/pulse' +import { PulseBlock, PulseV2Block } from '@/blocks/blocks/pulse' import { QdrantBlock } from '@/blocks/blocks/qdrant' import { RDSBlock } from '@/blocks/blocks/rds' import { RedditBlock } from '@/blocks/blocks/reddit' -import { ReductoBlock } from '@/blocks/blocks/reducto' +import { ReductoBlock, ReductoV2Block } from '@/blocks/blocks/reducto' import { ResendBlock } from '@/blocks/blocks/resend' import { ResponseBlock } from '@/blocks/blocks/response' import { RouterBlock, RouterV2Block } from '@/blocks/blocks/router' @@ -125,11 +125,11 @@ import { StagehandBlock } from '@/blocks/blocks/stagehand' import { StartTriggerBlock } from '@/blocks/blocks/start_trigger' import { StarterBlock } from '@/blocks/blocks/starter' import { StripeBlock } from '@/blocks/blocks/stripe' -import { SttBlock } from '@/blocks/blocks/stt' +import { SttBlock, SttV2Block } from '@/blocks/blocks/stt' import { SupabaseBlock } from '@/blocks/blocks/supabase' import { TavilyBlock } from '@/blocks/blocks/tavily' import { TelegramBlock } from '@/blocks/blocks/telegram' -import { TextractBlock } from '@/blocks/blocks/textract' +import { TextractBlock, TextractV2Block } from '@/blocks/blocks/textract' import { ThinkingBlock } from '@/blocks/blocks/thinking' import { TinybirdBlock } from '@/blocks/blocks/tinybird' import { TranslateBlock } from '@/blocks/blocks/translate' @@ -140,7 +140,7 @@ import { TwilioVoiceBlock } from '@/blocks/blocks/twilio_voice' import { TypeformBlock } from '@/blocks/blocks/typeform' import { VariablesBlock } from '@/blocks/blocks/variables' import { VideoGeneratorBlock, VideoGeneratorV2Block } from '@/blocks/blocks/video_generator' -import { VisionBlock } from '@/blocks/blocks/vision' +import { VisionBlock, VisionV2Block } from '@/blocks/blocks/vision' import { WaitBlock } from '@/blocks/blocks/wait' import { WealthboxBlock } from '@/blocks/blocks/wealthbox' import { WebflowBlock } from '@/blocks/blocks/webflow' @@ -194,8 +194,10 @@ export const registry: Record = { exa: ExaBlock, file: FileBlock, file_v2: FileV2Block, + file_v3: FileV3Block, firecrawl: FirecrawlBlock, fireflies: FirefliesBlock, + fireflies_v2: FirefliesV2Block, function: FunctionBlock, generic_webhook: GenericWebhookBlock, github: GitHubBlock, @@ -214,6 +216,7 @@ export const registry: Record = { google_sheets: GoogleSheetsBlock, google_sheets_v2: GoogleSheetsV2Block, google_slides: GoogleSlidesBlock, + google_slides_v2: GoogleSlidesV2Block, google_vault: GoogleVaultBlock, grafana: GrafanaBlock, grain: GrainBlock, @@ -269,10 +272,12 @@ export const registry: Record = { postgresql: PostgreSQLBlock, posthog: PostHogBlock, pulse: PulseBlock, + pulse_v2: PulseV2Block, qdrant: QdrantBlock, rds: RDSBlock, reddit: RedditBlock, reducto: ReductoBlock, + reducto_v2: ReductoV2Block, resend: ResendBlock, response: ResponseBlock, router: RouterBlock, @@ -300,10 +305,12 @@ export const registry: Record = { starter: StarterBlock, stripe: StripeBlock, stt: SttBlock, + stt_v2: SttV2Block, supabase: SupabaseBlock, tavily: TavilyBlock, telegram: TelegramBlock, textract: TextractBlock, + textract_v2: TextractV2Block, thinking: ThinkingBlock, tinybird: TinybirdBlock, translate: TranslateBlock, @@ -316,6 +323,7 @@ export const registry: Record = { video_generator: VideoGeneratorBlock, video_generator_v2: VideoGeneratorV2Block, vision: VisionBlock, + vision_v2: VisionV2Block, wait: WaitBlock, wealthbox: WealthboxBlock, webflow: WebflowBlock, diff --git a/apps/sim/blocks/types.ts b/apps/sim/blocks/types.ts index c59ad427c..352f10642 100644 --- a/apps/sim/blocks/types.ts +++ b/apps/sim/blocks/types.ts @@ -9,7 +9,8 @@ export type PrimitiveValueType = | 'boolean' | 'json' | 'array' - | 'files' + | 'file' + | 'file[]' | 'any' export type BlockCategory = 'blocks' | 'tools' | 'triggers' diff --git a/apps/sim/blocks/utils.ts b/apps/sim/blocks/utils.ts index 52ddbf3b9..7de0b518a 100644 --- a/apps/sim/blocks/utils.ts +++ b/apps/sim/blocks/utils.ts @@ -249,3 +249,62 @@ export function createVersionedToolSelector> } } } + +const DEFAULT_MULTIPLE_FILES_ERROR = + 'File reference must be a single file, not an array. Use to select one file.' + +/** + * Normalizes file input from block params to a consistent format. + * Handles the case where template resolution JSON.stringify's arrays/objects + * when they're placed in short-input fields (advanced mode). + * + * @param fileParam - The file parameter which could be: + * - undefined/null (no files) + * - An array of file objects (basic mode or properly resolved) + * - A single file object + * - A JSON string of file(s) (from advanced mode template resolution) + * @param options.single - If true, returns single file object and throws if multiple provided + * @param options.errorMessage - Custom error message when single is true and multiple files provided + * @returns Normalized file(s), or undefined if no files + */ +export function normalizeFileInput( + fileParam: unknown, + options: { single: true; errorMessage?: string } +): object | undefined +export function normalizeFileInput( + fileParam: unknown, + options?: { single?: false } +): object[] | undefined +export function normalizeFileInput( + fileParam: unknown, + options?: { single?: boolean; errorMessage?: string } +): object | object[] | undefined { + if (!fileParam) return undefined + + if (typeof fileParam === 'string') { + try { + fileParam = JSON.parse(fileParam) + } catch { + return undefined + } + } + + let files: object[] | undefined + + if (Array.isArray(fileParam)) { + files = fileParam.length > 0 ? fileParam : undefined + } else if (typeof fileParam === 'object' && fileParam !== null) { + files = [fileParam] + } + + if (!files) return undefined + + if (options?.single) { + if (files.length > 1) { + throw new Error(options.errorMessage ?? DEFAULT_MULTIPLE_FILES_ERROR) + } + return files[0] + } + + return files +} diff --git a/apps/sim/executor/execution/block-executor.ts b/apps/sim/executor/execution/block-executor.ts index 6c97dd1a1..f6de271cb 100644 --- a/apps/sim/executor/execution/block-executor.ts +++ b/apps/sim/executor/execution/block-executor.ts @@ -138,7 +138,7 @@ export class BlockExecutor { normalizedOutput = this.normalizeOutput(output) } - if (ctx.includeFileBase64 && containsUserFileWithMetadata(normalizedOutput)) { + if (containsUserFileWithMetadata(normalizedOutput)) { normalizedOutput = (await hydrateUserFilesWithBase64(normalizedOutput, { requestId: ctx.metadata.requestId, executionId: ctx.executionId, diff --git a/apps/sim/executor/handlers/api/api-handler.test.ts b/apps/sim/executor/handlers/api/api-handler.test.ts index 3af7fac6f..0f7f0186a 100644 --- a/apps/sim/executor/handlers/api/api-handler.test.ts +++ b/apps/sim/executor/handlers/api/api-handler.test.ts @@ -1,6 +1,7 @@ import '@sim/testing/mocks/executor' import { beforeEach, describe, expect, it, type Mock, vi } from 'vitest' +import { validateUrlWithDNS } from '@/lib/core/security/input-validation.server' import { BlockType } from '@/executor/constants' import { ApiBlockHandler } from '@/executor/handlers/api/api-handler' import type { ExecutionContext } from '@/executor/types' @@ -9,8 +10,13 @@ import { executeTool } from '@/tools' import type { ToolConfig } from '@/tools/types' import { getTool } from '@/tools/utils' +vi.mock('@/lib/core/security/input-validation.server', () => ({ + validateUrlWithDNS: vi.fn(), +})) + const mockGetTool = vi.mocked(getTool) const mockExecuteTool = executeTool as Mock +const mockValidateUrlWithDNS = vi.mocked(validateUrlWithDNS) describe('ApiBlockHandler', () => { let handler: ApiBlockHandler @@ -63,6 +69,12 @@ describe('ApiBlockHandler', () => { // Reset mocks using vi vi.clearAllMocks() + mockValidateUrlWithDNS.mockResolvedValue({ + isValid: true, + resolvedIP: '93.184.216.34', + originalHostname: 'example.com', + }) + // Set up mockGetTool to return the mockApiTool mockGetTool.mockImplementation((toolId) => { if (toolId === 'http_request') { @@ -130,8 +142,13 @@ describe('ApiBlockHandler', () => { it('should throw error for invalid URL format (no protocol)', async () => { const inputs = { url: 'example.com/api' } + mockValidateUrlWithDNS.mockResolvedValueOnce({ + isValid: false, + error: 'url must be a valid URL', + }) + await expect(handler.execute(mockContext, mockBlock, inputs)).rejects.toThrow( - 'Invalid URL: "example.com/api" - URL must include protocol (try "https://example.com/api")' + 'url must be a valid URL' ) expect(mockExecuteTool).not.toHaveBeenCalled() }) @@ -139,8 +156,13 @@ describe('ApiBlockHandler', () => { it('should throw error for generally invalid URL format', async () => { const inputs = { url: 'htp:/invalid-url' } + mockValidateUrlWithDNS.mockResolvedValueOnce({ + isValid: false, + error: 'url must use https:// protocol', + }) + await expect(handler.execute(mockContext, mockBlock, inputs)).rejects.toThrow( - /^Invalid URL: "htp:\/invalid-url" - URL must include protocol/ + 'url must use https:// protocol' ) expect(mockExecuteTool).not.toHaveBeenCalled() }) diff --git a/apps/sim/executor/handlers/api/api-handler.ts b/apps/sim/executor/handlers/api/api-handler.ts index 775b88674..562067cdf 100644 --- a/apps/sim/executor/handlers/api/api-handler.ts +++ b/apps/sim/executor/handlers/api/api-handler.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { validateUrlWithDNS } from '@/lib/core/security/input-validation.server' import { BlockType, HTTP } from '@/executor/constants' import type { BlockHandler, ExecutionContext } from '@/executor/types' import type { SerializedBlock } from '@/serializer/types' @@ -41,16 +42,9 @@ export class ApiBlockHandler implements BlockHandler { } } - if (!urlToValidate.match(/^https?:\/\//i)) { - throw new Error( - `Invalid URL: "${urlToValidate}" - URL must include protocol (try "https://${urlToValidate}")` - ) - } - - try { - new URL(urlToValidate) - } catch (e: any) { - throw new Error(`Invalid URL format: "${urlToValidate}" - ${e.message}`) + const urlValidation = await validateUrlWithDNS(urlToValidate, 'url') + if (!urlValidation.isValid) { + throw new Error(urlValidation.error) } } diff --git a/apps/sim/executor/utils/block-reference.test.ts b/apps/sim/executor/utils/block-reference.test.ts index 6f110c2bc..470522b77 100644 --- a/apps/sim/executor/utils/block-reference.test.ts +++ b/apps/sim/executor/utils/block-reference.test.ts @@ -133,7 +133,7 @@ describe('resolveBlockReference', () => { 'block-1': { input: { type: 'string' }, conversationId: { type: 'string' }, - files: { type: 'files' }, + files: { type: 'file[]' }, }, }, }) @@ -206,7 +206,7 @@ describe('resolveBlockReference', () => { }, }, blockOutputSchemas: { - 'block-1': { files: { type: 'files' } }, + 'block-1': { files: { type: 'file[]' } }, }, }) @@ -218,7 +218,7 @@ describe('resolveBlockReference', () => { const ctx = createContext({ blockData: { 'block-1': { files: [] } }, blockOutputSchemas: { - 'block-1': { files: { type: 'files' } }, + 'block-1': { files: { type: 'file[]' } }, }, }) diff --git a/apps/sim/executor/utils/block-reference.ts b/apps/sim/executor/utils/block-reference.ts index 590e9d869..4ae41a2b1 100644 --- a/apps/sim/executor/utils/block-reference.ts +++ b/apps/sim/executor/utils/block-reference.ts @@ -32,7 +32,7 @@ export class InvalidFieldError extends Error { function isFileType(value: unknown): boolean { if (typeof value !== 'object' || value === null) return false const typed = value as { type?: string } - return typed.type === 'file[]' || typed.type === 'files' + return typed.type === 'file' || typed.type === 'file[]' } function isArrayType(value: unknown): value is { type: 'array'; items?: unknown } { diff --git a/apps/sim/executor/utils/file-tool-processor.ts b/apps/sim/executor/utils/file-tool-processor.ts index b5d7e9dd2..1f0b10374 100644 --- a/apps/sim/executor/utils/file-tool-processor.ts +++ b/apps/sim/executor/utils/file-tool-processor.ts @@ -1,5 +1,7 @@ import { createLogger } from '@sim/logger' +import { isUserFile } from '@/lib/core/utils/user-file' import { uploadExecutionFile, uploadFileFromRawData } from '@/lib/uploads/contexts/execution' +import { downloadFileFromUrl } from '@/lib/uploads/utils/file-utils.server' import type { ExecutionContext, UserFile } from '@/executor/types' import type { ToolConfig, ToolFileData } from '@/tools/types' @@ -93,31 +95,39 @@ export class FileToolProcessor { } /** - * Convert various file data formats to UserFile by storing in execution filesystem + * Convert various file data formats to UserFile by storing in execution filesystem. + * If the input is already a UserFile, returns it unchanged. */ private static async processFileData( - fileData: ToolFileData, + fileData: ToolFileData | UserFile, context: ExecutionContext ): Promise { + // If already a UserFile (e.g., from tools that handle their own file storage), + // return it directly without re-processing + if (isUserFile(fileData)) { + return fileData as UserFile + } + + const data = fileData as ToolFileData try { let buffer: Buffer | null = null - if (Buffer.isBuffer(fileData.data)) { - buffer = fileData.data + if (Buffer.isBuffer(data.data)) { + buffer = data.data } else if ( - fileData.data && - typeof fileData.data === 'object' && - 'type' in fileData.data && - 'data' in fileData.data + data.data && + typeof data.data === 'object' && + 'type' in data.data && + 'data' in data.data ) { - const serializedBuffer = fileData.data as { type: string; data: number[] } + const serializedBuffer = data.data as { type: string; data: number[] } if (serializedBuffer.type === 'Buffer' && Array.isArray(serializedBuffer.data)) { buffer = Buffer.from(serializedBuffer.data) } else { - throw new Error(`Invalid serialized buffer format for ${fileData.name}`) + throw new Error(`Invalid serialized buffer format for ${data.name}`) } - } else if (typeof fileData.data === 'string' && fileData.data) { - let base64Data = fileData.data + } else if (typeof data.data === 'string' && data.data) { + let base64Data = data.data if (base64Data.includes('-') || base64Data.includes('_')) { base64Data = base64Data.replace(/-/g, '+').replace(/_/g, '/') @@ -126,20 +136,13 @@ export class FileToolProcessor { buffer = Buffer.from(base64Data, 'base64') } - if (!buffer && fileData.url) { - const response = await fetch(fileData.url) - - if (!response.ok) { - throw new Error(`Failed to download file from ${fileData.url}: ${response.statusText}`) - } - - const arrayBuffer = await response.arrayBuffer() - buffer = Buffer.from(arrayBuffer) + if (!buffer && data.url) { + buffer = await downloadFileFromUrl(data.url) } if (buffer) { if (buffer.length === 0) { - throw new Error(`File '${fileData.name}' has zero bytes`) + throw new Error(`File '${data.name}' has zero bytes`) } return await uploadExecutionFile( @@ -149,23 +152,23 @@ export class FileToolProcessor { executionId: context.executionId || '', }, buffer, - fileData.name, - fileData.mimeType, + data.name, + data.mimeType, context.userId ) } - if (!fileData.data) { + if (!data.data) { throw new Error( - `File data for '${fileData.name}' must have either 'data' (Buffer/base64) or 'url' property` + `File data for '${data.name}' must have either 'data' (Buffer/base64) or 'url' property` ) } return uploadFileFromRawData( { - name: fileData.name, - data: fileData.data, - mimeType: fileData.mimeType, + name: data.name, + data: data.data, + mimeType: data.mimeType, }, { workspaceId: context.workspaceId || '', @@ -175,7 +178,7 @@ export class FileToolProcessor { context.userId ) } catch (error) { - logger.error(`Error processing file data for '${fileData.name}':`, error) + logger.error(`Error processing file data for '${data.name}':`, error) throw error } } diff --git a/apps/sim/hooks/use-collaborative-workflow.ts b/apps/sim/hooks/use-collaborative-workflow.ts index 01268a575..8a7ef9c90 100644 --- a/apps/sim/hooks/use-collaborative-workflow.ts +++ b/apps/sim/hooks/use-collaborative-workflow.ts @@ -1274,8 +1274,6 @@ export function useCollaborativeWorkflow() { const operationId = crypto.randomUUID() - const currentActiveWorkflowId = useWorkflowRegistry.getState().activeWorkflowId - addToQueue({ id: operationId, operation: { @@ -1283,7 +1281,7 @@ export function useCollaborativeWorkflow() { target: OPERATION_TARGETS.SUBBLOCK, payload: { blockId, subblockId, value }, }, - workflowId: currentActiveWorkflowId || '', + workflowId: activeWorkflowId, userId: session?.user?.id || 'unknown', }) }, diff --git a/apps/sim/lib/a2a/push-notifications.ts b/apps/sim/lib/a2a/push-notifications.ts index da7fa3ad0..fc0adf195 100644 --- a/apps/sim/lib/a2a/push-notifications.ts +++ b/apps/sim/lib/a2a/push-notifications.ts @@ -4,7 +4,10 @@ import { a2aPushNotificationConfig, a2aTask } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { eq } from 'drizzle-orm' import { isTriggerDevEnabled } from '@/lib/core/config/feature-flags' -import { secureFetchWithPinnedIP, validateUrlWithDNS } from '@/lib/core/security/input-validation' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' const logger = createLogger('A2APushNotifications') diff --git a/apps/sim/lib/a2a/utils.ts b/apps/sim/lib/a2a/utils.ts index 3eddb5d8d..11d3c7ab5 100644 --- a/apps/sim/lib/a2a/utils.ts +++ b/apps/sim/lib/a2a/utils.ts @@ -7,6 +7,7 @@ import { ClientFactoryOptions, } from '@a2a-js/sdk/client' import { createLogger } from '@sim/logger' +import { isInternalFileUrl } from '@/lib/uploads/utils/file-utils' import { A2A_TERMINAL_STATES } from './constants' const logger = createLogger('A2AUtils') @@ -94,11 +95,13 @@ export function extractFileContent(message: Message): A2AFile[] { .map((part) => { const file = part.file as unknown as Record const uri = (file.url as string) || (file.uri as string) + const hasBytes = Boolean(file.bytes) + const canUseUri = Boolean(uri) && (!hasBytes || (uri ? !isInternalFileUrl(uri) : true)) return { name: file.name as string | undefined, mimeType: file.mimeType as string | undefined, - ...(uri ? { uri } : {}), - ...(file.bytes ? { bytes: file.bytes as string } : {}), + ...(canUseUri ? { uri } : {}), + ...(hasBytes ? { bytes: file.bytes as string } : {}), } }) } diff --git a/apps/sim/lib/core/security/input-validation.server.ts b/apps/sim/lib/core/security/input-validation.server.ts new file mode 100644 index 000000000..e8c0ec861 --- /dev/null +++ b/apps/sim/lib/core/security/input-validation.server.ts @@ -0,0 +1,312 @@ +import dns from 'dns/promises' +import http from 'http' +import https from 'https' +import type { LookupFunction } from 'net' +import { createLogger } from '@sim/logger' +import * as ipaddr from 'ipaddr.js' +import { type ValidationResult, validateExternalUrl } from '@/lib/core/security/input-validation' + +const logger = createLogger('InputValidation') + +/** + * Result type for async URL validation with resolved IP + */ +export interface AsyncValidationResult extends ValidationResult { + resolvedIP?: string + originalHostname?: string +} + +/** + * Checks if an IP address is private or reserved (not routable on the public internet) + * Uses ipaddr.js for robust handling of all IP formats including: + * - Octal notation (0177.0.0.1) + * - Hex notation (0x7f000001) + * - IPv4-mapped IPv6 (::ffff:127.0.0.1) + * - Various edge cases that regex patterns miss + */ +function isPrivateOrReservedIP(ip: string): boolean { + try { + if (!ipaddr.isValid(ip)) { + return true + } + + const addr = ipaddr.process(ip) + const range = addr.range() + + return range !== 'unicast' + } catch { + return true + } +} + +/** + * Validates a URL and resolves its DNS to prevent SSRF via DNS rebinding + * + * This function: + * 1. Performs basic URL validation (protocol, format) + * 2. Resolves the hostname to an IP address + * 3. Validates the resolved IP is not private/reserved + * 4. Returns the resolved IP for use in the actual request + * + * @param url - The URL to validate + * @param paramName - Name of the parameter for error messages + * @returns AsyncValidationResult with resolved IP for DNS pinning + */ +export async function validateUrlWithDNS( + url: string | null | undefined, + paramName = 'url' +): Promise { + const basicValidation = validateExternalUrl(url, paramName) + if (!basicValidation.isValid) { + return basicValidation + } + + const parsedUrl = new URL(url!) + const hostname = parsedUrl.hostname + + try { + const { address } = await dns.lookup(hostname) + + if (isPrivateOrReservedIP(address)) { + logger.warn('URL resolves to blocked IP address', { + paramName, + hostname, + resolvedIP: address, + }) + return { + isValid: false, + error: `${paramName} resolves to a blocked IP address`, + } + } + + return { + isValid: true, + resolvedIP: address, + originalHostname: hostname, + } + } catch (error) { + logger.warn('DNS lookup failed for URL', { + paramName, + hostname, + error: error instanceof Error ? error.message : String(error), + }) + return { + isValid: false, + error: `${paramName} hostname could not be resolved`, + } + } +} + +export interface SecureFetchOptions { + method?: string + headers?: Record + body?: string | Buffer | Uint8Array + timeout?: number + maxRedirects?: number +} + +export class SecureFetchHeaders { + private headers: Map + + constructor(headers: Record) { + this.headers = new Map(Object.entries(headers).map(([k, v]) => [k.toLowerCase(), v])) + } + + get(name: string): string | null { + return this.headers.get(name.toLowerCase()) ?? null + } + + toRecord(): Record { + const record: Record = {} + for (const [key, value] of this.headers) { + record[key] = value + } + return record + } + + [Symbol.iterator]() { + return this.headers.entries() + } +} + +export interface SecureFetchResponse { + ok: boolean + status: number + statusText: string + headers: SecureFetchHeaders + text: () => Promise + json: () => Promise + arrayBuffer: () => Promise +} + +const DEFAULT_MAX_REDIRECTS = 5 + +function isRedirectStatus(status: number): boolean { + return status >= 300 && status < 400 && status !== 304 +} + +function resolveRedirectUrl(baseUrl: string, location: string): string { + try { + return new URL(location, baseUrl).toString() + } catch { + throw new Error(`Invalid redirect location: ${location}`) + } +} + +/** + * Performs a fetch with IP pinning to prevent DNS rebinding attacks. + * Uses the pre-resolved IP address while preserving the original hostname for TLS SNI. + * Follows redirects securely by validating each redirect target. + */ +export async function secureFetchWithPinnedIP( + url: string, + resolvedIP: string, + options: SecureFetchOptions = {}, + redirectCount = 0 +): Promise { + const maxRedirects = options.maxRedirects ?? DEFAULT_MAX_REDIRECTS + + return new Promise((resolve, reject) => { + const parsed = new URL(url) + const isHttps = parsed.protocol === 'https:' + const defaultPort = isHttps ? 443 : 80 + const port = parsed.port ? Number.parseInt(parsed.port, 10) : defaultPort + + const isIPv6 = resolvedIP.includes(':') + const family = isIPv6 ? 6 : 4 + + const lookup: LookupFunction = (_hostname, options, callback) => { + if (options.all) { + callback(null, [{ address: resolvedIP, family }]) + } else { + callback(null, resolvedIP, family) + } + } + + const agentOptions: http.AgentOptions = { lookup } + + const agent = isHttps ? new https.Agent(agentOptions) : new http.Agent(agentOptions) + + // Remove accept-encoding since Node.js http/https doesn't auto-decompress + // Headers are lowercase due to Web Headers API normalization in executeToolRequest + const { 'accept-encoding': _, ...sanitizedHeaders } = options.headers ?? {} + + const requestOptions: http.RequestOptions = { + hostname: parsed.hostname, + port, + path: parsed.pathname + parsed.search, + method: options.method || 'GET', + headers: sanitizedHeaders, + agent, + timeout: options.timeout || 300000, // Default 5 minutes + } + + const protocol = isHttps ? https : http + const req = protocol.request(requestOptions, (res) => { + const statusCode = res.statusCode || 0 + const location = res.headers.location + + if (isRedirectStatus(statusCode) && location && redirectCount < maxRedirects) { + res.resume() + const redirectUrl = resolveRedirectUrl(url, location) + + validateUrlWithDNS(redirectUrl, 'redirectUrl') + .then((validation) => { + if (!validation.isValid) { + reject(new Error(`Redirect blocked: ${validation.error}`)) + return + } + return secureFetchWithPinnedIP( + redirectUrl, + validation.resolvedIP!, + options, + redirectCount + 1 + ) + }) + .then((response) => { + if (response) resolve(response) + }) + .catch(reject) + return + } + + if (isRedirectStatus(statusCode) && location && redirectCount >= maxRedirects) { + res.resume() + reject(new Error(`Too many redirects (max: ${maxRedirects})`)) + return + } + + const chunks: Buffer[] = [] + + res.on('data', (chunk: Buffer) => chunks.push(chunk)) + + res.on('error', (error) => { + reject(error) + }) + + res.on('end', () => { + const bodyBuffer = Buffer.concat(chunks) + const body = bodyBuffer.toString('utf-8') + const headersRecord: Record = {} + for (const [key, value] of Object.entries(res.headers)) { + if (typeof value === 'string') { + headersRecord[key.toLowerCase()] = value + } else if (Array.isArray(value)) { + headersRecord[key.toLowerCase()] = value.join(', ') + } + } + + resolve({ + ok: statusCode >= 200 && statusCode < 300, + status: statusCode, + statusText: res.statusMessage || '', + headers: new SecureFetchHeaders(headersRecord), + text: async () => body, + json: async () => JSON.parse(body), + arrayBuffer: async () => + bodyBuffer.buffer.slice( + bodyBuffer.byteOffset, + bodyBuffer.byteOffset + bodyBuffer.byteLength + ), + }) + }) + }) + + req.on('error', (error) => { + reject(error) + }) + + req.on('timeout', () => { + req.destroy() + reject(new Error(`Request timed out after ${requestOptions.timeout}ms`)) + }) + + if (options.body) { + req.write(options.body) + } + + req.end() + }) +} + +/** + * Validates a URL and performs a secure fetch with DNS pinning in one call. + * Combines validateUrlWithDNS and secureFetchWithPinnedIP for convenience. + * + * @param url - The URL to fetch + * @param options - Fetch options (method, headers, body, etc.) + * @param paramName - Name of the parameter for error messages (default: 'url') + * @returns SecureFetchResponse + * @throws Error if URL validation fails + */ +export async function secureFetchWithValidation( + url: string, + options: SecureFetchOptions = {}, + paramName = 'url' +): Promise { + const validation = await validateUrlWithDNS(url, paramName) + if (!validation.isValid) { + throw new Error(validation.error) + } + return secureFetchWithPinnedIP(url, validation.resolvedIP!, options) +} diff --git a/apps/sim/lib/core/security/input-validation.test.ts b/apps/sim/lib/core/security/input-validation.test.ts index 7575b6546..a2b842d40 100644 --- a/apps/sim/lib/core/security/input-validation.test.ts +++ b/apps/sim/lib/core/security/input-validation.test.ts @@ -18,8 +18,8 @@ import { validatePathSegment, validateProxyUrl, validateS3BucketName, - validateUrlWithDNS, } from '@/lib/core/security/input-validation' +import { validateUrlWithDNS } from '@/lib/core/security/input-validation.server' import { sanitizeForLogging } from '@/lib/core/security/redaction' vi.mock('@sim/logger', () => loggerMock) diff --git a/apps/sim/lib/core/security/input-validation.ts b/apps/sim/lib/core/security/input-validation.ts index e27524b2c..e156c7ad4 100644 --- a/apps/sim/lib/core/security/input-validation.ts +++ b/apps/sim/lib/core/security/input-validation.ts @@ -1,7 +1,3 @@ -import dns from 'dns/promises' -import http from 'http' -import https from 'https' -import type { LookupFunction } from 'net' import { createLogger } from '@sim/logger' import * as ipaddr from 'ipaddr.js' @@ -765,263 +761,6 @@ function isPrivateOrReservedIP(ip: string): boolean { } } -/** - * Result type for async URL validation with resolved IP - */ -export interface AsyncValidationResult extends ValidationResult { - resolvedIP?: string - originalHostname?: string -} - -/** - * Validates a URL and resolves its DNS to prevent SSRF via DNS rebinding - * - * This function: - * 1. Performs basic URL validation (protocol, format) - * 2. Resolves the hostname to an IP address - * 3. Validates the resolved IP is not private/reserved - * 4. Returns the resolved IP for use in the actual request - * - * @param url - The URL to validate - * @param paramName - Name of the parameter for error messages - * @returns AsyncValidationResult with resolved IP for DNS pinning - */ -export async function validateUrlWithDNS( - url: string | null | undefined, - paramName = 'url' -): Promise { - const basicValidation = validateExternalUrl(url, paramName) - if (!basicValidation.isValid) { - return basicValidation - } - - const parsedUrl = new URL(url!) - const hostname = parsedUrl.hostname - - try { - const { address } = await dns.lookup(hostname) - - if (isPrivateOrReservedIP(address)) { - logger.warn('URL resolves to blocked IP address', { - paramName, - hostname, - resolvedIP: address, - }) - return { - isValid: false, - error: `${paramName} resolves to a blocked IP address`, - } - } - - return { - isValid: true, - resolvedIP: address, - originalHostname: hostname, - } - } catch (error) { - logger.warn('DNS lookup failed for URL', { - paramName, - hostname, - error: error instanceof Error ? error.message : String(error), - }) - return { - isValid: false, - error: `${paramName} hostname could not be resolved`, - } - } -} -export interface SecureFetchOptions { - method?: string - headers?: Record - body?: string - timeout?: number - maxRedirects?: number -} - -export class SecureFetchHeaders { - private headers: Map - - constructor(headers: Record) { - this.headers = new Map(Object.entries(headers).map(([k, v]) => [k.toLowerCase(), v])) - } - - get(name: string): string | null { - return this.headers.get(name.toLowerCase()) ?? null - } - - toRecord(): Record { - const record: Record = {} - for (const [key, value] of this.headers) { - record[key] = value - } - return record - } - - [Symbol.iterator]() { - return this.headers.entries() - } -} - -export interface SecureFetchResponse { - ok: boolean - status: number - statusText: string - headers: SecureFetchHeaders - text: () => Promise - json: () => Promise - arrayBuffer: () => Promise -} - -const DEFAULT_MAX_REDIRECTS = 5 - -function isRedirectStatus(status: number): boolean { - return status >= 300 && status < 400 && status !== 304 -} - -function resolveRedirectUrl(baseUrl: string, location: string): string { - try { - return new URL(location, baseUrl).toString() - } catch { - throw new Error(`Invalid redirect location: ${location}`) - } -} - -/** - * Performs a fetch with IP pinning to prevent DNS rebinding attacks. - * Uses the pre-resolved IP address while preserving the original hostname for TLS SNI. - * Follows redirects securely by validating each redirect target. - */ -export async function secureFetchWithPinnedIP( - url: string, - resolvedIP: string, - options: SecureFetchOptions = {}, - redirectCount = 0 -): Promise { - const maxRedirects = options.maxRedirects ?? DEFAULT_MAX_REDIRECTS - - return new Promise((resolve, reject) => { - const parsed = new URL(url) - const isHttps = parsed.protocol === 'https:' - const defaultPort = isHttps ? 443 : 80 - const port = parsed.port ? Number.parseInt(parsed.port, 10) : defaultPort - - const isIPv6 = resolvedIP.includes(':') - const family = isIPv6 ? 6 : 4 - - const lookup: LookupFunction = (_hostname, options, callback) => { - if (options.all) { - callback(null, [{ address: resolvedIP, family }]) - } else { - callback(null, resolvedIP, family) - } - } - - const agentOptions: http.AgentOptions = { lookup } - - const agent = isHttps ? new https.Agent(agentOptions) : new http.Agent(agentOptions) - - // Remove accept-encoding since Node.js http/https doesn't auto-decompress - // Headers are lowercase due to Web Headers API normalization in executeToolRequest - const { 'accept-encoding': _, ...sanitizedHeaders } = options.headers ?? {} - - const requestOptions: http.RequestOptions = { - hostname: parsed.hostname, - port, - path: parsed.pathname + parsed.search, - method: options.method || 'GET', - headers: sanitizedHeaders, - agent, - timeout: options.timeout || 300000, // Default 5 minutes - } - - const protocol = isHttps ? https : http - const req = protocol.request(requestOptions, (res) => { - const statusCode = res.statusCode || 0 - const location = res.headers.location - - if (isRedirectStatus(statusCode) && location && redirectCount < maxRedirects) { - res.resume() - const redirectUrl = resolveRedirectUrl(url, location) - - validateUrlWithDNS(redirectUrl, 'redirectUrl') - .then((validation) => { - if (!validation.isValid) { - reject(new Error(`Redirect blocked: ${validation.error}`)) - return - } - return secureFetchWithPinnedIP( - redirectUrl, - validation.resolvedIP!, - options, - redirectCount + 1 - ) - }) - .then((response) => { - if (response) resolve(response) - }) - .catch(reject) - return - } - - if (isRedirectStatus(statusCode) && location && redirectCount >= maxRedirects) { - res.resume() - reject(new Error(`Too many redirects (max: ${maxRedirects})`)) - return - } - - const chunks: Buffer[] = [] - - res.on('data', (chunk: Buffer) => chunks.push(chunk)) - - res.on('error', (error) => { - reject(error) - }) - - res.on('end', () => { - const bodyBuffer = Buffer.concat(chunks) - const body = bodyBuffer.toString('utf-8') - const headersRecord: Record = {} - for (const [key, value] of Object.entries(res.headers)) { - if (typeof value === 'string') { - headersRecord[key.toLowerCase()] = value - } else if (Array.isArray(value)) { - headersRecord[key.toLowerCase()] = value.join(', ') - } - } - - resolve({ - ok: statusCode >= 200 && statusCode < 300, - status: statusCode, - statusText: res.statusMessage || '', - headers: new SecureFetchHeaders(headersRecord), - text: async () => body, - json: async () => JSON.parse(body), - arrayBuffer: async () => - bodyBuffer.buffer.slice( - bodyBuffer.byteOffset, - bodyBuffer.byteOffset + bodyBuffer.byteLength - ), - }) - }) - }) - - req.on('error', (error) => { - reject(error) - }) - - req.on('timeout', () => { - req.destroy() - reject(new Error(`Request timed out after ${requestOptions.timeout}ms`)) - }) - - if (options.body) { - req.write(options.body) - } - - req.end() - }) -} - /** * Validates an Airtable ID (base, table, or webhook ID) * diff --git a/apps/sim/lib/core/utils/logging.ts b/apps/sim/lib/core/utils/logging.ts new file mode 100644 index 000000000..5670d6a5d --- /dev/null +++ b/apps/sim/lib/core/utils/logging.ts @@ -0,0 +1,19 @@ +/** + * Sanitize URLs for logging by stripping query/hash and truncating. + */ +export function sanitizeUrlForLog(url: string, maxLength = 120): string { + if (!url) return '' + + const trimmed = url.trim() + try { + const hasProtocol = /^[a-zA-Z][a-zA-Z\d+\-.]*:/.test(trimmed) + const parsed = new URL(trimmed, hasProtocol ? undefined : 'http://localhost') + const origin = parsed.origin === 'null' ? '' : parsed.origin + const sanitized = `${origin}${parsed.pathname}` + const result = sanitized || parsed.pathname || trimmed + return result.length > maxLength ? `${result.slice(0, maxLength)}...` : result + } catch { + const withoutQuery = trimmed.split('?')[0].split('#')[0] + return withoutQuery.length > maxLength ? `${withoutQuery.slice(0, maxLength)}...` : withoutQuery + } +} diff --git a/apps/sim/lib/core/utils/urls.ts b/apps/sim/lib/core/utils/urls.ts index 22e164cf1..5021d4494 100644 --- a/apps/sim/lib/core/utils/urls.ts +++ b/apps/sim/lib/core/utils/urls.ts @@ -24,6 +24,22 @@ export function getBaseUrl(): string { return `${protocol}${baseUrl}` } +/** + * Ensures a URL is absolute by prefixing the base URL when a relative path is provided. + * @param pathOrUrl - Relative path (e.g., /api/files/serve/...) or absolute URL + */ +export function ensureAbsoluteUrl(pathOrUrl: string): string { + if (!pathOrUrl) { + throw new Error('URL is required') + } + + if (pathOrUrl.startsWith('/')) { + return `${getBaseUrl()}${pathOrUrl}` + } + + return pathOrUrl +} + /** * Returns just the domain and port part of the application URL * @returns The domain with port if applicable (e.g., 'localhost:3000' or 'sim.ai') diff --git a/apps/sim/lib/core/utils/user-file.ts b/apps/sim/lib/core/utils/user-file.ts index f2b034047..0069eb4fb 100644 --- a/apps/sim/lib/core/utils/user-file.ts +++ b/apps/sim/lib/core/utils/user-file.ts @@ -55,3 +55,20 @@ export function filterUserFileForDisplay(data: Record): Record< } return filtered } + +/** + * Extracts base64 content from either a raw base64 string or a UserFile object. + * Useful for tools that accept file input in either format. + * @returns The base64 string, or undefined if not found + */ +export function extractBase64FromFileInput( + input: string | UserFileLike | null | undefined +): string | undefined { + if (typeof input === 'string') { + return input + } + if (input?.base64) { + return input.base64 + } + return undefined +} diff --git a/apps/sim/lib/execution/files.ts b/apps/sim/lib/execution/files.ts index 9eb26905e..5ac2c50b0 100644 --- a/apps/sim/lib/execution/files.ts +++ b/apps/sim/lib/execution/files.ts @@ -1,5 +1,4 @@ import { createLogger } from '@sim/logger' -import { v4 as uuidv4 } from 'uuid' import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' import { TRIGGER_TYPES } from '@/lib/workflows/triggers/triggers' import type { InputFormatField } from '@/lib/workflows/types' @@ -11,7 +10,7 @@ const logger = createLogger('ExecutionFiles') const MAX_FILE_SIZE = 20 * 1024 * 1024 // 20MB /** - * Process a single file for workflow execution - handles both base64 ('file' type) and URL pass-through ('url' type) + * Process a single file for workflow execution - handles base64 ('file' type) and URL downloads ('url' type) */ export async function processExecutionFile( file: { type: string; data: string; name: string; mime?: string }, @@ -60,14 +59,28 @@ export async function processExecutionFile( } if (file.type === 'url' && file.data) { - return { - id: uuidv4(), - url: file.data, - name: file.name, - size: 0, - type: file.mime || 'application/octet-stream', - key: `url/${file.name}`, + const { downloadFileFromUrl } = await import('@/lib/uploads/utils/file-utils.server') + const buffer = await downloadFileFromUrl(file.data) + + if (buffer.length > MAX_FILE_SIZE) { + const fileSizeMB = (buffer.length / (1024 * 1024)).toFixed(2) + throw new Error( + `File "${file.name}" exceeds the maximum size limit of 20MB (actual size: ${fileSizeMB}MB)` + ) } + + logger.debug(`[${requestId}] Uploading file from URL: ${file.name} (${buffer.length} bytes)`) + + const userFile = await uploadExecutionFile( + executionContext, + buffer, + file.name, + file.mime || 'application/octet-stream', + userId + ) + + logger.debug(`[${requestId}] Successfully uploaded ${file.name} from URL`) + return userFile } return null @@ -163,7 +176,7 @@ export async function processInputFileFields( } const inputFormat = extractInputFormatFromBlock(startBlock) - const fileFields = inputFormat.filter((field) => field.type === 'files') + const fileFields = inputFormat.filter((field) => field.type === 'file[]') if (fileFields.length === 0) { return input diff --git a/apps/sim/lib/knowledge/documents/document-processor.ts b/apps/sim/lib/knowledge/documents/document-processor.ts index 632e91fa8..37896d9a3 100644 --- a/apps/sim/lib/knowledge/documents/document-processor.ts +++ b/apps/sim/lib/knowledge/documents/document-processor.ts @@ -7,6 +7,7 @@ import { parseBuffer, parseFile } from '@/lib/file-parsers' import type { FileParseMetadata } from '@/lib/file-parsers/types' import { retryWithExponentialBackoff } from '@/lib/knowledge/documents/utils' import { StorageService } from '@/lib/uploads' +import { isInternalFileUrl } from '@/lib/uploads/utils/file-utils' import { downloadFileFromUrl } from '@/lib/uploads/utils/file-utils.server' import { mistralParserTool } from '@/tools/mistral/parser' @@ -245,7 +246,7 @@ async function handleFileForOCR( userId?: string, workspaceId?: string | null ) { - const isExternalHttps = fileUrl.startsWith('https://') && !fileUrl.includes('/api/files/serve/') + const isExternalHttps = fileUrl.startsWith('https://') && !isInternalFileUrl(fileUrl) if (isExternalHttps) { if (mimeType === 'application/pdf') { @@ -489,7 +490,7 @@ async function parseWithMistralOCR( workspaceId ) - logger.info(`Mistral OCR: Using presigned URL for ${filename}: ${httpsUrl.substring(0, 120)}...`) + logger.info(`Mistral OCR: Using presigned URL for ${filename}: ${httpsUrl}`) let pageCount = 0 if (mimeType === 'application/pdf' && buffer) { diff --git a/apps/sim/lib/mcp/workflow-tool-schema.ts b/apps/sim/lib/mcp/workflow-tool-schema.ts index 7af927ff1..4678e96b3 100644 --- a/apps/sim/lib/mcp/workflow-tool-schema.ts +++ b/apps/sim/lib/mcp/workflow-tool-schema.ts @@ -153,7 +153,7 @@ export function generateToolInputSchema(inputFormat: InputFormatField[]): McpToo // Handle array types if (fieldType === 'array') { - if (field.type === 'files') { + if (field.type === 'file[]') { property.items = { type: 'object', properties: { diff --git a/apps/sim/lib/uploads/contexts/execution/execution-file-manager.ts b/apps/sim/lib/uploads/contexts/execution/execution-file-manager.ts index bbf2a123e..6c237668c 100644 --- a/apps/sim/lib/uploads/contexts/execution/execution-file-manager.ts +++ b/apps/sim/lib/uploads/contexts/execution/execution-file-manager.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { isUserFileWithMetadata } from '@/lib/core/utils/user-file' +import { StorageService } from '@/lib/uploads' import type { ExecutionContext } from '@/lib/uploads/contexts/execution/utils' import { generateExecutionFileKey, generateFileId } from '@/lib/uploads/contexts/execution/utils' import type { UserFile } from '@/executor/types' @@ -89,10 +90,7 @@ export async function uploadExecutionFile( } try { - const { uploadFile, generatePresignedDownloadUrl } = await import( - '@/lib/uploads/core/storage-service' - ) - const fileInfo = await uploadFile({ + const fileInfo = await StorageService.uploadFile({ file: fileBuffer, fileName: storageKey, contentType, @@ -102,21 +100,24 @@ export async function uploadExecutionFile( metadata, // Pass metadata for cloud storage and database tracking }) - // Generate presigned URL for file access (10 minutes expiration) - const fullUrl = await generatePresignedDownloadUrl(fileInfo.key, 'execution', 600) + const presignedUrl = await StorageService.generatePresignedDownloadUrl( + fileInfo.key, + 'execution', + 5 * 60 + ) const userFile: UserFile = { id: fileId, name: fileName, size: fileBuffer.length, type: contentType, - url: fullUrl, // Presigned URL for external access and downstream workflow usage + url: presignedUrl, key: fileInfo.key, - context: 'execution', // Preserve context in file object + context: 'execution', + base64: fileBuffer.toString('base64'), } logger.info(`Successfully uploaded execution file: ${fileName} (${fileBuffer.length} bytes)`, { - url: fullUrl, key: fileInfo.key, }) return userFile @@ -135,8 +136,7 @@ export async function downloadExecutionFile(userFile: UserFile): Promise logger.info(`Downloading execution file: ${userFile.name}`) try { - const { downloadFile } = await import('@/lib/uploads/core/storage-service') - const fileBuffer = await downloadFile({ + const fileBuffer = await StorageService.downloadFile({ key: userFile.key, context: 'execution', }) diff --git a/apps/sim/lib/uploads/utils/file-schemas.ts b/apps/sim/lib/uploads/utils/file-schemas.ts new file mode 100644 index 000000000..b010a99b6 --- /dev/null +++ b/apps/sim/lib/uploads/utils/file-schemas.ts @@ -0,0 +1,52 @@ +import { z } from 'zod' +import { isInternalFileUrl } from '@/lib/uploads/utils/file-utils' + +const isUrlLike = (value: string) => + value.startsWith('http://') || value.startsWith('https://') || value.startsWith('/') + +export const RawFileInputSchema = z + .object({ + id: z.string().optional(), + key: z.string().optional(), + path: z.string().optional(), + url: z.string().optional(), + name: z.string().min(1), + size: z.number().nonnegative(), + type: z.string().optional(), + uploadedAt: z.union([z.string(), z.date()]).optional(), + expiresAt: z.union([z.string(), z.date()]).optional(), + context: z.string().optional(), + base64: z.string().optional(), + }) + .passthrough() + .refine((data) => Boolean(data.key || data.path || data.url), { + message: 'File must include key, path, or url', + }) + .refine( + (data) => { + if (data.key || data.path) { + return true + } + if (!data.url) { + return true + } + return isInternalFileUrl(data.url) + }, + { message: 'File url must reference an uploaded file' } + ) + .refine( + (data) => { + if (data.key || !data.path) { + return true + } + if (!isUrlLike(data.path)) { + return true + } + return isInternalFileUrl(data.path) + }, + { message: 'File path must reference an uploaded file' } + ) + +export const RawFileInputArraySchema = z.array(RawFileInputSchema) + +export const FileInputSchema = z.union([RawFileInputSchema, z.string()]) diff --git a/apps/sim/lib/uploads/utils/file-utils.server.ts b/apps/sim/lib/uploads/utils/file-utils.server.ts index c2f14e97e..b759918d0 100644 --- a/apps/sim/lib/uploads/utils/file-utils.server.ts +++ b/apps/sim/lib/uploads/utils/file-utils.server.ts @@ -1,19 +1,141 @@ 'use server' import type { Logger } from '@sim/logger' -import { secureFetchWithPinnedIP, validateUrlWithDNS } from '@/lib/core/security/input-validation' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' import type { StorageContext } from '@/lib/uploads' +import { StorageService } from '@/lib/uploads' import { isExecutionFile } from '@/lib/uploads/contexts/execution/utils' -import { inferContextFromKey } from '@/lib/uploads/utils/file-utils' +import { + extractStorageKey, + inferContextFromKey, + isInternalFileUrl, + processSingleFileToUserFile, + type RawFileInput, +} from '@/lib/uploads/utils/file-utils' +import { verifyFileAccess } from '@/app/api/files/authorization' import type { UserFile } from '@/executor/types' +/** + * Result type for file input resolution + */ +export interface FileResolutionResult { + fileUrl?: string + error?: { + status: number + message: string + } +} + +/** + * Options for resolving file input to a URL + */ +export interface ResolveFileInputOptions { + file?: RawFileInput + filePath?: string + userId: string + requestId: string + logger: Logger +} + +/** + * Resolves file input (either a file object or filePath string) to a publicly accessible URL. + * Handles: + * - Processing raw file input via processSingleFileToUserFile + * - Resolving internal URLs via resolveInternalFileUrl + * - Generating presigned URLs for storage keys + * - Validating external URLs via validateUrlWithDNS + */ +export async function resolveFileInputToUrl( + options: ResolveFileInputOptions +): Promise { + const { file, filePath, userId, requestId, logger } = options + + if (file) { + let userFile: UserFile + try { + userFile = processSingleFileToUserFile(file, requestId, logger) + } catch (error) { + return { + error: { + status: 400, + message: error instanceof Error ? error.message : 'Failed to process file', + }, + } + } + + let fileUrl = userFile.url || '' + + // Handle internal URLs + if (fileUrl && isInternalFileUrl(fileUrl)) { + const resolution = await resolveInternalFileUrl(fileUrl, userId, requestId, logger) + if (resolution.error) { + return { error: resolution.error } + } + fileUrl = resolution.fileUrl || '' + } + + // Generate presigned URL if we have a key but no URL + if (!fileUrl && userFile.key) { + const context = (userFile.context as StorageContext) || inferContextFromKey(userFile.key) + const hasAccess = await verifyFileAccess(userFile.key, userId, undefined, context, false) + + if (!hasAccess) { + logger.warn(`[${requestId}] Unauthorized presigned URL generation attempt`, { + userId, + key: userFile.key, + context, + }) + return { error: { status: 404, message: 'File not found' } } + } + + fileUrl = await StorageService.generatePresignedDownloadUrl(userFile.key, context, 5 * 60) + } + + return { fileUrl } + } + + if (filePath) { + let fileUrl = filePath + + if (isInternalFileUrl(filePath)) { + const resolution = await resolveInternalFileUrl(filePath, userId, requestId, logger) + if (resolution.error) { + return { error: resolution.error } + } + fileUrl = resolution.fileUrl || fileUrl + } else if (filePath.startsWith('/')) { + logger.warn(`[${requestId}] Invalid internal path`, { + userId, + path: filePath.substring(0, 50), + }) + return { + error: { + status: 400, + message: 'Invalid file path. Only uploaded files are supported for internal paths.', + }, + } + } else { + const urlValidation = await validateUrlWithDNS(fileUrl, 'filePath') + if (!urlValidation.isValid) { + return { error: { status: 400, message: urlValidation.error || 'Invalid URL' } } + } + } + + return { fileUrl } + } + + return { error: { status: 400, message: 'File input is required' } } +} + /** * Download a file from a URL (internal or external) * For internal URLs, uses direct storage access (server-side only) * For external URLs, validates DNS/SSRF and uses secure fetch with IP pinning */ export async function downloadFileFromUrl(fileUrl: string, timeoutMs = 180000): Promise { - const { isInternalFileUrl } = await import('./file-utils') const { parseInternalFileUrl } = await import('./file-utils') if (isInternalFileUrl(fileUrl)) { @@ -38,6 +160,39 @@ export async function downloadFileFromUrl(fileUrl: string, timeoutMs = 180000): return Buffer.from(await response.arrayBuffer()) } +export async function resolveInternalFileUrl( + filePath: string, + userId: string, + requestId: string, + logger: Logger +): Promise<{ fileUrl?: string; error?: { status: number; message: string } }> { + if (!isInternalFileUrl(filePath)) { + return { fileUrl: filePath } + } + + try { + const storageKey = extractStorageKey(filePath) + const context = inferContextFromKey(storageKey) + const hasAccess = await verifyFileAccess(storageKey, userId, undefined, context, false) + + if (!hasAccess) { + logger.warn(`[${requestId}] Unauthorized presigned URL generation attempt`, { + userId, + key: storageKey, + context, + }) + return { error: { status: 404, message: 'File not found' } } + } + + const fileUrl = await StorageService.generatePresignedDownloadUrl(storageKey, context, 5 * 60) + logger.info(`[${requestId}] Generated presigned URL for ${context} file`) + return { fileUrl } + } catch (error) { + logger.error(`[${requestId}] Failed to generate presigned URL:`, error) + return { error: { status: 500, message: 'Failed to generate file access URL' } } + } +} + /** * Downloads a file from storage (execution or regular) * @param userFile - UserFile object diff --git a/apps/sim/lib/uploads/utils/file-utils.ts b/apps/sim/lib/uploads/utils/file-utils.ts index 7b1d925ec..8c1ed04d5 100644 --- a/apps/sim/lib/uploads/utils/file-utils.ts +++ b/apps/sim/lib/uploads/utils/file-utils.ts @@ -438,6 +438,7 @@ export interface RawFileInput { uploadedAt?: string | Date expiresAt?: string | Date context?: string + base64?: string } /** @@ -450,43 +451,92 @@ function isCompleteUserFile(file: RawFileInput): file is UserFile { typeof file.url === 'string' && typeof file.size === 'number' && typeof file.type === 'string' && - typeof file.key === 'string' && - typeof file.uploadedAt === 'string' && - typeof file.expiresAt === 'string' + typeof file.key === 'string' ) } +function isUrlLike(value: string): boolean { + return value.startsWith('http://') || value.startsWith('https://') || value.startsWith('/') +} + /** - * Converts a single raw file object to UserFile format - * @param file - Raw file object - * @param requestId - Request ID for logging - * @param logger - Logger instance - * @returns UserFile object - * @throws Error if file has no storage key + * Extracts HTTPS URL from a file input object (UserFile or RawFileInput) + * Returns null if no valid HTTPS URL is found */ -export function processSingleFileToUserFile( - file: RawFileInput, - requestId: string, - logger: Logger -): UserFile { - if (isCompleteUserFile(file)) { - return file +export function resolveHttpsUrlFromFileInput(fileInput: unknown): string | null { + if (!fileInput || typeof fileInput !== 'object') { + return null } - const storageKey = file.key || (file.path ? extractStorageKey(file.path) : null) + const record = fileInput as Record + const url = + typeof record.url === 'string' + ? record.url.trim() + : typeof record.path === 'string' + ? record.path.trim() + : '' + if (!url || !url.startsWith('https://')) { + return null + } + + return url +} + +function resolveStorageKeyFromRawFile(file: RawFileInput): string | null { + if (file.key) { + return file.key + } + + if (file.path) { + if (isUrlLike(file.path)) { + return isInternalFileUrl(file.path) ? extractStorageKey(file.path) : null + } + return file.path + } + + if (file.url) { + return isInternalFileUrl(file.url) ? extractStorageKey(file.url) : null + } + + return null +} + +function resolveInternalFileUrl(file: RawFileInput): string { + if (file.url && isInternalFileUrl(file.url)) { + return file.url + } + if (file.path && isInternalFileUrl(file.path)) { + return file.path + } + return '' +} + +/** + * Core conversion logic from RawFileInput to UserFile + */ +function convertToUserFile(file: RawFileInput, requestId: string, logger: Logger): UserFile | null { + if (isCompleteUserFile(file)) { + return { + ...file, + url: resolveInternalFileUrl(file) || file.url, + } + } + + const storageKey = resolveStorageKeyFromRawFile(file) if (!storageKey) { - logger.warn(`[${requestId}] File has no storage key: ${file.name || 'unknown'}`) - throw new Error(`File has no storage key: ${file.name || 'unknown'}`) + return null } const userFile: UserFile = { id: file.id || `file-${Date.now()}`, name: file.name, - url: file.url || file.path || '', + url: resolveInternalFileUrl(file), size: file.size, type: file.type || 'application/octet-stream', key: storageKey, + context: file.context, + base64: file.base64, } logger.info(`[${requestId}] Converted file to UserFile: ${userFile.name} (key: ${userFile.key})`) @@ -494,27 +544,52 @@ export function processSingleFileToUserFile( } /** - * Converts raw file objects (from file-upload or variable references) to UserFile format - * @param files - Array of raw file objects - * @param requestId - Request ID for logging - * @param logger - Logger instance - * @returns Array of UserFile objects + * Converts a single raw file object to UserFile format + * @throws Error if file is an array or has no storage key + */ +export function processSingleFileToUserFile( + file: RawFileInput, + requestId: string, + logger: Logger +): UserFile { + if (Array.isArray(file)) { + const errorMsg = `Expected a single file but received an array with ${file.length} file(s). Use a file input that accepts multiple files, or select a specific file from the array (e.g., {{block.files[0]}}).` + logger.error(`[${requestId}] ${errorMsg}`) + throw new Error(errorMsg) + } + + const userFile = convertToUserFile(file, requestId, logger) + if (!userFile) { + const errorMsg = `File has no storage key: ${file.name || 'unknown'}` + logger.warn(`[${requestId}] ${errorMsg}`) + throw new Error(errorMsg) + } + + return userFile +} + +/** + * Converts raw file objects to UserFile format, accepting single or array input */ export function processFilesToUserFiles( - files: RawFileInput[], + files: RawFileInput | RawFileInput[], requestId: string, logger: Logger ): UserFile[] { + const filesArray = Array.isArray(files) ? files : [files] const userFiles: UserFile[] = [] - for (const file of files) { - try { - const userFile = processSingleFileToUserFile(file, requestId, logger) + for (const file of filesArray) { + if (Array.isArray(file)) { + logger.warn(`[${requestId}] Skipping nested array in file input`) + continue + } + + const userFile = convertToUserFile(file, requestId, logger) + if (userFile) { userFiles.push(userFile) - } catch (error) { - logger.warn( - `[${requestId}] Skipping file: ${error instanceof Error ? error.message : 'Unknown error'}` - ) + } else { + logger.warn(`[${requestId}] Skipping file without storage key: ${file.name || 'unknown'}`) } } diff --git a/apps/sim/lib/webhooks/attachment-processor.ts b/apps/sim/lib/webhooks/attachment-processor.ts index cf2adbcef..0cbbf494e 100644 --- a/apps/sim/lib/webhooks/attachment-processor.ts +++ b/apps/sim/lib/webhooks/attachment-processor.ts @@ -77,7 +77,7 @@ export class WebhookAttachmentProcessor { userId?: string } ): Promise { - return uploadFileFromRawData( + const userFile = await uploadFileFromRawData( { name: attachment.name, data: attachment.data, @@ -86,5 +86,14 @@ export class WebhookAttachmentProcessor { executionContext, executionContext.userId ) + + if (userFile.base64) { + return userFile + } + + return { + ...userFile, + base64: attachment.data.toString('base64'), + } } } diff --git a/apps/sim/lib/webhooks/rss-polling-service.ts b/apps/sim/lib/webhooks/rss-polling-service.ts index ce282ef0d..5fbdeaba3 100644 --- a/apps/sim/lib/webhooks/rss-polling-service.ts +++ b/apps/sim/lib/webhooks/rss-polling-service.ts @@ -5,7 +5,10 @@ import { and, eq, isNull, or, sql } from 'drizzle-orm' import { nanoid } from 'nanoid' import Parser from 'rss-parser' import { pollingIdempotency } from '@/lib/core/idempotency/service' -import { secureFetchWithPinnedIP, validateUrlWithDNS } from '@/lib/core/security/input-validation' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' import { getBaseUrl } from '@/lib/core/utils/urls' import { MAX_CONSECUTIVE_FAILURES } from '@/triggers/constants' diff --git a/apps/sim/lib/webhooks/utils.server.ts b/apps/sim/lib/webhooks/utils.server.ts index f3cd6b436..8b99f7dec 100644 --- a/apps/sim/lib/webhooks/utils.server.ts +++ b/apps/sim/lib/webhooks/utils.server.ts @@ -10,7 +10,8 @@ import { type SecureFetchResponse, secureFetchWithPinnedIP, validateUrlWithDNS, -} from '@/lib/core/security/input-validation' +} from '@/lib/core/security/input-validation.server' +import { sanitizeUrlForLog } from '@/lib/core/utils/logging' import type { DbOrTx } from '@/lib/db/types' import { getProviderIdFromServiceId } from '@/lib/oauth' import { @@ -114,7 +115,7 @@ async function fetchWithDNSPinning( const urlValidation = await validateUrlWithDNS(url, 'contentUrl') if (!urlValidation.isValid) { logger.warn(`[${requestId}] Invalid content URL: ${urlValidation.error}`, { - url: url.substring(0, 100), + url, }) return null } @@ -133,7 +134,7 @@ async function fetchWithDNSPinning( } catch (error) { logger.error(`[${requestId}] Error fetching URL with DNS pinning`, { error: error instanceof Error ? error.message : String(error), - url: url.substring(0, 100), + url: sanitizeUrlForLog(url), }) return null } diff --git a/apps/sim/lib/workflows/blocks/block-outputs.ts b/apps/sim/lib/workflows/blocks/block-outputs.ts index 96833fa87..edb95fdf0 100644 --- a/apps/sim/lib/workflows/blocks/block-outputs.ts +++ b/apps/sim/lib/workflows/blocks/block-outputs.ts @@ -123,13 +123,13 @@ function filterOutputsByCondition( const CHAT_OUTPUTS: OutputDefinition = { input: { type: 'string', description: 'User message' }, conversationId: { type: 'string', description: 'Conversation ID' }, - files: { type: 'files', description: 'Uploaded files' }, + files: { type: 'file[]', description: 'Uploaded files' }, } const UNIFIED_START_OUTPUTS: OutputDefinition = { input: { type: 'string', description: 'Primary user input or message' }, conversationId: { type: 'string', description: 'Conversation thread identifier' }, - files: { type: 'files', description: 'User uploaded files' }, + files: { type: 'file[]', description: 'User uploaded files' }, } function applyInputFormatFields( @@ -341,6 +341,17 @@ function expandFileTypeProperties(path: string): string[] { return USER_FILE_ACCESSIBLE_PROPERTIES.map((prop) => `${path}.${prop}`) } +type FileOutputType = 'file' | 'file[]' + +function isFileOutputDefinition(value: unknown): value is { type: FileOutputType } { + if (!value || typeof value !== 'object' || !('type' in value)) { + return false + } + + const { type } = value as { type?: unknown } + return type === 'file' || type === 'file[]' +} + export function getBlockOutputPaths( blockType: string, subBlocks?: Record, @@ -373,13 +384,7 @@ function getFilePropertyType(outputs: OutputDefinition, pathParts: string[]): st current = (current as Record)[part] } - if ( - current && - typeof current === 'object' && - 'type' in current && - ((current as { type: unknown }).type === 'files' || - (current as { type: unknown }).type === 'file[]') - ) { + if (isFileOutputDefinition(current)) { return USER_FILE_PROPERTY_TYPES[lastPart as keyof typeof USER_FILE_PROPERTY_TYPES] } @@ -485,7 +490,7 @@ function generateOutputPaths(outputs: Record, prefix = ''): string[ paths.push(currentPath) } else if (typeof value === 'object' && value !== null) { if ('type' in value && typeof value.type === 'string') { - if (value.type === 'files' || value.type === 'file[]') { + if (isFileOutputDefinition(value)) { paths.push(...expandFileTypeProperties(currentPath)) continue } @@ -546,7 +551,7 @@ function generateOutputPathsWithTypes( paths.push({ path: currentPath, type: value }) } else if (typeof value === 'object' && value !== null) { if ('type' in value && typeof value.type === 'string') { - if (value.type === 'files' || value.type === 'file[]') { + if (isFileOutputDefinition(value)) { paths.push({ path: currentPath, type: value.type }) for (const prop of USER_FILE_ACCESSIBLE_PROPERTIES) { paths.push({ diff --git a/apps/sim/lib/workflows/operations/deployment-utils.ts b/apps/sim/lib/workflows/operations/deployment-utils.ts index c0dce11aa..7da79ce2b 100644 --- a/apps/sim/lib/workflows/operations/deployment-utils.ts +++ b/apps/sim/lib/workflows/operations/deployment-utils.ts @@ -98,7 +98,7 @@ export function getInputFormatExample( case 'array': exampleData[field.name] = [1, 2, 3] break - case 'files': + case 'file[]': exampleData[field.name] = [ { data: 'data:application/pdf;base64,...', diff --git a/apps/sim/lib/workflows/types.ts b/apps/sim/lib/workflows/types.ts index 4596ce9e9..9e51d7ff1 100644 --- a/apps/sim/lib/workflows/types.ts +++ b/apps/sim/lib/workflows/types.ts @@ -1,6 +1,6 @@ export interface InputFormatField { name?: string - type?: 'string' | 'number' | 'boolean' | 'object' | 'array' | 'files' | string + type?: 'string' | 'number' | 'boolean' | 'object' | 'array' | 'file[]' | string description?: string value?: unknown } diff --git a/apps/sim/tools/discord/send_message.ts b/apps/sim/tools/discord/send_message.ts index 074285e48..abe95515d 100644 --- a/apps/sim/tools/discord/send_message.ts +++ b/apps/sim/tools/discord/send_message.ts @@ -72,6 +72,7 @@ export const discordSendMessageTool: ToolConfig< outputs: { message: { type: 'string', description: 'Success or error message' }, + files: { type: 'file[]', description: 'Files attached to the message' }, data: { type: 'object', description: 'Discord message data', diff --git a/apps/sim/tools/discord/types.ts b/apps/sim/tools/discord/types.ts index cad8f68c7..76a5d016e 100644 --- a/apps/sim/tools/discord/types.ts +++ b/apps/sim/tools/discord/types.ts @@ -1,3 +1,6 @@ +import type { UserFile } from '@/executor/types' +import type { ToolFileData } from '@/tools/types' + export interface DiscordMessage { id: string content: string @@ -58,7 +61,7 @@ export interface DiscordSendMessageParams extends DiscordAuthParams { description?: string color?: string | number } - files?: any[] + files?: UserFile[] } export interface DiscordGetMessagesParams extends DiscordAuthParams { @@ -83,6 +86,7 @@ interface BaseDiscordResponse { export interface DiscordSendMessageResponse extends BaseDiscordResponse { output: { message: string + files?: ToolFileData[] data?: DiscordMessage } } diff --git a/apps/sim/tools/dropbox/download.ts b/apps/sim/tools/dropbox/download.ts index e489b3d21..8adf286e6 100644 --- a/apps/sim/tools/dropbox/download.ts +++ b/apps/sim/tools/dropbox/download.ts @@ -1,10 +1,20 @@ import type { DropboxDownloadParams, DropboxDownloadResponse } from '@/tools/dropbox/types' import type { ToolConfig } from '@/tools/types' +/** + * Escapes non-ASCII characters in JSON string for HTTP header safety. + * Dropbox API requires characters 0x7F and all non-ASCII to be escaped as \uXXXX. + */ +function httpHeaderSafeJson(value: object): string { + return JSON.stringify(value).replace(/[\u007f-\uffff]/g, (c) => { + return '\\u' + ('0000' + c.charCodeAt(0).toString(16)).slice(-4) + }) +} + export const dropboxDownloadTool: ToolConfig = { id: 'dropbox_download', name: 'Dropbox Download File', - description: 'Download a file from Dropbox and get a temporary link', + description: 'Download a file from Dropbox with metadata and content', version: '1.0.0', oauth: { @@ -22,7 +32,7 @@ export const dropboxDownloadTool: ToolConfig { if (!params.accessToken) { @@ -30,45 +40,74 @@ export const dropboxDownloadTool: ToolConfig ({ - path: params.path, - }), }, - transformResponse: async (response) => { - const data = await response.json() - + transformResponse: async (response, params) => { if (!response.ok) { + const errorText = await response.text() return { success: false, - error: data.error_summary || data.error?.message || 'Failed to download file', + error: errorText || 'Failed to download file', output: {}, } } + const apiResultHeader = + response.headers.get('dropbox-api-result') || response.headers.get('Dropbox-API-Result') + const metadata = apiResultHeader ? JSON.parse(apiResultHeader) : undefined + const contentType = response.headers.get('content-type') || 'application/octet-stream' + const arrayBuffer = await response.arrayBuffer() + const buffer = Buffer.from(arrayBuffer) + const resolvedName = metadata?.name || params?.path?.split('/').pop() || 'download' + + let temporaryLink: string | undefined + if (params?.accessToken) { + try { + const linkResponse = await fetch('https://api.dropboxapi.com/2/files/get_temporary_link', { + method: 'POST', + headers: { + Authorization: `Bearer ${params.accessToken}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ path: params.path }), + }) + if (linkResponse.ok) { + const linkData = await linkResponse.json() + temporaryLink = linkData.link + } + } catch { + temporaryLink = undefined + } + } + return { success: true, output: { - file: data.metadata, - content: '', // Content will be available via the temporary link - temporaryLink: data.link, + file: { + name: resolvedName, + mimeType: contentType, + data: buffer.toString('base64'), + size: buffer.length, + }, + content: buffer.toString('base64'), + metadata, + temporaryLink, }, } }, outputs: { file: { - type: 'object', + type: 'file', + description: 'Downloaded file stored in execution files', + }, + metadata: { + type: 'json', description: 'The file metadata', - properties: { - id: { type: 'string', description: 'Unique identifier for the file' }, - name: { type: 'string', description: 'Name of the file' }, - path_display: { type: 'string', description: 'Display path of the file' }, - size: { type: 'number', description: 'Size of the file in bytes' }, - }, }, temporaryLink: { type: 'string', diff --git a/apps/sim/tools/dropbox/types.ts b/apps/sim/tools/dropbox/types.ts index b48f30cc8..d722e2810 100644 --- a/apps/sim/tools/dropbox/types.ts +++ b/apps/sim/tools/dropbox/types.ts @@ -1,4 +1,5 @@ -import type { ToolResponse } from '@/tools/types' +import type { UserFileLike } from '@/lib/core/utils/user-file' +import type { ToolFileData, ToolResponse } from '@/tools/types' // ===== Core Types ===== @@ -70,7 +71,9 @@ export interface DropboxBaseParams { export interface DropboxUploadParams extends DropboxBaseParams { path: string - fileContent: string // Base64 encoded file content + file?: UserFileLike + // Legacy field for backwards compatibility + fileContent?: string fileName?: string mode?: 'add' | 'overwrite' autorename?: boolean @@ -91,8 +94,9 @@ export interface DropboxDownloadParams extends DropboxBaseParams { export interface DropboxDownloadResponse extends ToolResponse { output: { - file?: DropboxFileMetadata + file?: ToolFileData content?: string // Base64 encoded file content + metadata?: DropboxFileMetadata temporaryLink?: string } } diff --git a/apps/sim/tools/dropbox/upload.ts b/apps/sim/tools/dropbox/upload.ts index 557322763..1a8914bc4 100644 --- a/apps/sim/tools/dropbox/upload.ts +++ b/apps/sim/tools/dropbox/upload.ts @@ -20,11 +20,18 @@ export const dropboxUploadTool: ToolConfig