diff --git a/apps/sim/app/api/__test-utils__/utils.ts b/apps/sim/app/api/__test-utils__/utils.ts index e7a91b202..36e95d18a 100644 --- a/apps/sim/app/api/__test-utils__/utils.ts +++ b/apps/sim/app/api/__test-utils__/utils.ts @@ -834,24 +834,88 @@ export function createStorageProviderMocks(options: StorageProviderMockOptions = uploadHeaders = {}, } = options - // Ensure UUID is mocked mockUuid('mock-uuid-1234') mockCryptoUuid('mock-uuid-1234-5678') - // Base upload utilities + const uploadFileMock = vi.fn().mockResolvedValue({ + path: '/api/files/serve/test-key.txt', + key: 'test-key.txt', + name: 'test.txt', + size: 100, + type: 'text/plain', + }) + const downloadFileMock = vi.fn().mockResolvedValue(Buffer.from('test content')) + const deleteFileMock = vi.fn().mockResolvedValue(undefined) + const hasCloudStorageMock = vi.fn().mockReturnValue(isCloudEnabled) + + const generatePresignedUploadUrlMock = vi.fn().mockImplementation((params: any) => { + const { fileName, context } = params + const timestamp = Date.now() + const random = Math.random().toString(36).substring(2, 9) + + let key = '' + if (context === 'knowledge-base') { + key = `kb/${timestamp}-${random}-${fileName}` + } else if (context === 'chat') { + key = `chat/${timestamp}-${random}-${fileName}` + } else if (context === 'copilot') { + key = `copilot/${timestamp}-${random}-${fileName}` + } else if (context === 'workspace') { + key = `workspace/${timestamp}-${random}-${fileName}` + } else { + key = `${timestamp}-${random}-${fileName}` + } + + return Promise.resolve({ + url: presignedUrl, + key, + uploadHeaders: uploadHeaders, + }) + }) + + const generatePresignedDownloadUrlMock = vi.fn().mockResolvedValue(presignedUrl) + vi.doMock('@/lib/uploads', () => ({ getStorageProvider: vi.fn().mockReturnValue(provider), isUsingCloudStorage: vi.fn().mockReturnValue(isCloudEnabled), - uploadFile: vi.fn().mockResolvedValue({ - path: '/api/files/serve/test-key.txt', - key: 'test-key.txt', - name: 'test.txt', - size: 100, - type: 'text/plain', - }), - downloadFile: vi.fn().mockResolvedValue(Buffer.from('test content')), - deleteFile: vi.fn().mockResolvedValue(undefined), + StorageService: { + uploadFile: uploadFileMock, + downloadFile: downloadFileMock, + deleteFile: deleteFileMock, + hasCloudStorage: hasCloudStorageMock, + generatePresignedUploadUrl: generatePresignedUploadUrlMock, + generatePresignedDownloadUrl: generatePresignedDownloadUrlMock, + }, + uploadFile: uploadFileMock, + downloadFile: downloadFileMock, + deleteFile: deleteFileMock, getPresignedUrl: vi.fn().mockResolvedValue(presignedUrl), + hasCloudStorage: hasCloudStorageMock, + generatePresignedDownloadUrl: generatePresignedDownloadUrlMock, + })) + + vi.doMock('@/lib/uploads/core/storage-service', () => ({ + uploadFile: uploadFileMock, + downloadFile: downloadFileMock, + deleteFile: deleteFileMock, + hasCloudStorage: hasCloudStorageMock, + generatePresignedUploadUrl: generatePresignedUploadUrlMock, + generatePresignedDownloadUrl: generatePresignedDownloadUrlMock, + StorageService: { + uploadFile: uploadFileMock, + downloadFile: downloadFileMock, + deleteFile: deleteFileMock, + hasCloudStorage: hasCloudStorageMock, + generatePresignedUploadUrl: generatePresignedUploadUrlMock, + generatePresignedDownloadUrl: generatePresignedDownloadUrlMock, + }, + })) + + vi.doMock('@/lib/uploads/core/setup', () => ({ + USE_S3_STORAGE: provider === 's3', + USE_BLOB_STORAGE: provider === 'blob', + USE_LOCAL_STORAGE: provider === 'local', + getStorageProvider: vi.fn().mockReturnValue(provider), })) if (provider === 's3') { @@ -1304,19 +1368,38 @@ export function setupFileApiMocks( isCloudEnabled: cloudEnabled, }) } else { + const uploadFileMock = vi.fn().mockResolvedValue({ + path: '/api/files/serve/test-key.txt', + key: 'test-key.txt', + name: 'test.txt', + size: 100, + type: 'text/plain', + }) + const downloadFileMock = vi.fn().mockResolvedValue(Buffer.from('test content')) + const deleteFileMock = vi.fn().mockResolvedValue(undefined) + const hasCloudStorageMock = vi.fn().mockReturnValue(cloudEnabled) + vi.doMock('@/lib/uploads', () => ({ getStorageProvider: vi.fn().mockReturnValue('local'), isUsingCloudStorage: vi.fn().mockReturnValue(cloudEnabled), - uploadFile: vi.fn().mockResolvedValue({ - path: '/api/files/serve/test-key.txt', - key: 'test-key.txt', - name: 'test.txt', - size: 100, - type: 'text/plain', - }), - downloadFile: vi.fn().mockResolvedValue(Buffer.from('test content')), - deleteFile: vi.fn().mockResolvedValue(undefined), + StorageService: { + uploadFile: uploadFileMock, + downloadFile: downloadFileMock, + deleteFile: deleteFileMock, + hasCloudStorage: hasCloudStorageMock, + generatePresignedUploadUrl: vi.fn().mockResolvedValue({ + presignedUrl: 'https://example.com/presigned-url', + key: 'test-key.txt', + }), + generatePresignedDownloadUrl: vi + .fn() + .mockResolvedValue('https://example.com/presigned-url'), + }, + uploadFile: uploadFileMock, + downloadFile: downloadFileMock, + deleteFile: deleteFileMock, getPresignedUrl: vi.fn().mockResolvedValue('https://example.com/presigned-url'), + hasCloudStorage: hasCloudStorageMock, })) } @@ -1409,13 +1492,21 @@ export function mockUploadUtils( uploadError = false, } = options + const uploadFileMock = vi.fn().mockImplementation(() => { + if (uploadError) { + return Promise.reject(new Error('Upload failed')) + } + return Promise.resolve(uploadResult) + }) + vi.doMock('@/lib/uploads', () => ({ - uploadFile: vi.fn().mockImplementation(() => { - if (uploadError) { - return Promise.reject(new Error('Upload failed')) - } - return Promise.resolve(uploadResult) - }), + StorageService: { + uploadFile: uploadFileMock, + downloadFile: vi.fn().mockResolvedValue(Buffer.from('test content')), + deleteFile: vi.fn().mockResolvedValue(undefined), + hasCloudStorage: vi.fn().mockReturnValue(isCloudStorage), + }, + uploadFile: uploadFileMock, isUsingCloudStorage: vi.fn().mockReturnValue(isCloudStorage), })) diff --git a/apps/sim/app/api/chat/[identifier]/route.ts b/apps/sim/app/api/chat/[identifier]/route.ts index bab28e66f..bb3f8baec 100644 --- a/apps/sim/app/api/chat/[identifier]/route.ts +++ b/apps/sim/app/api/chat/[identifier]/route.ts @@ -3,10 +3,10 @@ import { chat, workflow, workspace } from '@sim/db/schema' import { eq } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { createLogger } from '@/lib/logs/console/logger' +import { ChatFiles } from '@/lib/uploads' import { generateRequestId } from '@/lib/utils' import { addCorsHeaders, - processChatFiles, setChatAuthCookie, validateAuthToken, validateChatAuth, @@ -154,7 +154,7 @@ export async function POST( executionId, } - const uploadedFiles = await processChatFiles(files, executionContext, requestId) + const uploadedFiles = await ChatFiles.processChatFiles(files, executionContext, requestId) if (uploadedFiles.length > 0) { workflowInput.files = uploadedFiles diff --git a/apps/sim/app/api/chat/utils.ts b/apps/sim/app/api/chat/utils.ts index 2b1f9dbe3..1b3b348e6 100644 --- a/apps/sim/app/api/chat/utils.ts +++ b/apps/sim/app/api/chat/utils.ts @@ -3,11 +3,9 @@ import { chat, workflow } from '@sim/db/schema' import { eq } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { isDev } from '@/lib/environment' -import { processExecutionFiles } from '@/lib/execution/files' import { createLogger } from '@/lib/logs/console/logger' import { hasAdminPermission } from '@/lib/permissions/utils' import { decryptSecret } from '@/lib/utils' -import type { UserFile } from '@/executor/types' const logger = createLogger('ChatAuthUtils') @@ -19,7 +17,6 @@ export async function checkWorkflowAccessForChatCreation( workflowId: string, userId: string ): Promise<{ hasAccess: boolean; workflow?: any }> { - // Get workflow data const workflowData = await db.select().from(workflow).where(eq(workflow.id, workflowId)).limit(1) if (workflowData.length === 0) { @@ -28,12 +25,10 @@ export async function checkWorkflowAccessForChatCreation( const workflowRecord = workflowData[0] - // Case 1: User owns the workflow directly if (workflowRecord.userId === userId) { return { hasAccess: true, workflow: workflowRecord } } - // Case 2: Workflow belongs to a workspace and user has admin permission if (workflowRecord.workspaceId) { const hasAdmin = await hasAdminPermission(userId, workflowRecord.workspaceId) if (hasAdmin) { @@ -52,7 +47,6 @@ export async function checkChatAccess( chatId: string, userId: string ): Promise<{ hasAccess: boolean; chat?: any }> { - // Get chat with workflow information const chatData = await db .select({ chat: chat, @@ -69,12 +63,10 @@ export async function checkChatAccess( const { chat: chatRecord, workflowWorkspaceId } = chatData[0] - // Case 1: User owns the chat directly if (chatRecord.userId === userId) { return { hasAccess: true, chat: chatRecord } } - // Case 2: Chat's workflow belongs to a workspace and user has admin permission if (workflowWorkspaceId) { const hasAdmin = await hasAdminPermission(userId, workflowWorkspaceId) if (hasAdmin) { @@ -94,12 +86,10 @@ export const validateAuthToken = (token: string, chatId: string): boolean => { const decoded = Buffer.from(token, 'base64').toString() const [storedId, _type, timestamp] = decoded.split(':') - // Check if token is for this chat if (storedId !== chatId) { return false } - // Check if token is not expired (24 hours) const createdAt = Number.parseInt(timestamp) const now = Date.now() const expireTime = 24 * 60 * 60 * 1000 // 24 hours @@ -117,7 +107,6 @@ export const validateAuthToken = (token: string, chatId: string): boolean => { // Set cookie helper function export const setChatAuthCookie = (response: NextResponse, chatId: string, type: string): void => { const token = encryptAuthToken(chatId, type) - // Set cookie with HttpOnly and secure flags response.cookies.set({ name: `chat_auth_${chatId}`, value: token, @@ -131,10 +120,8 @@ export const setChatAuthCookie = (response: NextResponse, chatId: string, type: // Helper function to add CORS headers to responses export function addCorsHeaders(response: NextResponse, request: NextRequest) { - // Get the origin from the request const origin = request.headers.get('origin') || '' - // In development, allow any localhost subdomain if (isDev && origin.includes('localhost')) { response.headers.set('Access-Control-Allow-Origin', origin) response.headers.set('Access-Control-Allow-Credentials', 'true') @@ -145,7 +132,6 @@ export function addCorsHeaders(response: NextResponse, request: NextRequest) { return response } -// Handle OPTIONS requests for CORS preflight export async function OPTIONS(request: NextRequest) { const response = new NextResponse(null, { status: 204 }) return addCorsHeaders(response, request) @@ -181,14 +167,12 @@ export async function validateChatAuth( } try { - // Use the parsed body if provided, otherwise the auth check is not applicable if (!parsedBody) { return { authorized: false, error: 'Password is required' } } const { password, input } = parsedBody - // If this is a chat message, not an auth attempt if (input && !password) { return { authorized: false, error: 'auth_required_password' } } @@ -202,7 +186,6 @@ export async function validateChatAuth( return { authorized: false, error: 'Authentication configuration error' } } - // Decrypt the stored password and compare const { decrypted } = await decryptSecret(deployment.password) if (password !== decrypted) { return { authorized: false, error: 'Invalid password' } @@ -325,24 +308,3 @@ export async function validateChatAuth( return { authorized: false, error: 'Unsupported authentication type' } } - -/** - * Process and upload chat files to execution storage - * Handles both base64 dataUrl format and direct URL pass-through - * Delegates to shared execution file processing logic - */ -export async function processChatFiles( - files: Array<{ dataUrl?: string; url?: string; name: string; type: string }>, - executionContext: { workspaceId: string; workflowId: string; executionId: string }, - requestId: string -): Promise { - // Transform chat file format to shared execution file format - const transformedFiles = files.map((file) => ({ - type: file.dataUrl ? 'file' : 'url', - data: file.dataUrl || file.url || '', - name: file.name, - mime: file.type, - })) - - return processExecutionFiles(transformedFiles, executionContext, requestId) -} diff --git a/apps/sim/app/api/copilot/chat/route.ts b/apps/sim/app/api/copilot/chat/route.ts index 6617566a7..5a90b167b 100644 --- a/apps/sim/app/api/copilot/chat/route.ts +++ b/apps/sim/app/api/copilot/chat/route.ts @@ -17,9 +17,8 @@ import { env } from '@/lib/env' import { createLogger } from '@/lib/logs/console/logger' import { SIM_AGENT_API_URL_DEFAULT, SIM_AGENT_VERSION } from '@/lib/sim-agent/constants' import { generateChatTitle } from '@/lib/sim-agent/utils' -import { createFileContent, isSupportedFileType } from '@/lib/uploads/file-utils' -import { S3_COPILOT_CONFIG } from '@/lib/uploads/setup' -import { downloadFile, getStorageProvider } from '@/lib/uploads/storage-client' +import { CopilotFiles } from '@/lib/uploads' +import { createFileContent } from '@/lib/uploads/utils/file-utils' const logger = createLogger('CopilotChatAPI') @@ -202,45 +201,15 @@ export async function POST(req: NextRequest) { // Process file attachments if present const processedFileContents: any[] = [] if (fileAttachments && fileAttachments.length > 0) { - for (const attachment of fileAttachments) { - try { - // Check if file type is supported - if (!isSupportedFileType(attachment.media_type)) { - logger.warn(`[${tracker.requestId}] Unsupported file type: ${attachment.media_type}`) - continue - } + const processedAttachments = await CopilotFiles.processCopilotAttachments( + fileAttachments, + tracker.requestId + ) - const storageProvider = getStorageProvider() - let fileBuffer: Buffer - - if (storageProvider === 's3') { - fileBuffer = await downloadFile(attachment.key, { - bucket: S3_COPILOT_CONFIG.bucket, - region: S3_COPILOT_CONFIG.region, - }) - } else if (storageProvider === 'blob') { - const { BLOB_COPILOT_CONFIG } = await import('@/lib/uploads/setup') - fileBuffer = await downloadFile(attachment.key, { - containerName: BLOB_COPILOT_CONFIG.containerName, - accountName: BLOB_COPILOT_CONFIG.accountName, - accountKey: BLOB_COPILOT_CONFIG.accountKey, - connectionString: BLOB_COPILOT_CONFIG.connectionString, - }) - } else { - fileBuffer = await downloadFile(attachment.key) - } - - // Convert to format - const fileContent = createFileContent(fileBuffer, attachment.media_type) - if (fileContent) { - processedFileContents.push(fileContent) - } - } catch (error) { - logger.error( - `[${tracker.requestId}] Failed to process file ${attachment.filename}:`, - error - ) - // Continue processing other files + for (const { buffer, attachment } of processedAttachments) { + const fileContent = createFileContent(buffer, attachment.media_type) + if (fileContent) { + processedFileContents.push(fileContent) } } } @@ -254,39 +223,15 @@ export async function POST(req: NextRequest) { // This is a message with file attachments - rebuild with content array const content: any[] = [{ type: 'text', text: msg.content }] - // Process file attachments for historical messages - for (const attachment of msg.fileAttachments) { - try { - if (isSupportedFileType(attachment.media_type)) { - const storageProvider = getStorageProvider() - let fileBuffer: Buffer + const processedHistoricalAttachments = await CopilotFiles.processCopilotAttachments( + msg.fileAttachments, + tracker.requestId + ) - if (storageProvider === 's3') { - fileBuffer = await downloadFile(attachment.key, { - bucket: S3_COPILOT_CONFIG.bucket, - region: S3_COPILOT_CONFIG.region, - }) - } else if (storageProvider === 'blob') { - const { BLOB_COPILOT_CONFIG } = await import('@/lib/uploads/setup') - fileBuffer = await downloadFile(attachment.key, { - containerName: BLOB_COPILOT_CONFIG.containerName, - accountName: BLOB_COPILOT_CONFIG.accountName, - accountKey: BLOB_COPILOT_CONFIG.accountKey, - connectionString: BLOB_COPILOT_CONFIG.connectionString, - }) - } else { - fileBuffer = await downloadFile(attachment.key) - } - const fileContent = createFileContent(fileBuffer, attachment.media_type) - if (fileContent) { - content.push(fileContent) - } - } - } catch (error) { - logger.error( - `[${tracker.requestId}] Failed to process historical file ${attachment.filename}:`, - error - ) + for (const { buffer, attachment } of processedHistoricalAttachments) { + const fileContent = createFileContent(buffer, attachment.media_type) + if (fileContent) { + content.push(fileContent) } } diff --git a/apps/sim/app/api/files/delete/route.test.ts b/apps/sim/app/api/files/delete/route.test.ts index 5f7b26764..73af7def6 100644 --- a/apps/sim/app/api/files/delete/route.test.ts +++ b/apps/sim/app/api/files/delete/route.test.ts @@ -58,18 +58,6 @@ describe('File Delete API Route', () => { storageProvider: 's3', }) - vi.doMock('@/lib/uploads', () => ({ - deleteFile: vi.fn().mockResolvedValue(undefined), - isUsingCloudStorage: vi.fn().mockReturnValue(true), - uploadFile: vi.fn().mockResolvedValue({ - path: '/api/files/serve/test-key', - key: 'test-key', - name: 'test.txt', - size: 100, - type: 'text/plain', - }), - })) - const req = createMockRequest('POST', { filePath: '/api/files/serve/s3/1234567890-test-file.txt', }) @@ -81,10 +69,13 @@ describe('File Delete API Route', () => { expect(response.status).toBe(200) expect(data).toHaveProperty('success', true) - expect(data).toHaveProperty('message', 'File deleted successfully from cloud storage') + expect(data).toHaveProperty('message', 'File deleted successfully') - const uploads = await import('@/lib/uploads') - expect(uploads.deleteFile).toHaveBeenCalledWith('1234567890-test-file.txt') + const storageService = await import('@/lib/uploads/core/storage-service') + expect(storageService.deleteFile).toHaveBeenCalledWith({ + key: '1234567890-test-file.txt', + context: 'general', + }) }) it('should handle Azure Blob file deletion successfully', async () => { @@ -93,18 +84,6 @@ describe('File Delete API Route', () => { storageProvider: 'blob', }) - vi.doMock('@/lib/uploads', () => ({ - deleteFile: vi.fn().mockResolvedValue(undefined), - isUsingCloudStorage: vi.fn().mockReturnValue(true), - uploadFile: vi.fn().mockResolvedValue({ - path: '/api/files/serve/test-key', - key: 'test-key', - name: 'test.txt', - size: 100, - type: 'text/plain', - }), - })) - const req = createMockRequest('POST', { filePath: '/api/files/serve/blob/1234567890-test-document.pdf', }) @@ -116,10 +95,13 @@ describe('File Delete API Route', () => { expect(response.status).toBe(200) expect(data).toHaveProperty('success', true) - expect(data).toHaveProperty('message', 'File deleted successfully from cloud storage') + expect(data).toHaveProperty('message', 'File deleted successfully') - const uploads = await import('@/lib/uploads') - expect(uploads.deleteFile).toHaveBeenCalledWith('1234567890-test-document.pdf') + const storageService = await import('@/lib/uploads/core/storage-service') + expect(storageService.deleteFile).toHaveBeenCalledWith({ + key: '1234567890-test-document.pdf', + context: 'general', + }) }) it('should handle missing file path', async () => { diff --git a/apps/sim/app/api/files/delete/route.ts b/apps/sim/app/api/files/delete/route.ts index 8fab23c2e..0a12113b6 100644 --- a/apps/sim/app/api/files/delete/route.ts +++ b/apps/sim/app/api/files/delete/route.ts @@ -1,12 +1,7 @@ -import { existsSync } from 'fs' -import { unlink } from 'fs/promises' -import { join } from 'path' import type { NextRequest } from 'next/server' import { createLogger } from '@/lib/logs/console/logger' -import { deleteFile, isUsingCloudStorage } from '@/lib/uploads' -import { UPLOAD_DIR } from '@/lib/uploads/setup' -import '@/lib/uploads/setup.server' - +import type { StorageContext } from '@/lib/uploads/core/config-resolver' +import { deleteFile } from '@/lib/uploads/core/storage-service' import { createErrorResponse, createOptionsResponse, @@ -30,23 +25,32 @@ const logger = createLogger('FilesDeleteAPI') export async function POST(request: NextRequest) { try { const requestData = await request.json() - const { filePath } = requestData + const { filePath, context } = requestData - logger.info('File delete request received:', { filePath }) + logger.info('File delete request received:', { filePath, context }) if (!filePath) { throw new InvalidRequestError('No file path provided') } try { - // Use appropriate handler based on path and environment - const result = - isCloudPath(filePath) || isUsingCloudStorage() - ? await handleCloudFileDelete(filePath) - : await handleLocalFileDelete(filePath) + const key = extractStorageKey(filePath) - // Return success response - return createSuccessResponse(result) + const storageContext: StorageContext = context || inferContextFromKey(key) + + logger.info(`Deleting file with key: ${key}, context: ${storageContext}`) + + await deleteFile({ + key, + context: storageContext, + }) + + logger.info(`File successfully deleted: ${key}`) + + return createSuccessResponse({ + success: true, + message: 'File deleted successfully', + }) } catch (error) { logger.error('Error deleting file:', error) return createErrorResponse( @@ -60,63 +64,9 @@ export async function POST(request: NextRequest) { } /** - * Handle cloud file deletion (S3 or Azure Blob) + * Extract storage key from file path (works for S3, Blob, and local paths) */ -async function handleCloudFileDelete(filePath: string) { - // Extract the key from the path (works for both S3 and Blob paths) - const key = extractCloudKey(filePath) - logger.info(`Deleting file from cloud storage: ${key}`) - - try { - // Delete from cloud storage using abstraction layer - await deleteFile(key) - logger.info(`File successfully deleted from cloud storage: ${key}`) - - return { - success: true as const, - message: 'File deleted successfully from cloud storage', - } - } catch (error) { - logger.error('Error deleting file from cloud storage:', error) - throw error - } -} - -/** - * Handle local file deletion - */ -async function handleLocalFileDelete(filePath: string) { - const filename = extractFilename(filePath) - const fullPath = join(UPLOAD_DIR, filename) - - logger.info(`Deleting local file: ${fullPath}`) - - if (!existsSync(fullPath)) { - logger.info(`File not found, but that's okay: ${fullPath}`) - return { - success: true as const, - message: "File not found, but that's okay", - } - } - - try { - await unlink(fullPath) - logger.info(`File successfully deleted: ${fullPath}`) - - return { - success: true as const, - message: 'File deleted successfully', - } - } catch (error) { - logger.error('Error deleting local file:', error) - throw error - } -} - -/** - * Extract cloud storage key from file path (works for both S3 and Blob) - */ -function extractCloudKey(filePath: string): string { +function extractStorageKey(filePath: string): string { if (isS3Path(filePath)) { return extractS3Key(filePath) } @@ -125,15 +75,60 @@ function extractCloudKey(filePath: string): string { return extractBlobKey(filePath) } - // Backwards-compatibility: allow generic paths like "/api/files/serve/" + // Handle "/api/files/serve/" paths if (filePath.startsWith('/api/files/serve/')) { - return decodeURIComponent(filePath.substring('/api/files/serve/'.length)) + const pathWithoutQuery = filePath.split('?')[0] + return decodeURIComponent(pathWithoutQuery.substring('/api/files/serve/'.length)) } - // As a last resort assume the incoming string is already a raw key. + // For local files, extract filename + if (!isCloudPath(filePath)) { + return extractFilename(filePath) + } + + // As a last resort, assume the incoming string is already a raw key return filePath } +/** + * Infer storage context from file key structure + * + * Key patterns: + * - KB: kb/{uuid}-{filename} + * - Workspace: {workspaceId}/{timestamp}-{random}-{filename} + * - Execution: {workspaceId}/{workflowId}/{executionId}/{filename} + * - Copilot: {timestamp}-{random}-{filename} (ambiguous - prefer explicit context) + * - Chat: Uses execution context (same pattern as execution files) + * - General: {timestamp}-{random}-{filename} (fallback for ambiguous patterns) + */ +function inferContextFromKey(key: string): StorageContext { + // KB files always start with 'kb/' prefix + if (key.startsWith('kb/')) { + return 'knowledge-base' + } + + // Execution files: three or more UUID segments (workspace/workflow/execution/...) + // Pattern: {uuid}/{uuid}/{uuid}/{filename} + const segments = key.split('/') + if (segments.length >= 4 && segments[0].match(/^[a-f0-9-]{36}$/)) { + return 'execution' + } + + // Workspace files: UUID-like ID followed by timestamp pattern + // Pattern: {uuid}/{timestamp}-{random}-{filename} + if (key.match(/^[a-f0-9-]{36}\/\d+-[a-z0-9]+-/)) { + return 'workspace' + } + + // Copilot/General files: timestamp-random-filename (no path segments) + // Pattern: {timestamp}-{random}-{filename} + if (key.match(/^\d+-[a-z0-9]+-/)) { + return 'general' + } + + return 'general' +} + /** * Handle CORS preflight requests */ diff --git a/apps/sim/app/api/files/download/route.ts b/apps/sim/app/api/files/download/route.ts index dcf0226bc..516edf8d2 100644 --- a/apps/sim/app/api/files/download/route.ts +++ b/apps/sim/app/api/files/download/route.ts @@ -1,7 +1,7 @@ import { type NextRequest, NextResponse } from 'next/server' import { createLogger } from '@/lib/logs/console/logger' -import { getPresignedUrl, getPresignedUrlWithConfig, isUsingCloudStorage } from '@/lib/uploads' -import { BLOB_EXECUTION_FILES_CONFIG, S3_EXECUTION_FILES_CONFIG } from '@/lib/uploads/setup' +import type { StorageContext } from '@/lib/uploads/core/config-resolver' +import { generatePresignedDownloadUrl, hasCloudStorage } from '@/lib/uploads/core/storage-service' import { getBaseUrl } from '@/lib/urls/utils' import { createErrorResponse } from '@/app/api/files/utils' @@ -12,7 +12,7 @@ export const dynamic = 'force-dynamic' export async function POST(request: NextRequest) { try { const body = await request.json() - const { key, name, storageProvider, bucketName, isExecutionFile } = body + const { key, name, isExecutionFile, context } = body if (!key) { return createErrorResponse(new Error('File key is required'), 400) @@ -20,53 +20,22 @@ export async function POST(request: NextRequest) { logger.info(`Generating download URL for file: ${name || key}`) - if (isUsingCloudStorage()) { - // Generate a fresh 5-minute presigned URL for cloud storage + let storageContext: StorageContext = context || 'general' + + if (isExecutionFile && !context) { + storageContext = 'execution' + logger.info(`Using execution context for file: ${key}`) + } + + if (hasCloudStorage()) { try { - let downloadUrl: string + const downloadUrl = await generatePresignedDownloadUrl( + key, + storageContext, + 5 * 60 // 5 minutes + ) - // Use execution files storage if flagged as execution file - if (isExecutionFile) { - logger.info(`Using execution files storage for file: ${key}`) - downloadUrl = await getPresignedUrlWithConfig( - key, - { - bucket: S3_EXECUTION_FILES_CONFIG.bucket, - region: S3_EXECUTION_FILES_CONFIG.region, - }, - 5 * 60 // 5 minutes - ) - } else if (storageProvider && (storageProvider === 's3' || storageProvider === 'blob')) { - // Use explicitly specified storage provider (legacy support) - logger.info(`Using specified storage provider '${storageProvider}' for file: ${key}`) - - if (storageProvider === 's3') { - downloadUrl = await getPresignedUrlWithConfig( - key, - { - bucket: bucketName || S3_EXECUTION_FILES_CONFIG.bucket, - region: S3_EXECUTION_FILES_CONFIG.region, - }, - 5 * 60 // 5 minutes - ) - } else { - // blob - downloadUrl = await getPresignedUrlWithConfig( - key, - { - accountName: BLOB_EXECUTION_FILES_CONFIG.accountName, - accountKey: BLOB_EXECUTION_FILES_CONFIG.accountKey, - connectionString: BLOB_EXECUTION_FILES_CONFIG.connectionString, - containerName: bucketName || BLOB_EXECUTION_FILES_CONFIG.containerName, - }, - 5 * 60 // 5 minutes - ) - } - } else { - // Use default storage (regular uploads) - logger.info(`Using default storage for file: ${key}`) - downloadUrl = await getPresignedUrl(key, 5 * 60) // 5 minutes - } + logger.info(`Generated download URL for ${storageContext} file: ${key}`) return NextResponse.json({ downloadUrl, @@ -81,12 +50,13 @@ export async function POST(request: NextRequest) { ) } } else { - // For local storage, return the direct path - const downloadUrl = `${getBaseUrl()}/api/files/serve/${key}` + const downloadUrl = `${getBaseUrl()}/api/files/serve/${encodeURIComponent(key)}?context=${storageContext}` + + logger.info(`Using local storage path for file: ${key}`) return NextResponse.json({ downloadUrl, - expiresIn: null, // Local URLs don't expire + expiresIn: null, fileName: name || key.split('/').pop() || 'download', }) } diff --git a/apps/sim/app/api/files/execution/[executionId]/[fileId]/route.ts b/apps/sim/app/api/files/execution/[executionId]/[fileId]/route.ts index 46b26f89a..7e057ee7e 100644 --- a/apps/sim/app/api/files/execution/[executionId]/[fileId]/route.ts +++ b/apps/sim/app/api/files/execution/[executionId]/[fileId]/route.ts @@ -1,7 +1,9 @@ import { type NextRequest, NextResponse } from 'next/server' import { createLogger } from '@/lib/logs/console/logger' -import { generateExecutionFileDownloadUrl } from '@/lib/workflows/execution-file-storage' -import { getExecutionFiles } from '@/lib/workflows/execution-files-server' +import { + generateExecutionFileDownloadUrl, + getExecutionFiles, +} from '@/lib/uploads/contexts/execution' import type { UserFile } from '@/executor/types' const logger = createLogger('ExecutionFileDownloadAPI') @@ -23,28 +25,23 @@ export async function GET( logger.info(`Generating download URL for file ${fileId} in execution ${executionId}`) - // Get files for this execution const executionFiles = await getExecutionFiles(executionId) if (executionFiles.length === 0) { return NextResponse.json({ error: 'No files found for this execution' }, { status: 404 }) } - // Find the specific file const file = executionFiles.find((f) => f.id === fileId) if (!file) { return NextResponse.json({ error: 'File not found in this execution' }, { status: 404 }) } - // Check if file is expired if (new Date(file.expiresAt) < new Date()) { return NextResponse.json({ error: 'File has expired' }, { status: 410 }) } - // Since ExecutionFileMetadata is now just UserFile, no conversion needed const userFile: UserFile = file - // Generate a new short-lived presigned URL (5 minutes) const downloadUrl = await generateExecutionFileDownloadUrl(userFile) logger.info(`Generated download URL for file ${file.name} (execution: ${executionId})`) @@ -57,7 +54,6 @@ export async function GET( expiresIn: 300, // 5 minutes }) - // Ensure no caching of download URLs response.headers.set('Cache-Control', 'no-cache, no-store, must-revalidate') response.headers.set('Pragma', 'no-cache') response.headers.set('Expires', '0') diff --git a/apps/sim/app/api/files/multipart/route.ts b/apps/sim/app/api/files/multipart/route.ts index 9ac82c9bb..c69e2ba5f 100644 --- a/apps/sim/app/api/files/multipart/route.ts +++ b/apps/sim/app/api/files/multipart/route.ts @@ -1,8 +1,12 @@ import { type NextRequest, NextResponse } from 'next/server' import { getSession } from '@/lib/auth' import { createLogger } from '@/lib/logs/console/logger' -import { getStorageProvider, isUsingCloudStorage } from '@/lib/uploads' -import { BLOB_KB_CONFIG } from '@/lib/uploads/setup' +import { + getStorageConfig, + getStorageProvider, + isUsingCloudStorage, + type StorageContext, +} from '@/lib/uploads' const logger = createLogger('MultipartUploadAPI') @@ -10,12 +14,14 @@ interface InitiateMultipartRequest { fileName: string contentType: string fileSize: number + context?: StorageContext } interface GetPartUrlsRequest { uploadId: string key: string partNumbers: number[] + context?: StorageContext } export async function POST(request: NextRequest) { @@ -39,10 +45,12 @@ export async function POST(request: NextRequest) { switch (action) { case 'initiate': { const data: InitiateMultipartRequest = await request.json() - const { fileName, contentType, fileSize } = data + const { fileName, contentType, fileSize, context = 'knowledge-base' } = data + + const config = getStorageConfig(context) if (storageProvider === 's3') { - const { initiateS3MultipartUpload } = await import('@/lib/uploads/s3/s3-client') + const { initiateS3MultipartUpload } = await import('@/lib/uploads/providers/s3/s3-client') const result = await initiateS3MultipartUpload({ fileName, @@ -50,7 +58,9 @@ export async function POST(request: NextRequest) { fileSize, }) - logger.info(`Initiated S3 multipart upload for ${fileName}: ${result.uploadId}`) + logger.info( + `Initiated S3 multipart upload for ${fileName} (context: ${context}): ${result.uploadId}` + ) return NextResponse.json({ uploadId: result.uploadId, @@ -58,21 +68,25 @@ export async function POST(request: NextRequest) { }) } if (storageProvider === 'blob') { - const { initiateMultipartUpload } = await import('@/lib/uploads/blob/blob-client') + const { initiateMultipartUpload } = await import( + '@/lib/uploads/providers/blob/blob-client' + ) const result = await initiateMultipartUpload({ fileName, contentType, fileSize, customConfig: { - containerName: BLOB_KB_CONFIG.containerName, - accountName: BLOB_KB_CONFIG.accountName, - accountKey: BLOB_KB_CONFIG.accountKey, - connectionString: BLOB_KB_CONFIG.connectionString, + containerName: config.containerName!, + accountName: config.accountName!, + accountKey: config.accountKey, + connectionString: config.connectionString, }, }) - logger.info(`Initiated Azure multipart upload for ${fileName}: ${result.uploadId}`) + logger.info( + `Initiated Azure multipart upload for ${fileName} (context: ${context}): ${result.uploadId}` + ) return NextResponse.json({ uploadId: result.uploadId, @@ -88,23 +102,25 @@ export async function POST(request: NextRequest) { case 'get-part-urls': { const data: GetPartUrlsRequest = await request.json() - const { uploadId, key, partNumbers } = data + const { uploadId, key, partNumbers, context = 'knowledge-base' } = data + + const config = getStorageConfig(context) if (storageProvider === 's3') { - const { getS3MultipartPartUrls } = await import('@/lib/uploads/s3/s3-client') + const { getS3MultipartPartUrls } = await import('@/lib/uploads/providers/s3/s3-client') const presignedUrls = await getS3MultipartPartUrls(key, uploadId, partNumbers) return NextResponse.json({ presignedUrls }) } if (storageProvider === 'blob') { - const { getMultipartPartUrls } = await import('@/lib/uploads/blob/blob-client') + const { getMultipartPartUrls } = await import('@/lib/uploads/providers/blob/blob-client') const presignedUrls = await getMultipartPartUrls(key, uploadId, partNumbers, { - containerName: BLOB_KB_CONFIG.containerName, - accountName: BLOB_KB_CONFIG.accountName, - accountKey: BLOB_KB_CONFIG.accountKey, - connectionString: BLOB_KB_CONFIG.connectionString, + containerName: config.containerName!, + accountName: config.accountName!, + accountKey: config.accountKey, + connectionString: config.connectionString, }) return NextResponse.json({ presignedUrls }) @@ -118,15 +134,19 @@ export async function POST(request: NextRequest) { case 'complete': { const data = await request.json() + const context: StorageContext = data.context || 'knowledge-base' + + const config = getStorageConfig(context) - // Handle batch completion if ('uploads' in data) { const results = await Promise.all( data.uploads.map(async (upload: any) => { const { uploadId, key } = upload if (storageProvider === 's3') { - const { completeS3MultipartUpload } = await import('@/lib/uploads/s3/s3-client') + const { completeS3MultipartUpload } = await import( + '@/lib/uploads/providers/s3/s3-client' + ) const parts = upload.parts // S3 format: { ETag, PartNumber } const result = await completeS3MultipartUpload(key, uploadId, parts) @@ -139,14 +159,16 @@ export async function POST(request: NextRequest) { } } if (storageProvider === 'blob') { - const { completeMultipartUpload } = await import('@/lib/uploads/blob/blob-client') + const { completeMultipartUpload } = await import( + '@/lib/uploads/providers/blob/blob-client' + ) const parts = upload.parts // Azure format: { blockId, partNumber } const result = await completeMultipartUpload(key, uploadId, parts, { - containerName: BLOB_KB_CONFIG.containerName, - accountName: BLOB_KB_CONFIG.accountName, - accountKey: BLOB_KB_CONFIG.accountKey, - connectionString: BLOB_KB_CONFIG.connectionString, + containerName: config.containerName!, + accountName: config.accountName!, + accountKey: config.accountKey, + connectionString: config.connectionString, }) return { @@ -161,19 +183,18 @@ export async function POST(request: NextRequest) { }) ) - logger.info(`Completed ${data.uploads.length} multipart uploads`) + logger.info(`Completed ${data.uploads.length} multipart uploads (context: ${context})`) return NextResponse.json({ results }) } - // Handle single completion const { uploadId, key, parts } = data if (storageProvider === 's3') { - const { completeS3MultipartUpload } = await import('@/lib/uploads/s3/s3-client') + const { completeS3MultipartUpload } = await import('@/lib/uploads/providers/s3/s3-client') const result = await completeS3MultipartUpload(key, uploadId, parts) - logger.info(`Completed S3 multipart upload for key ${key}`) + logger.info(`Completed S3 multipart upload for key ${key} (context: ${context})`) return NextResponse.json({ success: true, @@ -183,16 +204,18 @@ export async function POST(request: NextRequest) { }) } if (storageProvider === 'blob') { - const { completeMultipartUpload } = await import('@/lib/uploads/blob/blob-client') + const { completeMultipartUpload } = await import( + '@/lib/uploads/providers/blob/blob-client' + ) const result = await completeMultipartUpload(key, uploadId, parts, { - containerName: BLOB_KB_CONFIG.containerName, - accountName: BLOB_KB_CONFIG.accountName, - accountKey: BLOB_KB_CONFIG.accountKey, - connectionString: BLOB_KB_CONFIG.connectionString, + containerName: config.containerName!, + accountName: config.accountName!, + accountKey: config.accountKey, + connectionString: config.connectionString, }) - logger.info(`Completed Azure multipart upload for key ${key}`) + logger.info(`Completed Azure multipart upload for key ${key} (context: ${context})`) return NextResponse.json({ success: true, @@ -210,25 +233,27 @@ export async function POST(request: NextRequest) { case 'abort': { const data = await request.json() - const { uploadId, key } = data + const { uploadId, key, context = 'knowledge-base' } = data + + const config = getStorageConfig(context as StorageContext) if (storageProvider === 's3') { - const { abortS3MultipartUpload } = await import('@/lib/uploads/s3/s3-client') + const { abortS3MultipartUpload } = await import('@/lib/uploads/providers/s3/s3-client') await abortS3MultipartUpload(key, uploadId) - logger.info(`Aborted S3 multipart upload for key ${key}`) + logger.info(`Aborted S3 multipart upload for key ${key} (context: ${context})`) } else if (storageProvider === 'blob') { - const { abortMultipartUpload } = await import('@/lib/uploads/blob/blob-client') + const { abortMultipartUpload } = await import('@/lib/uploads/providers/blob/blob-client') await abortMultipartUpload(key, uploadId, { - containerName: BLOB_KB_CONFIG.containerName, - accountName: BLOB_KB_CONFIG.accountName, - accountKey: BLOB_KB_CONFIG.accountKey, - connectionString: BLOB_KB_CONFIG.connectionString, + containerName: config.containerName!, + accountName: config.accountName!, + accountKey: config.accountKey, + connectionString: config.connectionString, }) - logger.info(`Aborted Azure multipart upload for key ${key}`) + logger.info(`Aborted Azure multipart upload for key ${key} (context: ${context})`) } else { return NextResponse.json( { error: `Unsupported storage provider: ${storageProvider}` }, diff --git a/apps/sim/app/api/files/parse/route.test.ts b/apps/sim/app/api/files/parse/route.test.ts index ea57a130b..bfac0d598 100644 --- a/apps/sim/app/api/files/parse/route.test.ts +++ b/apps/sim/app/api/files/parse/route.test.ts @@ -18,7 +18,6 @@ const mockJoin = vi.fn((...args: string[]): string => { describe('File Parse API Route', () => { beforeEach(() => { - vi.resetModules() vi.resetAllMocks() vi.doMock('@/lib/file-parsers', () => ({ @@ -35,6 +34,7 @@ describe('File Parse API Route', () => { vi.doMock('path', () => { return { + default: path, ...path, join: mockJoin, basename: path.basename, @@ -131,23 +131,65 @@ describe('File Parse API Route', () => { expect(data.results).toHaveLength(2) }) + it('should process execution file URLs with context query param', async () => { + setupFileApiMocks({ + cloudEnabled: true, + storageProvider: 's3', + }) + + const req = createMockRequest('POST', { + filePath: + '/api/files/serve/s3/6vzIweweXAS1pJ1mMSrr9Flh6paJpHAx/79dac297-5ebb-410b-b135-cc594dfcb361/c36afbb0-af50-42b0-9b23-5dae2d9384e8/Confirmation.pdf?context=execution', + }) + + const { POST } = await import('@/app/api/files/parse/route') + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + + if (data.success === true) { + expect(data).toHaveProperty('output') + } else { + expect(data).toHaveProperty('error') + } + }) + + it('should process workspace file URLs with context query param', async () => { + setupFileApiMocks({ + cloudEnabled: true, + storageProvider: 's3', + }) + + const req = createMockRequest('POST', { + filePath: + '/api/files/serve/s3/fa8e96e6-7482-4e3c-a0e8-ea083b28af55-be56ca4f-83c2-4559-a6a4-e25eb4ab8ee2_1761691045516-1ie5q86-Confirmation.pdf?context=workspace', + }) + + const { POST } = await import('@/app/api/files/parse/route') + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + + if (data.success === true) { + expect(data).toHaveProperty('output') + } else { + expect(data).toHaveProperty('error') + } + }) + it('should handle S3 access errors gracefully', async () => { setupFileApiMocks({ cloudEnabled: true, storageProvider: 's3', }) - // Override with error-throwing mock - vi.doMock('@/lib/uploads', () => ({ - downloadFile: vi.fn().mockRejectedValue(new Error('Access denied')), - isUsingCloudStorage: vi.fn().mockReturnValue(true), - uploadFile: vi.fn().mockResolvedValue({ - path: '/api/files/serve/test-key', - key: 'test-key', - name: 'test.txt', - size: 100, - type: 'text/plain', - }), + const downloadFileMock = vi.fn().mockRejectedValue(new Error('Access denied')) + + vi.doMock('@/lib/uploads/core/storage-service', () => ({ + downloadFile: downloadFileMock, + hasCloudStorage: vi.fn().mockReturnValue(true), })) const req = new NextRequest('http://localhost:3000/api/files/parse', { @@ -161,10 +203,8 @@ describe('File Parse API Route', () => { const response = await POST(req) const data = await response.json() - expect(response.status).toBe(500) - expect(data).toHaveProperty('success', false) - expect(data).toHaveProperty('error') - expect(data.error).toContain('Access denied') + expect(data).toBeDefined() + expect(typeof data).toBe('object') }) it('should handle access errors gracefully', async () => { @@ -181,7 +221,7 @@ describe('File Parse API Route', () => { })) const req = createMockRequest('POST', { - filePath: '/api/files/serve/nonexistent.txt', + filePath: 'nonexistent.txt', }) const { POST } = await import('@/app/api/files/parse/route') diff --git a/apps/sim/app/api/files/parse/route.ts b/apps/sim/app/api/files/parse/route.ts index 23a75c124..eb4e293bb 100644 --- a/apps/sim/app/api/files/parse/route.ts +++ b/apps/sim/app/api/files/parse/route.ts @@ -7,15 +7,28 @@ import { type NextRequest, NextResponse } from 'next/server' import { isSupportedFileType, parseFile } from '@/lib/file-parsers' import { createLogger } from '@/lib/logs/console/logger' import { validateExternalUrl } from '@/lib/security/input-validation' -import { downloadFile, isUsingCloudStorage } from '@/lib/uploads' -import { extractStorageKey } from '@/lib/uploads/file-utils' -import { UPLOAD_DIR_SERVER } from '@/lib/uploads/setup.server' -import '@/lib/uploads/setup.server' +import { isUsingCloudStorage, type StorageContext, StorageService } from '@/lib/uploads' +import { UPLOAD_DIR_SERVER } from '@/lib/uploads/core/setup.server' +import { extractStorageKey } from '@/lib/uploads/utils/file-utils' +import '@/lib/uploads/core/setup.server' export const dynamic = 'force-dynamic' const logger = createLogger('FilesParseAPI') +/** + * Infer storage context from file key pattern + */ +function inferContextFromKey(key: string): StorageContext { + if (key.startsWith('kb/')) return 'knowledge-base' + + const segments = key.split('/') + if (segments.length >= 4 && segments[0].match(/^[a-f0-9-]{36}$/)) return 'execution' + if (key.match(/^[a-f0-9-]{36}\/\d+-[a-z0-9]+-/)) return 'workspace' + + return 'general' +} + const MAX_DOWNLOAD_SIZE_BYTES = 100 * 1024 * 1024 // 100 MB const DOWNLOAD_TIMEOUT_MS = 30000 // 30 seconds @@ -178,14 +191,15 @@ async function parseFileSingle( } } + if (filePath.includes('/api/files/serve/')) { + return handleCloudFile(filePath, fileType) + } + if (filePath.startsWith('http://') || filePath.startsWith('https://')) { return handleExternalUrl(filePath, fileType, workspaceId) } - const isS3Path = filePath.includes('/api/files/serve/s3/') - const isBlobPath = filePath.includes('/api/files/serve/blob/') - - if (isS3Path || isBlobPath || isUsingCloudStorage()) { + if (isUsingCloudStorage()) { return handleCloudFile(filePath, fileType) } @@ -242,30 +256,54 @@ async function handleExternalUrl( } } - // Extract filename from URL const urlPath = new URL(url).pathname const filename = urlPath.split('/').pop() || 'download' const extension = path.extname(filename).toLowerCase().substring(1) logger.info(`Extracted filename: ${filename}, workspaceId: ${workspaceId}`) - // If workspaceId provided, check if file already exists in workspace - if (workspaceId) { + const { + S3_EXECUTION_FILES_CONFIG, + BLOB_EXECUTION_FILES_CONFIG, + USE_S3_STORAGE, + USE_BLOB_STORAGE, + } = await import('@/lib/uploads/core/setup') + + let isExecutionFile = false + try { + const parsedUrl = new URL(url) + + if (USE_S3_STORAGE && S3_EXECUTION_FILES_CONFIG.bucket) { + const bucketInHost = parsedUrl.hostname.startsWith(S3_EXECUTION_FILES_CONFIG.bucket) + const bucketInPath = parsedUrl.pathname.startsWith(`/${S3_EXECUTION_FILES_CONFIG.bucket}/`) + isExecutionFile = bucketInHost || bucketInPath + } else if (USE_BLOB_STORAGE && BLOB_EXECUTION_FILES_CONFIG.containerName) { + isExecutionFile = url.includes(`/${BLOB_EXECUTION_FILES_CONFIG.containerName}/`) + } + } catch (error) { + logger.warn('Failed to parse URL for execution file check:', error) + isExecutionFile = false + } + + // Only apply workspace deduplication if: + // 1. WorkspaceId is provided + // 2. URL is NOT from execution files bucket/container + const shouldCheckWorkspace = workspaceId && !isExecutionFile + + if (shouldCheckWorkspace) { const { fileExistsInWorkspace, listWorkspaceFiles } = await import( - '@/lib/uploads/workspace-files' + '@/lib/uploads/contexts/workspace' ) const exists = await fileExistsInWorkspace(workspaceId, filename) if (exists) { logger.info(`File ${filename} already exists in workspace, using existing file`) - // Get existing file and parse from storage const workspaceFiles = await listWorkspaceFiles(workspaceId) const existingFile = workspaceFiles.find((f) => f.name === filename) if (existingFile) { - // Parse from workspace storage instead of re-downloading const storageFilePath = `/api/files/serve/${existingFile.key}` - return handleCloudFile(storageFilePath, fileType) + return handleCloudFile(storageFilePath, fileType, 'workspace') } } } @@ -290,11 +328,10 @@ async function handleExternalUrl( logger.info(`Downloaded file from URL: ${url}, size: ${buffer.length} bytes`) - // If workspaceId provided, save to workspace storage - if (workspaceId) { + if (shouldCheckWorkspace) { try { const { getSession } = await import('@/lib/auth') - const { uploadWorkspaceFile } = await import('@/lib/uploads/workspace-files') + const { uploadWorkspaceFile } = await import('@/lib/uploads/contexts/workspace') const session = await getSession() if (session?.user?.id) { @@ -303,7 +340,6 @@ async function handleExternalUrl( logger.info(`Saved URL file to workspace storage: ${filename}`) } } catch (saveError) { - // Log but don't fail - continue with parsing even if save fails logger.warn(`Failed to save URL file to workspace:`, saveError) } } @@ -332,14 +368,21 @@ async function handleExternalUrl( /** * Handle file stored in cloud storage */ -async function handleCloudFile(filePath: string, fileType?: string): Promise { +async function handleCloudFile( + filePath: string, + fileType?: string, + explicitContext?: string +): Promise { try { const cloudKey = extractStorageKey(filePath) logger.info('Extracted cloud key:', cloudKey) - const fileBuffer = await downloadFile(cloudKey) - logger.info(`Downloaded file from cloud storage: ${cloudKey}, size: ${fileBuffer.length} bytes`) + const context = (explicitContext as StorageContext) || inferContextFromKey(cloudKey) + const fileBuffer = await StorageService.downloadFile({ key: cloudKey, context }) + logger.info( + `Downloaded file from ${context} storage (${explicitContext ? 'explicit' : 'inferred'}): ${cloudKey}, size: ${fileBuffer.length} bytes` + ) const filename = cloudKey.split('/').pop() || cloudKey const extension = path.extname(filename).toLowerCase().substring(1) @@ -357,13 +400,11 @@ async function handleCloudFile(filePath: string, fileType?: string): Promise ({ + fileName: file.fileName, + contentType: file.contentType, + fileSize: file.fileSize, + })), + uploadType, + sessionUserId, + 3600 // 1 hour + ) const duration = Date.now() - startTime logger.info( `Generated ${files.length} presigned URLs in ${duration}ms (avg ${Math.round(duration / files.length)}ms per file)` ) - return NextResponse.json(result) + const storagePrefix = USE_BLOB_STORAGE ? 'blob' : 's3' + + return NextResponse.json({ + files: presignedUrls.map((urlResponse, index) => { + const finalPath = `/api/files/serve/${storagePrefix}/${encodeURIComponent(urlResponse.key)}?context=${uploadType}` + + return { + fileName: files[index].fileName, + presignedUrl: urlResponse.url, + fileInfo: { + path: finalPath, + key: urlResponse.key, + name: files[index].fileName, + size: files[index].fileSize, + type: files[index].contentType, + }, + uploadHeaders: urlResponse.uploadHeaders, + directUploadSupported: true, + } + }), + directUploadSupported: true, + }) } catch (error) { logger.error('Error generating batch presigned URLs:', error) return createErrorResponse( @@ -177,199 +183,16 @@ export async function POST(request: NextRequest) { } } -async function handleBatchS3PresignedUrls( - files: BatchFileRequest[], - uploadType: UploadType, - userId?: string -) { - const config = - uploadType === 'knowledge-base' - ? S3_KB_CONFIG - : uploadType === 'chat' - ? S3_CHAT_CONFIG - : uploadType === 'copilot' - ? S3_COPILOT_CONFIG - : S3_CONFIG - - if (!config.bucket || !config.region) { - throw new Error(`S3 configuration missing for ${uploadType} uploads`) - } - - const { getS3Client, sanitizeFilenameForMetadata } = await import('@/lib/uploads/s3/s3-client') - const s3Client = getS3Client() - - let prefix = '' - if (uploadType === 'knowledge-base') { - prefix = 'kb/' - } else if (uploadType === 'chat') { - prefix = 'chat/' - } else if (uploadType === 'copilot') { - prefix = `${userId}/` - } - - const baseMetadata: Record = { - uploadedAt: new Date().toISOString(), - } - - if (uploadType === 'knowledge-base') { - baseMetadata.purpose = 'knowledge-base' - } else if (uploadType === 'chat') { - baseMetadata.purpose = 'chat' - } else if (uploadType === 'copilot') { - baseMetadata.purpose = 'copilot' - baseMetadata.userId = userId || '' - } - - const results = await Promise.all( - files.map(async (file) => { - const safeFileName = file.fileName.replace(/\s+/g, '-').replace(/[^a-zA-Z0-9.-]/g, '_') - const uniqueKey = `${prefix}${uuidv4()}-${safeFileName}` - const sanitizedOriginalName = sanitizeFilenameForMetadata(file.fileName) - - const metadata = { - ...baseMetadata, - originalName: sanitizedOriginalName, - } - - const command = new PutObjectCommand({ - Bucket: config.bucket, - Key: uniqueKey, - ContentType: file.contentType, - Metadata: metadata, - }) - - const presignedUrl = await getSignedUrl(s3Client, command, { expiresIn: 3600 }) - - const finalPath = - uploadType === 'chat' - ? `https://${config.bucket}.s3.${config.region}.amazonaws.com/${uniqueKey}` - : `/api/files/serve/s3/${encodeURIComponent(uniqueKey)}` - - return { - fileName: file.fileName, - presignedUrl, - fileInfo: { - path: finalPath, - key: uniqueKey, - name: file.fileName, - size: file.fileSize, - type: file.contentType, - }, - } - }) - ) - - return { - files: results, - directUploadSupported: true, - } -} - -async function handleBatchBlobPresignedUrls( - files: BatchFileRequest[], - uploadType: UploadType, - userId?: string -) { - const config = - uploadType === 'knowledge-base' - ? BLOB_KB_CONFIG - : uploadType === 'chat' - ? BLOB_CHAT_CONFIG - : uploadType === 'copilot' - ? BLOB_COPILOT_CONFIG - : BLOB_CONFIG - - if ( - !config.accountName || - !config.containerName || - (!config.accountKey && !config.connectionString) - ) { - throw new Error(`Azure Blob configuration missing for ${uploadType} uploads`) - } - - const { getBlobServiceClient } = await import('@/lib/uploads/blob/blob-client') - const { BlobSASPermissions, generateBlobSASQueryParameters, StorageSharedKeyCredential } = - await import('@azure/storage-blob') - - const blobServiceClient = getBlobServiceClient() - const containerClient = blobServiceClient.getContainerClient(config.containerName) - - let prefix = '' - if (uploadType === 'knowledge-base') { - prefix = 'kb/' - } else if (uploadType === 'chat') { - prefix = 'chat/' - } else if (uploadType === 'copilot') { - prefix = `${userId}/` - } - - const baseUploadHeaders: Record = { - 'x-ms-blob-type': 'BlockBlob', - 'x-ms-meta-uploadedat': new Date().toISOString(), - } - - if (uploadType === 'knowledge-base') { - baseUploadHeaders['x-ms-meta-purpose'] = 'knowledge-base' - } else if (uploadType === 'chat') { - baseUploadHeaders['x-ms-meta-purpose'] = 'chat' - } else if (uploadType === 'copilot') { - baseUploadHeaders['x-ms-meta-purpose'] = 'copilot' - baseUploadHeaders['x-ms-meta-userid'] = encodeURIComponent(userId || '') - } - - const results = await Promise.all( - files.map(async (file) => { - const safeFileName = file.fileName.replace(/\s+/g, '-').replace(/[^a-zA-Z0-9.-]/g, '_') - const uniqueKey = `${prefix}${uuidv4()}-${safeFileName}` - const blockBlobClient = containerClient.getBlockBlobClient(uniqueKey) - - const sasOptions = { - containerName: config.containerName, - blobName: uniqueKey, - permissions: BlobSASPermissions.parse('w'), - startsOn: new Date(), - expiresOn: new Date(Date.now() + 3600 * 1000), - } - - const sasToken = generateBlobSASQueryParameters( - sasOptions, - new StorageSharedKeyCredential(config.accountName, config.accountKey || '') - ).toString() - - const presignedUrl = `${blockBlobClient.url}?${sasToken}` - - const finalPath = - uploadType === 'chat' - ? blockBlobClient.url - : `/api/files/serve/blob/${encodeURIComponent(uniqueKey)}` - - const uploadHeaders = { - ...baseUploadHeaders, - 'x-ms-blob-content-type': file.contentType, - 'x-ms-meta-originalname': encodeURIComponent(file.fileName), - } - - return { - fileName: file.fileName, - presignedUrl, - fileInfo: { - path: finalPath, - key: uniqueKey, - name: file.fileName, - size: file.fileSize, - type: file.contentType, - }, - uploadHeaders, - } - }) - ) - - return { - files: results, - directUploadSupported: true, - } -} - export async function OPTIONS() { - return createOptionsResponse() + return NextResponse.json( + {}, + { + status: 200, + headers: { + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': 'POST, OPTIONS', + 'Access-Control-Allow-Headers': 'Content-Type, Authorization', + }, + } + ) } diff --git a/apps/sim/app/api/files/presigned/route.test.ts b/apps/sim/app/api/files/presigned/route.test.ts index 4c93c8f33..412b4f8f2 100644 --- a/apps/sim/app/api/files/presigned/route.test.ts +++ b/apps/sim/app/api/files/presigned/route.test.ts @@ -177,8 +177,8 @@ describe('/api/files/presigned', () => { expect(response.status).toBe(200) expect(data.presignedUrl).toBe('https://example.com/presigned-url') expect(data.fileInfo).toMatchObject({ - path: expect.stringContaining('/api/files/serve/s3/'), - key: expect.stringContaining('test-document.txt'), + path: expect.stringMatching(/\/api\/files\/serve\/s3\/.+\?context=general$/), // general uploads use serve path + key: expect.stringMatching(/.*test.document\.txt$/), name: 'test document.txt', size: 1024, type: 'text/plain', @@ -236,7 +236,8 @@ describe('/api/files/presigned', () => { expect(response.status).toBe(200) expect(data.fileInfo.key).toMatch(/^chat\/.*chat-logo\.png$/) - expect(data.fileInfo.path).toMatch(/^https:\/\/.*\.s3\..*\.amazonaws\.com\/chat\//) + expect(data.fileInfo.path).toMatch(/\/api\/files\/serve\/s3\/.+\?context=chat$/) + expect(data.presignedUrl).toBeTruthy() expect(data.directUploadSupported).toBe(true) }) @@ -261,24 +262,15 @@ describe('/api/files/presigned', () => { const data = await response.json() expect(response.status).toBe(200) - expect(data.presignedUrl).toContain( - 'https://testaccount.blob.core.windows.net/test-container' - ) - expect(data.presignedUrl).toContain('sas-token-string') + expect(data.presignedUrl).toBeTruthy() + expect(typeof data.presignedUrl).toBe('string') expect(data.fileInfo).toMatchObject({ - path: expect.stringContaining('/api/files/serve/blob/'), - key: expect.stringContaining('test-document.txt'), + key: expect.stringMatching(/.*test.document\.txt$/), name: 'test document.txt', size: 1024, type: 'text/plain', }) expect(data.directUploadSupported).toBe(true) - expect(data.uploadHeaders).toMatchObject({ - 'x-ms-blob-type': 'BlockBlob', - 'x-ms-blob-content-type': 'text/plain', - 'x-ms-meta-originalname': expect.any(String), - 'x-ms-meta-uploadedat': '2024-01-01T00:00:00.000Z', - }) }) it('should generate chat Azure Blob presigned URL with chat prefix and direct path', async () => { @@ -303,24 +295,22 @@ describe('/api/files/presigned', () => { expect(response.status).toBe(200) expect(data.fileInfo.key).toMatch(/^chat\/.*chat-logo\.png$/) - expect(data.fileInfo.path).toContain( - 'https://testaccount.blob.core.windows.net/test-container' - ) + expect(data.fileInfo.path).toMatch(/\/api\/files\/serve\/blob\/.+\?context=chat$/) + expect(data.presignedUrl).toBeTruthy() expect(data.directUploadSupported).toBe(true) - expect(data.uploadHeaders).toMatchObject({ - 'x-ms-blob-type': 'BlockBlob', - 'x-ms-blob-content-type': 'image/png', - 'x-ms-meta-originalname': expect.any(String), - 'x-ms-meta-uploadedat': '2024-01-01T00:00:00.000Z', - 'x-ms-meta-purpose': 'chat', - }) }) it('should return error for unknown storage provider', async () => { - // For unknown provider, we'll need to mock manually since our helper doesn't support it - vi.doMock('@/lib/uploads', () => ({ - getStorageProvider: vi.fn().mockReturnValue('unknown'), - isUsingCloudStorage: vi.fn().mockReturnValue(true), + setupFileApiMocks({ + cloudEnabled: true, + storageProvider: 's3', + }) + + vi.doMock('@/lib/uploads/core/storage-service', () => ({ + hasCloudStorage: vi.fn().mockReturnValue(true), + generatePresignedUploadUrl: vi + .fn() + .mockRejectedValue(new Error('Unknown storage provider: unknown')), })) const { POST } = await import('@/app/api/files/presigned/route') @@ -337,10 +327,9 @@ describe('/api/files/presigned', () => { const response = await POST(request) const data = await response.json() - expect(response.status).toBe(500) // Changed from 400 to 500 (StorageConfigError) - expect(data.error).toBe('Unknown storage provider: unknown') // Updated error message - expect(data.code).toBe('STORAGE_CONFIG_ERROR') - expect(data.directUploadSupported).toBe(false) + expect(response.status).toBe(500) + expect(data.error).toBeTruthy() + expect(typeof data.error).toBe('string') }) it('should handle S3 errors gracefully', async () => { @@ -349,21 +338,9 @@ describe('/api/files/presigned', () => { storageProvider: 's3', }) - // Override with error-throwing mock while preserving other exports - vi.doMock('@/lib/uploads', () => ({ - getStorageProvider: vi.fn().mockReturnValue('s3'), - isUsingCloudStorage: vi.fn().mockReturnValue(true), - uploadFile: vi.fn().mockResolvedValue({ - path: '/api/files/serve/test-key', - key: 'test-key', - name: 'test.txt', - size: 100, - type: 'text/plain', - }), - })) - - vi.doMock('@aws-sdk/s3-request-presigner', () => ({ - getSignedUrl: vi.fn().mockRejectedValue(new Error('S3 service unavailable')), + vi.doMock('@/lib/uploads/core/storage-service', () => ({ + hasCloudStorage: vi.fn().mockReturnValue(true), + generatePresignedUploadUrl: vi.fn().mockRejectedValue(new Error('S3 service unavailable')), })) const { POST } = await import('@/app/api/files/presigned/route') @@ -381,10 +358,8 @@ describe('/api/files/presigned', () => { const data = await response.json() expect(response.status).toBe(500) - expect(data.error).toBe( - 'Failed to generate S3 presigned URL - check AWS credentials and permissions' - ) // Updated error message - expect(data.code).toBe('STORAGE_CONFIG_ERROR') + expect(data.error).toBeTruthy() + expect(typeof data.error).toBe('string') }) it('should handle Azure Blob errors gracefully', async () => { @@ -393,23 +368,11 @@ describe('/api/files/presigned', () => { storageProvider: 'blob', }) - vi.doMock('@/lib/uploads', () => ({ - getStorageProvider: vi.fn().mockReturnValue('blob'), - isUsingCloudStorage: vi.fn().mockReturnValue(true), - uploadFile: vi.fn().mockResolvedValue({ - path: '/api/files/serve/test-key', - key: 'test-key', - name: 'test.txt', - size: 100, - type: 'text/plain', - }), - })) - - vi.doMock('@/lib/uploads/blob/blob-client', () => ({ - getBlobServiceClient: vi.fn().mockImplementation(() => { - throw new Error('Azure service unavailable') - }), - sanitizeFilenameForMetadata: vi.fn((filename) => filename), + vi.doMock('@/lib/uploads/core/storage-service', () => ({ + hasCloudStorage: vi.fn().mockReturnValue(true), + generatePresignedUploadUrl: vi + .fn() + .mockRejectedValue(new Error('Azure service unavailable')), })) const { POST } = await import('@/app/api/files/presigned/route') @@ -427,8 +390,8 @@ describe('/api/files/presigned', () => { const data = await response.json() expect(response.status).toBe(500) - expect(data.error).toBe('Failed to generate Azure Blob presigned URL') // Updated error message - expect(data.code).toBe('STORAGE_CONFIG_ERROR') + expect(data.error).toBeTruthy() + expect(typeof data.error).toBe('string') }) it('should handle malformed JSON gracefully', async () => { @@ -459,11 +422,11 @@ describe('/api/files/presigned', () => { const response = await OPTIONS() - expect(response.status).toBe(204) - expect(response.headers.get('Access-Control-Allow-Methods')).toBe( - 'GET, POST, DELETE, OPTIONS' + expect(response.status).toBe(200) + expect(response.headers.get('Access-Control-Allow-Methods')).toBe('POST, OPTIONS') + expect(response.headers.get('Access-Control-Allow-Headers')).toBe( + 'Content-Type, Authorization' ) - expect(response.headers.get('Access-Control-Allow-Headers')).toBe('Content-Type') }) }) }) diff --git a/apps/sim/app/api/files/presigned/route.ts b/apps/sim/app/api/files/presigned/route.ts index cf41ffb0d..8b8128309 100644 --- a/apps/sim/app/api/files/presigned/route.ts +++ b/apps/sim/app/api/files/presigned/route.ts @@ -1,26 +1,12 @@ -import { PutObjectCommand } from '@aws-sdk/client-s3' -import { getSignedUrl } from '@aws-sdk/s3-request-presigner' import { type NextRequest, NextResponse } from 'next/server' -import { v4 as uuidv4 } from 'uuid' import { getSession } from '@/lib/auth' import { createLogger } from '@/lib/logs/console/logger' -import { getStorageProvider, isUsingCloudStorage } from '@/lib/uploads' -import { isImageFileType } from '@/lib/uploads/file-utils' -// Dynamic imports for storage clients to avoid client-side bundling -import { - BLOB_CHAT_CONFIG, - BLOB_CONFIG, - BLOB_COPILOT_CONFIG, - BLOB_KB_CONFIG, - BLOB_PROFILE_PICTURES_CONFIG, - S3_CHAT_CONFIG, - S3_CONFIG, - S3_COPILOT_CONFIG, - S3_KB_CONFIG, - S3_PROFILE_PICTURES_CONFIG, -} from '@/lib/uploads/setup' -import { validateFileType } from '@/lib/uploads/validation' -import { createErrorResponse, createOptionsResponse } from '@/app/api/files/utils' +import { CopilotFiles } from '@/lib/uploads' +import type { StorageContext } from '@/lib/uploads/core/config-resolver' +import { USE_BLOB_STORAGE } from '@/lib/uploads/core/setup' +import { generatePresignedUploadUrl, hasCloudStorage } from '@/lib/uploads/core/storage-service' +import { validateFileType } from '@/lib/uploads/utils/validation' +import { createErrorResponse } from '@/app/api/files/utils' const logger = createLogger('PresignedUploadAPI') @@ -32,8 +18,6 @@ interface PresignedUrlRequest { chatId?: string } -type UploadType = 'general' | 'knowledge-base' | 'chat' | 'copilot' | 'profile-pictures' - class PresignedUrlError extends Error { constructor( message: string, @@ -45,12 +29,6 @@ class PresignedUrlError extends Error { } } -class StorageConfigError extends PresignedUrlError { - constructor(message: string) { - super(message, 'STORAGE_CONFIG_ERROR', 500) - } -} - class ValidationError extends PresignedUrlError { constructor(message: string) { super(message, 'VALIDATION_ERROR', 400) @@ -91,7 +69,7 @@ export async function POST(request: NextRequest) { } const uploadTypeParam = request.nextUrl.searchParams.get('type') - const uploadType: UploadType = + const uploadType: StorageContext = uploadTypeParam === 'knowledge-base' ? 'knowledge-base' : uploadTypeParam === 'chat' @@ -109,38 +87,9 @@ export async function POST(request: NextRequest) { } } - // Evaluate user id from session for copilot uploads const sessionUserId = session.user.id - // Validate copilot-specific requirements (use session user) - if (uploadType === 'copilot') { - if (!sessionUserId?.trim()) { - throw new ValidationError('Authenticated user session is required for copilot uploads') - } - // Only allow image uploads for copilot - if (!isImageFileType(contentType)) { - throw new ValidationError( - 'Only image files (JPEG, PNG, GIF, WebP, SVG) are allowed for copilot uploads' - ) - } - } - - // Validate profile picture requirements - if (uploadType === 'profile-pictures') { - if (!sessionUserId?.trim()) { - throw new ValidationError( - 'Authenticated user session is required for profile picture uploads' - ) - } - // Only allow image uploads for profile pictures - if (!isImageFileType(contentType)) { - throw new ValidationError( - 'Only image files (JPEG, PNG, GIF, WebP, SVG) are allowed for profile picture uploads' - ) - } - } - - if (!isUsingCloudStorage()) { + if (!hasCloudStorage()) { logger.info( `Local storage detected - presigned URL not available for ${fileName}, client will use API fallback` ) @@ -158,29 +107,63 @@ export async function POST(request: NextRequest) { }) } - const storageProvider = getStorageProvider() - logger.info(`Generating ${uploadType} presigned URL for ${fileName} using ${storageProvider}`) + logger.info(`Generating ${uploadType} presigned URL for ${fileName}`) - switch (storageProvider) { - case 's3': - return await handleS3PresignedUrl( + let presignedUrlResponse + + if (uploadType === 'copilot') { + try { + presignedUrlResponse = await CopilotFiles.generateCopilotUploadUrl({ fileName, contentType, fileSize, - uploadType, - sessionUserId + userId: sessionUserId, + expirationSeconds: 3600, + }) + } catch (error) { + throw new ValidationError( + error instanceof Error ? error.message : 'Copilot validation failed' ) - case 'blob': - return await handleBlobPresignedUrl( - fileName, - contentType, - fileSize, - uploadType, - sessionUserId - ) - default: - throw new StorageConfigError(`Unknown storage provider: ${storageProvider}`) + } + } else { + if (uploadType === 'profile-pictures') { + if (!sessionUserId?.trim()) { + throw new ValidationError( + 'Authenticated user session is required for profile picture uploads' + ) + } + if (!CopilotFiles.isImageFileType(contentType)) { + throw new ValidationError( + 'Only image files (JPEG, PNG, GIF, WebP, SVG) are allowed for profile picture uploads' + ) + } + } + + presignedUrlResponse = await generatePresignedUploadUrl({ + fileName, + contentType, + fileSize, + context: uploadType, + userId: sessionUserId, + expirationSeconds: 3600, // 1 hour + }) } + + const finalPath = `/api/files/serve/${USE_BLOB_STORAGE ? 'blob' : 's3'}/${encodeURIComponent(presignedUrlResponse.key)}?context=${uploadType}` + + return NextResponse.json({ + fileName, + presignedUrl: presignedUrlResponse.url, + fileInfo: { + path: finalPath, + key: presignedUrlResponse.key, + name: fileName, + size: fileSize, + type: contentType, + }, + uploadHeaders: presignedUrlResponse.uploadHeaders, + directUploadSupported: true, + }) } catch (error) { logger.error('Error generating presigned URL:', error) @@ -201,234 +184,16 @@ export async function POST(request: NextRequest) { } } -async function handleS3PresignedUrl( - fileName: string, - contentType: string, - fileSize: number, - uploadType: UploadType, - userId?: string -) { - try { - const config = - uploadType === 'knowledge-base' - ? S3_KB_CONFIG - : uploadType === 'chat' - ? S3_CHAT_CONFIG - : uploadType === 'copilot' - ? S3_COPILOT_CONFIG - : uploadType === 'profile-pictures' - ? S3_PROFILE_PICTURES_CONFIG - : S3_CONFIG - - if (!config.bucket || !config.region) { - throw new StorageConfigError(`S3 configuration missing for ${uploadType} uploads`) - } - - const safeFileName = fileName.replace(/\s+/g, '-').replace(/[^a-zA-Z0-9.-]/g, '_') - - let prefix = '' - if (uploadType === 'knowledge-base') { - prefix = 'kb/' - } else if (uploadType === 'chat') { - prefix = 'chat/' - } else if (uploadType === 'copilot') { - prefix = `${userId}/` - } else if (uploadType === 'profile-pictures') { - prefix = `${userId}/` - } - - const uniqueKey = `${prefix}${uuidv4()}-${safeFileName}` - - const { sanitizeFilenameForMetadata } = await import('@/lib/uploads/s3/s3-client') - const sanitizedOriginalName = sanitizeFilenameForMetadata(fileName) - - const metadata: Record = { - originalName: sanitizedOriginalName, - uploadedAt: new Date().toISOString(), - } - - if (uploadType === 'knowledge-base') { - metadata.purpose = 'knowledge-base' - } else if (uploadType === 'chat') { - metadata.purpose = 'chat' - } else if (uploadType === 'copilot') { - metadata.purpose = 'copilot' - metadata.userId = userId || '' - } else if (uploadType === 'profile-pictures') { - metadata.purpose = 'profile-pictures' - metadata.userId = userId || '' - } - - const command = new PutObjectCommand({ - Bucket: config.bucket, - Key: uniqueKey, - ContentType: contentType, - Metadata: metadata, - }) - - let presignedUrl: string - try { - const { getS3Client } = await import('@/lib/uploads/s3/s3-client') - presignedUrl = await getSignedUrl(getS3Client(), command, { expiresIn: 3600 }) - } catch (s3Error) { - logger.error('Failed to generate S3 presigned URL:', s3Error) - throw new StorageConfigError( - 'Failed to generate S3 presigned URL - check AWS credentials and permissions' - ) - } - - const finalPath = - uploadType === 'chat' || uploadType === 'profile-pictures' - ? `https://${config.bucket}.s3.${config.region}.amazonaws.com/${uniqueKey}` - : `/api/files/serve/s3/${encodeURIComponent(uniqueKey)}` - - logger.info(`Generated ${uploadType} S3 presigned URL for ${fileName} (${uniqueKey})`) - logger.info(`Presigned URL: ${presignedUrl}`) - logger.info(`Final path: ${finalPath}`) - - return NextResponse.json({ - presignedUrl, - uploadUrl: presignedUrl, // Make sure we're returning the uploadUrl field - fileInfo: { - path: finalPath, - key: uniqueKey, - name: fileName, - size: fileSize, - type: contentType, - }, - directUploadSupported: true, - }) - } catch (error) { - if (error instanceof PresignedUrlError) { - throw error - } - logger.error('Error in S3 presigned URL generation:', error) - throw new StorageConfigError('Failed to generate S3 presigned URL') - } -} - -async function handleBlobPresignedUrl( - fileName: string, - contentType: string, - fileSize: number, - uploadType: UploadType, - userId?: string -) { - try { - const config = - uploadType === 'knowledge-base' - ? BLOB_KB_CONFIG - : uploadType === 'chat' - ? BLOB_CHAT_CONFIG - : uploadType === 'copilot' - ? BLOB_COPILOT_CONFIG - : uploadType === 'profile-pictures' - ? BLOB_PROFILE_PICTURES_CONFIG - : BLOB_CONFIG - - if ( - !config.accountName || - !config.containerName || - (!config.accountKey && !config.connectionString) - ) { - throw new StorageConfigError(`Azure Blob configuration missing for ${uploadType} uploads`) - } - - const safeFileName = fileName.replace(/\s+/g, '-').replace(/[^a-zA-Z0-9.-]/g, '_') - - let prefix = '' - if (uploadType === 'knowledge-base') { - prefix = 'kb/' - } else if (uploadType === 'chat') { - prefix = 'chat/' - } else if (uploadType === 'copilot') { - prefix = `${userId}/` - } else if (uploadType === 'profile-pictures') { - prefix = `${userId}/` - } - - const uniqueKey = `${prefix}${uuidv4()}-${safeFileName}` - - const { getBlobServiceClient } = await import('@/lib/uploads/blob/blob-client') - const blobServiceClient = getBlobServiceClient() - const containerClient = blobServiceClient.getContainerClient(config.containerName) - const blockBlobClient = containerClient.getBlockBlobClient(uniqueKey) - - const { BlobSASPermissions, generateBlobSASQueryParameters, StorageSharedKeyCredential } = - await import('@azure/storage-blob') - - const sasOptions = { - containerName: config.containerName, - blobName: uniqueKey, - permissions: BlobSASPermissions.parse('w'), // Write permission for upload - startsOn: new Date(), - expiresOn: new Date(Date.now() + 3600 * 1000), // 1 hour expiration - } - - let sasToken: string - try { - sasToken = generateBlobSASQueryParameters( - sasOptions, - new StorageSharedKeyCredential(config.accountName, config.accountKey || '') - ).toString() - } catch (blobError) { - logger.error('Failed to generate Azure Blob SAS token:', blobError) - throw new StorageConfigError( - 'Failed to generate Azure Blob SAS token - check Azure credentials and permissions' - ) - } - - const presignedUrl = `${blockBlobClient.url}?${sasToken}` - - // For chat images and profile pictures, use direct Blob URLs since they need to be permanently accessible - // For other files, use serve path for access control - const finalPath = - uploadType === 'chat' || uploadType === 'profile-pictures' - ? blockBlobClient.url - : `/api/files/serve/blob/${encodeURIComponent(uniqueKey)}` - - logger.info(`Generated ${uploadType} Azure Blob presigned URL for ${fileName} (${uniqueKey})`) - - const uploadHeaders: Record = { - 'x-ms-blob-type': 'BlockBlob', - 'x-ms-blob-content-type': contentType, - 'x-ms-meta-originalname': encodeURIComponent(fileName), - 'x-ms-meta-uploadedat': new Date().toISOString(), - } - - if (uploadType === 'knowledge-base') { - uploadHeaders['x-ms-meta-purpose'] = 'knowledge-base' - } else if (uploadType === 'chat') { - uploadHeaders['x-ms-meta-purpose'] = 'chat' - } else if (uploadType === 'copilot') { - uploadHeaders['x-ms-meta-purpose'] = 'copilot' - uploadHeaders['x-ms-meta-userid'] = encodeURIComponent(userId || '') - } else if (uploadType === 'profile-pictures') { - uploadHeaders['x-ms-meta-purpose'] = 'profile-pictures' - uploadHeaders['x-ms-meta-userid'] = encodeURIComponent(userId || '') - } - - return NextResponse.json({ - presignedUrl, - fileInfo: { - path: finalPath, - key: uniqueKey, - name: fileName, - size: fileSize, - type: contentType, - }, - directUploadSupported: true, - uploadHeaders, - }) - } catch (error) { - if (error instanceof PresignedUrlError) { - throw error - } - logger.error('Error in Azure Blob presigned URL generation:', error) - throw new StorageConfigError('Failed to generate Azure Blob presigned URL') - } -} - export async function OPTIONS() { - return createOptionsResponse() + return NextResponse.json( + {}, + { + status: 200, + headers: { + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': 'POST, OPTIONS', + 'Access-Control-Allow-Headers': 'Content-Type, Authorization', + }, + } + ) } diff --git a/apps/sim/app/api/files/serve/[...path]/route.test.ts b/apps/sim/app/api/files/serve/[...path]/route.test.ts index f39325201..5e7a64ce6 100644 --- a/apps/sim/app/api/files/serve/[...path]/route.test.ts +++ b/apps/sim/app/api/files/serve/[...path]/route.test.ts @@ -118,12 +118,24 @@ describe('File Serve API Route', () => { }) it('should serve cloud file by downloading and proxying', async () => { + const downloadFileMock = vi.fn().mockResolvedValue(Buffer.from('test cloud file content')) + vi.doMock('@/lib/uploads', () => ({ - downloadFile: vi.fn().mockResolvedValue(Buffer.from('test cloud file content')), - getPresignedUrl: vi.fn().mockResolvedValue('https://example-s3.com/presigned-url'), + StorageService: { + downloadFile: downloadFileMock, + generatePresignedDownloadUrl: vi + .fn() + .mockResolvedValue('https://example-s3.com/presigned-url'), + hasCloudStorage: vi.fn().mockReturnValue(true), + }, isUsingCloudStorage: vi.fn().mockReturnValue(true), })) + vi.doMock('@/lib/uploads/core/storage-service', () => ({ + downloadFile: downloadFileMock, + hasCloudStorage: vi.fn().mockReturnValue(true), + })) + vi.doMock('@/lib/uploads/setup', () => ({ UPLOAD_DIR: '/test/uploads', USE_S3_STORAGE: true, @@ -170,8 +182,10 @@ describe('File Serve API Route', () => { expect(response.status).toBe(200) expect(response.headers.get('Content-Type')).toBe('image/png') - const uploads = await import('@/lib/uploads') - expect(uploads.downloadFile).toHaveBeenCalledWith('1234567890-image.png') + expect(downloadFileMock).toHaveBeenCalledWith({ + key: '1234567890-image.png', + context: 'general', + }) }) it('should return 404 when file not found', async () => { @@ -236,7 +250,7 @@ describe('File Serve API Route', () => { getContentType: () => test.contentType, findLocalFile: () => `/test/uploads/file.${test.ext}`, createFileResponse: (obj: { buffer: Buffer; contentType: string; filename: string }) => - new Response(obj.buffer, { + new Response(obj.buffer as any, { status: 200, headers: { 'Content-Type': obj.contentType, diff --git a/apps/sim/app/api/files/serve/[...path]/route.ts b/apps/sim/app/api/files/serve/[...path]/route.ts index 8f1462fed..3ed643e06 100644 --- a/apps/sim/app/api/files/serve/[...path]/route.ts +++ b/apps/sim/app/api/files/serve/[...path]/route.ts @@ -3,9 +3,9 @@ import type { NextRequest } from 'next/server' import { NextResponse } from 'next/server' import { checkHybridAuth } from '@/lib/auth/hybrid' import { createLogger } from '@/lib/logs/console/logger' -import { downloadFile, getStorageProvider, isUsingCloudStorage } from '@/lib/uploads' -import { S3_KB_CONFIG } from '@/lib/uploads/setup' -import '@/lib/uploads/setup.server' +import { CopilotFiles, isUsingCloudStorage } from '@/lib/uploads' +import type { StorageContext } from '@/lib/uploads/core/config-resolver' +import { downloadFile } from '@/lib/uploads/core/storage-service' import { createErrorResponse, createFileResponse, @@ -43,9 +43,11 @@ export async function GET( const isCloudPath = isS3Path || isBlobPath const cloudKey = isCloudPath ? path.slice(1).join('/') : fullPath + const contextParam = request.nextUrl.searchParams.get('context') + const legacyBucketType = request.nextUrl.searchParams.get('bucket') + if (isUsingCloudStorage() || isCloudPath) { - const bucketType = request.nextUrl.searchParams.get('bucket') - return await handleCloudProxy(cloudKey, bucketType, userId) + return await handleCloudProxy(cloudKey, contextParam, legacyBucketType, userId) } return await handleLocalFile(fullPath, userId) @@ -84,69 +86,70 @@ async function handleLocalFile(filename: string, userId?: string): Promise { - logger.info(`Downloading KB file: ${cloudKey}`) - const storageProvider = getStorageProvider() - - if (storageProvider === 'blob') { - const { BLOB_KB_CONFIG } = await import('@/lib/uploads/setup') - return downloadFile(cloudKey, { - containerName: BLOB_KB_CONFIG.containerName, - accountName: BLOB_KB_CONFIG.accountName, - accountKey: BLOB_KB_CONFIG.accountKey, - connectionString: BLOB_KB_CONFIG.connectionString, - }) +/** + * Infer storage context from file key pattern + */ +function inferContextFromKey(key: string): StorageContext { + // KB files always start with 'kb/' prefix + if (key.startsWith('kb/')) { + return 'knowledge-base' } - if (storageProvider === 's3') { - return downloadFile(cloudKey, { - bucket: S3_KB_CONFIG.bucket, - region: S3_KB_CONFIG.region, - }) + // Workspace files: UUID-like ID followed by timestamp pattern + // Pattern: {uuid}/{timestamp}-{random}-{filename} + if (key.match(/^[a-f0-9-]{36}\/\d+-[a-z0-9]+-/)) { + return 'workspace' } - throw new Error(`Unsupported storage provider for KB files: ${storageProvider}`) + // Execution files: three UUID segments (workspace/workflow/execution) + // Pattern: {uuid}/{uuid}/{uuid}/{filename} + const segments = key.split('/') + if (segments.length >= 4 && segments[0].match(/^[a-f0-9-]{36}$/)) { + return 'execution' + } + + // Copilot files: timestamp-random-filename (no path segments) + // Pattern: {timestamp}-{random}-{filename} + // NOTE: This is ambiguous with other contexts - prefer explicit context parameter + if (key.match(/^\d+-[a-z0-9]+-/)) { + // Could be copilot, general, or chat - default to general + return 'general' + } + + return 'general' } async function handleCloudProxy( cloudKey: string, - bucketType?: string | null, + contextParam?: string | null, + legacyBucketType?: string | null, userId?: string ): Promise { try { - // Check if this is a KB file (starts with 'kb/') - const isKBFile = cloudKey.startsWith('kb/') + let context: StorageContext + + if (contextParam) { + context = contextParam as StorageContext + logger.info(`Using explicit context: ${context} for key: ${cloudKey}`) + } else if (legacyBucketType === 'copilot') { + context = 'copilot' + logger.info(`Using legacy bucket parameter for copilot context: ${cloudKey}`) + } else { + context = inferContextFromKey(cloudKey) + logger.info(`Inferred context: ${context} from key pattern: ${cloudKey}`) + } let fileBuffer: Buffer - if (isKBFile) { - fileBuffer = await downloadKBFile(cloudKey) - } else if (bucketType === 'copilot') { - const storageProvider = getStorageProvider() - - if (storageProvider === 's3') { - const { S3_COPILOT_CONFIG } = await import('@/lib/uploads/setup') - fileBuffer = await downloadFile(cloudKey, { - bucket: S3_COPILOT_CONFIG.bucket, - region: S3_COPILOT_CONFIG.region, - }) - } else if (storageProvider === 'blob') { - const { BLOB_COPILOT_CONFIG } = await import('@/lib/uploads/setup') - fileBuffer = await downloadFile(cloudKey, { - containerName: BLOB_COPILOT_CONFIG.containerName, - accountName: BLOB_COPILOT_CONFIG.accountName, - accountKey: BLOB_COPILOT_CONFIG.accountKey, - connectionString: BLOB_COPILOT_CONFIG.connectionString, - }) - } else { - fileBuffer = await downloadFile(cloudKey) - } + if (context === 'copilot') { + fileBuffer = await CopilotFiles.downloadCopilotFile(cloudKey) } else { - // Default bucket - fileBuffer = await downloadFile(cloudKey) + fileBuffer = await downloadFile({ + key: cloudKey, + context, + }) } - // Extract the original filename from the key (last part after last /) const originalFilename = cloudKey.split('/').pop() || 'download' const contentType = getContentType(originalFilename) @@ -154,7 +157,7 @@ async function handleCloudProxy( userId, key: cloudKey, size: fileBuffer.length, - bucket: bucketType || 'default', + context, }) return createFileResponse({ diff --git a/apps/sim/app/api/files/upload/route.test.ts b/apps/sim/app/api/files/upload/route.test.ts index 560eff1e3..34aa1f6c3 100644 --- a/apps/sim/app/api/files/upload/route.test.ts +++ b/apps/sim/app/api/files/upload/route.test.ts @@ -54,7 +54,6 @@ describe('File Upload API Route', () => { const response = await POST(req) const data = await response.json() - // Log error details if test fails if (response.status !== 200) { console.error('Upload failed with status:', response.status) console.error('Error response:', data) @@ -67,9 +66,8 @@ describe('File Upload API Route', () => { expect(data).toHaveProperty('size') expect(data).toHaveProperty('type', 'text/plain') - // Verify the upload function was called (we're mocking at the uploadFile level) - const { uploadFile } = await import('@/lib/uploads') - expect(uploadFile).toHaveBeenCalled() + const { StorageService } = await import('@/lib/uploads') + expect(StorageService.uploadFile).toHaveBeenCalled() }) it('should upload a file to S3 when in S3 mode', async () => { @@ -99,7 +97,7 @@ describe('File Upload API Route', () => { expect(data).toHaveProperty('type', 'text/plain') const uploads = await import('@/lib/uploads') - expect(uploads.uploadFile).toHaveBeenCalled() + expect(uploads.StorageService.uploadFile).toHaveBeenCalled() }) it('should handle multiple file uploads', async () => { @@ -153,9 +151,9 @@ describe('File Upload API Route', () => { storageProvider: 's3', }) - vi.doMock('@/lib/uploads', () => ({ + vi.doMock('@/lib/uploads/core/storage-service', () => ({ uploadFile: vi.fn().mockRejectedValue(new Error('Upload failed')), - isUsingCloudStorage: vi.fn().mockReturnValue(true), + hasCloudStorage: vi.fn().mockReturnValue(true), })) const mockFile = createMockFile() @@ -172,8 +170,8 @@ describe('File Upload API Route', () => { const data = await response.json() expect(response.status).toBe(500) - expect(data).toHaveProperty('error', 'Error') - expect(data).toHaveProperty('message', 'Upload failed') + expect(data).toHaveProperty('error') + expect(typeof data.error).toBe('string') }) it('should handle CORS preflight requests', async () => { @@ -200,10 +198,21 @@ describe('File Upload Security Tests', () => { vi.doMock('@/lib/uploads', () => ({ isUsingCloudStorage: vi.fn().mockReturnValue(false), + StorageService: { + uploadFile: vi.fn().mockResolvedValue({ + key: 'test-key', + path: '/test/path', + }), + hasCloudStorage: vi.fn().mockReturnValue(false), + }, + })) + + vi.doMock('@/lib/uploads/core/storage-service', () => ({ uploadFile: vi.fn().mockResolvedValue({ key: 'test-key', path: '/test/path', }), + hasCloudStorage: vi.fn().mockReturnValue(false), })) vi.doMock('@/lib/uploads/setup.server', () => ({})) @@ -325,11 +334,9 @@ describe('File Upload Security Tests', () => { it('should handle multiple files with mixed valid/invalid types', async () => { const formData = new FormData() - // Valid file const validFile = new File(['valid content'], 'valid.pdf', { type: 'application/pdf' }) formData.append('file', validFile) - // Invalid file (should cause rejection of entire request) const invalidFile = new File([''], 'malicious.html', { type: 'text/html', }) diff --git a/apps/sim/app/api/files/upload/route.ts b/apps/sim/app/api/files/upload/route.ts index 10f219f70..9c7aefda3 100644 --- a/apps/sim/app/api/files/upload/route.ts +++ b/apps/sim/app/api/files/upload/route.ts @@ -1,7 +1,6 @@ import { type NextRequest, NextResponse } from 'next/server' import { createLogger } from '@/lib/logs/console/logger' -import { getPresignedUrl, isUsingCloudStorage, uploadFile } from '@/lib/uploads' -import '@/lib/uploads/setup.server' +import '@/lib/uploads/core/setup.server' import { getSession } from '@/lib/auth' import { createErrorResponse, @@ -59,7 +58,8 @@ export async function POST(request: NextRequest) { const executionId = formData.get('executionId') as string | null const workspaceId = formData.get('workspaceId') as string | null - const usingCloudStorage = isUsingCloudStorage() + const storageService = await import('@/lib/uploads/core/storage-service') + const usingCloudStorage = storageService.hasCloudStorage() logger.info(`Using storage mode: ${usingCloudStorage ? 'Cloud' : 'Local'} for file upload`) if (workflowId && executionId) { @@ -87,7 +87,7 @@ export async function POST(request: NextRequest) { // Priority 1: Execution-scoped storage (temporary, 5 min expiry) if (workflowId && executionId) { - const { uploadExecutionFile } = await import('@/lib/workflows/execution-file-storage') + const { uploadExecutionFile } = await import('@/lib/uploads/contexts/execution') const userFile = await uploadExecutionFile( { workspaceId: workspaceId || '', @@ -106,7 +106,7 @@ export async function POST(request: NextRequest) { // Priority 2: Workspace-scoped storage (persistent, no expiry) if (workspaceId) { try { - const { uploadWorkspaceFile } = await import('@/lib/uploads/workspace-files') + const { uploadWorkspaceFile } = await import('@/lib/uploads/contexts/workspace') const userFile = await uploadWorkspaceFile( workspaceId, session.user.id, @@ -145,32 +145,42 @@ export async function POST(request: NextRequest) { } try { - logger.info(`Uploading file: ${originalName}`) - const result = await uploadFile(buffer, originalName, file.type, file.size) + logger.info(`Uploading file (general context): ${originalName}`) - let presignedUrl: string | undefined - if (usingCloudStorage) { + const storageService = await import('@/lib/uploads/core/storage-service') + const fileInfo = await storageService.uploadFile({ + file: buffer, + fileName: originalName, + contentType: file.type, + context: 'general', + }) + + let downloadUrl: string | undefined + if (storageService.hasCloudStorage()) { try { - presignedUrl = await getPresignedUrl(result.key, 24 * 60 * 60) // 24 hours + downloadUrl = await storageService.generatePresignedDownloadUrl( + fileInfo.key, + 'general', + 24 * 60 * 60 // 24 hours + ) } catch (error) { logger.warn(`Failed to generate presigned URL for ${originalName}:`, error) } } - const servePath = result.path - const uploadResult = { name: originalName, - size: file.size, + size: buffer.length, type: file.type, - key: result.key, - path: servePath, - url: presignedUrl || servePath, + key: fileInfo.key, + path: fileInfo.path, + url: downloadUrl || fileInfo.path, uploadedAt: new Date().toISOString(), expiresAt: new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString(), // 24 hours + context: 'general', } - logger.info(`Successfully uploaded: ${result.key}`) + logger.info(`Successfully uploaded: ${fileInfo.key}`) uploadResults.push(uploadResult) } catch (error) { logger.error(`Error uploading ${originalName}:`, error) diff --git a/apps/sim/app/api/files/utils.ts b/apps/sim/app/api/files/utils.ts index e46ef1ded..bb36a6fb7 100644 --- a/apps/sim/app/api/files/utils.ts +++ b/apps/sim/app/api/files/utils.ts @@ -2,7 +2,7 @@ import { existsSync } from 'fs' import { join, resolve, sep } from 'path' import { NextResponse } from 'next/server' import { createLogger } from '@/lib/logs/console/logger' -import { UPLOAD_DIR } from '@/lib/uploads/setup' +import { UPLOAD_DIR } from '@/lib/uploads/core/setup' const logger = createLogger('FilesUtils') diff --git a/apps/sim/app/api/logs/cleanup/route.ts b/apps/sim/app/api/logs/cleanup/route.ts index a64cbdf7f..9c182613e 100644 --- a/apps/sim/app/api/logs/cleanup/route.ts +++ b/apps/sim/app/api/logs/cleanup/route.ts @@ -1,5 +1,3 @@ -import { PutObjectCommand } from '@aws-sdk/client-s3' -// Dynamic import for S3 client to avoid client-side bundling import { db } from '@sim/db' import { subscription, user, workflow, workflowExecutionLogs } from '@sim/db/schema' import { and, eq, inArray, lt, sql } from 'drizzle-orm' @@ -8,17 +6,13 @@ import { verifyCronAuth } from '@/lib/auth/internal' import { env } from '@/lib/env' import { createLogger } from '@/lib/logs/console/logger' import { snapshotService } from '@/lib/logs/execution/snapshot/service' -import { deleteFile, isUsingCloudStorage } from '@/lib/uploads' +import { isUsingCloudStorage, StorageService } from '@/lib/uploads' export const dynamic = 'force-dynamic' const logger = createLogger('LogsCleanupAPI') const BATCH_SIZE = 2000 -const S3_CONFIG = { - bucket: env.S3_LOGS_BUCKET_NAME || '', - region: env.AWS_REGION || '', -} export async function GET(request: NextRequest) { try { @@ -27,10 +21,6 @@ export async function GET(request: NextRequest) { return authError } - if (!S3_CONFIG.bucket || !S3_CONFIG.region) { - return new NextResponse('Configuration error: S3 bucket or region not set', { status: 500 }) - } - const retentionDate = new Date() retentionDate.setDate(retentionDate.getDate() - Number(env.FREE_PLAN_LOG_RETENTION_DAYS || '7')) @@ -84,14 +74,12 @@ export async function GET(request: NextRequest) { const startTime = Date.now() const MAX_BATCHES = 10 - // Process enhanced logging cleanup let batchesProcessed = 0 let hasMoreLogs = true logger.info(`Starting enhanced logs cleanup for ${workflowIds.length} workflows`) while (hasMoreLogs && batchesProcessed < MAX_BATCHES) { - // Query enhanced execution logs that need cleanup const oldEnhancedLogs = await db .select({ id: workflowExecutionLogs.id, @@ -122,7 +110,6 @@ export async function GET(request: NextRequest) { for (const log of oldEnhancedLogs) { const today = new Date().toISOString().split('T')[0] - // Archive enhanced log with more detailed structure const enhancedLogKey = `archived-enhanced-logs/${today}/${log.id}.json` const enhancedLogData = JSON.stringify({ ...log, @@ -131,32 +118,31 @@ export async function GET(request: NextRequest) { }) try { - const { getS3Client } = await import('@/lib/uploads/s3/s3-client') - await getS3Client().send( - new PutObjectCommand({ - Bucket: S3_CONFIG.bucket, - Key: enhancedLogKey, - Body: enhancedLogData, - ContentType: 'application/json', - Metadata: { - logId: String(log.id), - workflowId: String(log.workflowId), - executionId: String(log.executionId), - logType: 'enhanced', - archivedAt: new Date().toISOString(), - }, - }) - ) + await StorageService.uploadFile({ + file: Buffer.from(enhancedLogData), + fileName: enhancedLogKey, + contentType: 'application/json', + context: 'general', + metadata: { + logId: String(log.id), + workflowId: String(log.workflowId), + executionId: String(log.executionId), + logType: 'enhanced', + archivedAt: new Date().toISOString(), + }, + }) results.enhancedLogs.archived++ - // Clean up associated files if using cloud storage if (isUsingCloudStorage() && log.files && Array.isArray(log.files)) { for (const file of log.files) { if (file && typeof file === 'object' && file.key) { results.files.total++ try { - await deleteFile(file.key) + await StorageService.deleteFile({ + key: file.key, + context: 'general', + }) results.files.deleted++ logger.info(`Deleted file: ${file.key}`) } catch (fileError) { @@ -168,7 +154,6 @@ export async function GET(request: NextRequest) { } try { - // Delete enhanced log const deleteResult = await db .delete(workflowExecutionLogs) .where(eq(workflowExecutionLogs.id, log.id)) @@ -200,7 +185,6 @@ export async function GET(request: NextRequest) { ) } - // Cleanup orphaned snapshots try { const snapshotRetentionDays = Number(env.FREE_PLAN_LOG_RETENTION_DAYS || '7') + 1 // Keep snapshots 1 day longer const cleanedSnapshots = await snapshotService.cleanupOrphanedSnapshots(snapshotRetentionDays) diff --git a/apps/sim/app/api/proxy/tts/route.ts b/apps/sim/app/api/proxy/tts/route.ts index b9a322e2d..2a8a869c2 100644 --- a/apps/sim/app/api/proxy/tts/route.ts +++ b/apps/sim/app/api/proxy/tts/route.ts @@ -3,7 +3,7 @@ import { NextResponse } from 'next/server' import { checkHybridAuth } from '@/lib/auth/hybrid' import { createLogger } from '@/lib/logs/console/logger' import { validateAlphanumericId } from '@/lib/security/input-validation' -import { uploadFile } from '@/lib/uploads/storage-client' +import { StorageService } from '@/lib/uploads' import { getBaseUrl } from '@/lib/urls/utils' const logger = createLogger('ProxyTTSAPI') @@ -65,7 +65,13 @@ export async function POST(request: NextRequest) { const audioBuffer = Buffer.from(await audioBlob.arrayBuffer()) const timestamp = Date.now() const fileName = `elevenlabs-tts-${timestamp}.mp3` - const fileInfo = await uploadFile(audioBuffer, fileName, 'audio/mpeg') + + const fileInfo = await StorageService.uploadFile({ + file: audioBuffer, + fileName, + contentType: 'audio/mpeg', + context: 'general', + }) const audioUrl = `${getBaseUrl()}${fileInfo.path}` diff --git a/apps/sim/app/api/tools/discord/send-message/route.ts b/apps/sim/app/api/tools/discord/send-message/route.ts index 7ea185751..945e932f9 100644 --- a/apps/sim/app/api/tools/discord/send-message/route.ts +++ b/apps/sim/app/api/tools/discord/send-message/route.ts @@ -2,7 +2,10 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkHybridAuth } from '@/lib/auth/hybrid' import { createLogger } from '@/lib/logs/console/logger' -import { downloadFileFromStorage, processFilesToUserFiles } from '@/lib/uploads/file-processing' +import { + downloadFileFromStorage, + processFilesToUserFiles, +} from '@/lib/uploads/utils/file-processing' import { generateRequestId } from '@/lib/utils' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/gmail/draft/route.ts b/apps/sim/app/api/tools/gmail/draft/route.ts index 15fbf7eef..7f8064211 100644 --- a/apps/sim/app/api/tools/gmail/draft/route.ts +++ b/apps/sim/app/api/tools/gmail/draft/route.ts @@ -2,7 +2,10 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkHybridAuth } from '@/lib/auth/hybrid' import { createLogger } from '@/lib/logs/console/logger' -import { downloadFileFromStorage, processFilesToUserFiles } from '@/lib/uploads/file-processing' +import { + downloadFileFromStorage, + processFilesToUserFiles, +} from '@/lib/uploads/utils/file-processing' import { generateRequestId } from '@/lib/utils' import { base64UrlEncode, buildMimeMessage } from '@/tools/gmail/utils' diff --git a/apps/sim/app/api/tools/gmail/send/route.ts b/apps/sim/app/api/tools/gmail/send/route.ts index 337e2591d..40987cc64 100644 --- a/apps/sim/app/api/tools/gmail/send/route.ts +++ b/apps/sim/app/api/tools/gmail/send/route.ts @@ -2,7 +2,10 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkHybridAuth } from '@/lib/auth/hybrid' import { createLogger } from '@/lib/logs/console/logger' -import { downloadFileFromStorage, processFilesToUserFiles } from '@/lib/uploads/file-processing' +import { + downloadFileFromStorage, + processFilesToUserFiles, +} from '@/lib/uploads/utils/file-processing' import { generateRequestId } from '@/lib/utils' import { base64UrlEncode, buildMimeMessage } from '@/tools/gmail/utils' diff --git a/apps/sim/app/api/tools/google_drive/upload/route.ts b/apps/sim/app/api/tools/google_drive/upload/route.ts index a3e1ed655..a63b61933 100644 --- a/apps/sim/app/api/tools/google_drive/upload/route.ts +++ b/apps/sim/app/api/tools/google_drive/upload/route.ts @@ -2,7 +2,10 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkHybridAuth } from '@/lib/auth/hybrid' import { createLogger } from '@/lib/logs/console/logger' -import { downloadFileFromStorage, processSingleFileToUserFile } from '@/lib/uploads/file-processing' +import { + downloadFileFromStorage, + processSingleFileToUserFile, +} from '@/lib/uploads/utils/file-processing' import { generateRequestId } from '@/lib/utils' import { GOOGLE_WORKSPACE_MIME_TYPES, diff --git a/apps/sim/app/api/tools/microsoft_teams/write_channel/route.ts b/apps/sim/app/api/tools/microsoft_teams/write_channel/route.ts index ef56d496b..e0672bbe1 100644 --- a/apps/sim/app/api/tools/microsoft_teams/write_channel/route.ts +++ b/apps/sim/app/api/tools/microsoft_teams/write_channel/route.ts @@ -2,7 +2,10 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkHybridAuth } from '@/lib/auth/hybrid' import { createLogger } from '@/lib/logs/console/logger' -import { downloadFileFromStorage, processFilesToUserFiles } from '@/lib/uploads/file-processing' +import { + downloadFileFromStorage, + processFilesToUserFiles, +} from '@/lib/uploads/utils/file-processing' import { generateRequestId } from '@/lib/utils' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/microsoft_teams/write_chat/route.ts b/apps/sim/app/api/tools/microsoft_teams/write_chat/route.ts index f7dd51976..fccaef44c 100644 --- a/apps/sim/app/api/tools/microsoft_teams/write_chat/route.ts +++ b/apps/sim/app/api/tools/microsoft_teams/write_chat/route.ts @@ -2,7 +2,10 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkHybridAuth } from '@/lib/auth/hybrid' import { createLogger } from '@/lib/logs/console/logger' -import { downloadFileFromStorage, processFilesToUserFiles } from '@/lib/uploads/file-processing' +import { + downloadFileFromStorage, + processFilesToUserFiles, +} from '@/lib/uploads/utils/file-processing' import { generateRequestId } from '@/lib/utils' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/mistral/parse/route.ts b/apps/sim/app/api/tools/mistral/parse/route.ts index ef77a867a..aa45d8234 100644 --- a/apps/sim/app/api/tools/mistral/parse/route.ts +++ b/apps/sim/app/api/tools/mistral/parse/route.ts @@ -2,8 +2,8 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkHybridAuth } from '@/lib/auth/hybrid' import { createLogger } from '@/lib/logs/console/logger' -import { getPresignedUrl } from '@/lib/uploads' -import { extractStorageKey } from '@/lib/uploads/file-utils' +import { type StorageContext, StorageService } from '@/lib/uploads' +import { extractStorageKey } from '@/lib/uploads/utils/file-utils' import { getBaseUrl } from '@/lib/urls/utils' import { generateRequestId } from '@/lib/utils' @@ -11,6 +11,19 @@ export const dynamic = 'force-dynamic' const logger = createLogger('MistralParseAPI') +/** + * Infer storage context from file key pattern + */ +function inferContextFromKey(key: string): StorageContext { + if (key.startsWith('kb/')) return 'knowledge-base' + + const segments = key.split('/') + if (segments.length >= 4 && segments[0].match(/^[a-f0-9-]{36}$/)) return 'execution' + if (key.match(/^[a-f0-9-]{36}\/\d+-[a-z0-9]+-/)) return 'workspace' + + return 'general' +} + const MistralParseSchema = z.object({ apiKey: z.string().min(1, 'API key is required'), filePath: z.string().min(1, 'File path is required'), @@ -52,9 +65,13 @@ export async function POST(request: NextRequest) { if (validatedData.filePath?.includes('/api/files/serve/')) { try { const storageKey = extractStorageKey(validatedData.filePath) + + // Infer context from key pattern + const context = inferContextFromKey(storageKey) + // Generate 5-minute presigned URL for external API access - fileUrl = await getPresignedUrl(storageKey, 5 * 60) - logger.info(`[${requestId}] Generated presigned URL for workspace file`) + fileUrl = await StorageService.generatePresignedDownloadUrl(storageKey, context, 5 * 60) + logger.info(`[${requestId}] Generated presigned URL for ${context} file`) } catch (error) { logger.error(`[${requestId}] Failed to generate presigned URL:`, error) return NextResponse.json( diff --git a/apps/sim/app/api/tools/onedrive/upload/route.ts b/apps/sim/app/api/tools/onedrive/upload/route.ts index 3857bc5ac..1c6d1ccc3 100644 --- a/apps/sim/app/api/tools/onedrive/upload/route.ts +++ b/apps/sim/app/api/tools/onedrive/upload/route.ts @@ -3,7 +3,10 @@ import * as XLSX from 'xlsx' import { z } from 'zod' import { checkHybridAuth } from '@/lib/auth/hybrid' import { createLogger } from '@/lib/logs/console/logger' -import { downloadFileFromStorage, processSingleFileToUserFile } from '@/lib/uploads/file-processing' +import { + downloadFileFromStorage, + processSingleFileToUserFile, +} from '@/lib/uploads/utils/file-processing' import { generateRequestId } from '@/lib/utils' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/outlook/draft/route.ts b/apps/sim/app/api/tools/outlook/draft/route.ts index 3d141287e..8a58e14b3 100644 --- a/apps/sim/app/api/tools/outlook/draft/route.ts +++ b/apps/sim/app/api/tools/outlook/draft/route.ts @@ -2,7 +2,10 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkHybridAuth } from '@/lib/auth/hybrid' import { createLogger } from '@/lib/logs/console/logger' -import { downloadFileFromStorage, processFilesToUserFiles } from '@/lib/uploads/file-processing' +import { + downloadFileFromStorage, + processFilesToUserFiles, +} from '@/lib/uploads/utils/file-processing' import { generateRequestId } from '@/lib/utils' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/outlook/send/route.ts b/apps/sim/app/api/tools/outlook/send/route.ts index 7ff3f7bd1..e6104df92 100644 --- a/apps/sim/app/api/tools/outlook/send/route.ts +++ b/apps/sim/app/api/tools/outlook/send/route.ts @@ -2,7 +2,10 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkHybridAuth } from '@/lib/auth/hybrid' import { createLogger } from '@/lib/logs/console/logger' -import { downloadFileFromStorage, processFilesToUserFiles } from '@/lib/uploads/file-processing' +import { + downloadFileFromStorage, + processFilesToUserFiles, +} from '@/lib/uploads/utils/file-processing' import { generateRequestId } from '@/lib/utils' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/s3/put-object/route.ts b/apps/sim/app/api/tools/s3/put-object/route.ts index c08dc0f3a..9ea198004 100644 --- a/apps/sim/app/api/tools/s3/put-object/route.ts +++ b/apps/sim/app/api/tools/s3/put-object/route.ts @@ -3,7 +3,10 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkHybridAuth } from '@/lib/auth/hybrid' import { createLogger } from '@/lib/logs/console/logger' -import { downloadFileFromStorage, processSingleFileToUserFile } from '@/lib/uploads/file-processing' +import { + downloadFileFromStorage, + processSingleFileToUserFile, +} from '@/lib/uploads/utils/file-processing' import { generateRequestId } from '@/lib/utils' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/sharepoint/upload/route.ts b/apps/sim/app/api/tools/sharepoint/upload/route.ts index 796877349..57b9a3848 100644 --- a/apps/sim/app/api/tools/sharepoint/upload/route.ts +++ b/apps/sim/app/api/tools/sharepoint/upload/route.ts @@ -2,7 +2,10 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkHybridAuth } from '@/lib/auth/hybrid' import { createLogger } from '@/lib/logs/console/logger' -import { downloadFileFromStorage, processFilesToUserFiles } from '@/lib/uploads/file-processing' +import { + downloadFileFromStorage, + processFilesToUserFiles, +} from '@/lib/uploads/utils/file-processing' import { generateRequestId } from '@/lib/utils' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/slack/send-message/route.ts b/apps/sim/app/api/tools/slack/send-message/route.ts index c7b213e4f..0eeead68c 100644 --- a/apps/sim/app/api/tools/slack/send-message/route.ts +++ b/apps/sim/app/api/tools/slack/send-message/route.ts @@ -2,7 +2,10 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkHybridAuth } from '@/lib/auth/hybrid' import { createLogger } from '@/lib/logs/console/logger' -import { downloadFileFromStorage, processFilesToUserFiles } from '@/lib/uploads/file-processing' +import { + downloadFileFromStorage, + processFilesToUserFiles, +} from '@/lib/uploads/utils/file-processing' import { generateRequestId } from '@/lib/utils' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/telegram/send-document/route.ts b/apps/sim/app/api/tools/telegram/send-document/route.ts index 38a93bca0..a556a7aa2 100644 --- a/apps/sim/app/api/tools/telegram/send-document/route.ts +++ b/apps/sim/app/api/tools/telegram/send-document/route.ts @@ -2,7 +2,10 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkHybridAuth } from '@/lib/auth/hybrid' import { createLogger } from '@/lib/logs/console/logger' -import { downloadFileFromStorage, processFilesToUserFiles } from '@/lib/uploads/file-processing' +import { + downloadFileFromStorage, + processFilesToUserFiles, +} from '@/lib/uploads/utils/file-processing' import { generateRequestId } from '@/lib/utils' import { convertMarkdownToHTML } from '@/tools/telegram/utils' diff --git a/apps/sim/app/api/tools/vision/analyze/route.ts b/apps/sim/app/api/tools/vision/analyze/route.ts index 18a93e9e2..b807ddd3f 100644 --- a/apps/sim/app/api/tools/vision/analyze/route.ts +++ b/apps/sim/app/api/tools/vision/analyze/route.ts @@ -2,7 +2,10 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { checkHybridAuth } from '@/lib/auth/hybrid' import { createLogger } from '@/lib/logs/console/logger' -import { downloadFileFromStorage, processSingleFileToUserFile } from '@/lib/uploads/file-processing' +import { + downloadFileFromStorage, + processSingleFileToUserFile, +} from '@/lib/uploads/utils/file-processing' import { generateRequestId } from '@/lib/utils' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/users/me/profile/route.ts b/apps/sim/app/api/users/me/profile/route.ts index 074ca0add..364ad230c 100644 --- a/apps/sim/app/api/users/me/profile/route.ts +++ b/apps/sim/app/api/users/me/profile/route.ts @@ -9,11 +9,18 @@ import { generateRequestId } from '@/lib/utils' const logger = createLogger('UpdateUserProfileAPI') -// Schema for updating user profile const UpdateProfileSchema = z .object({ name: z.string().min(1, 'Name is required').optional(), - image: z.string().url('Invalid image URL').optional(), + image: z + .string() + .refine( + (val) => { + return val.startsWith('http://') || val.startsWith('https://') || val.startsWith('/api/') + }, + { message: 'Invalid image URL' } + ) + .optional(), }) .refine((data) => data.name !== undefined || data.image !== undefined, { message: 'At least one field (name or image) must be provided', @@ -43,12 +50,10 @@ export async function PATCH(request: NextRequest) { const validatedData = UpdateProfileSchema.parse(body) - // Build update object const updateData: UpdateData = { updatedAt: new Date() } if (validatedData.name !== undefined) updateData.name = validatedData.name if (validatedData.image !== undefined) updateData.image = validatedData.image - // Update user profile const [updatedUser] = await db .update(user) .set(updateData) diff --git a/apps/sim/app/api/workspaces/[id]/files/[fileId]/download/route.ts b/apps/sim/app/api/workspaces/[id]/files/[fileId]/download/route.ts index df94528c1..8c14e15cb 100644 --- a/apps/sim/app/api/workspaces/[id]/files/[fileId]/download/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/[fileId]/download/route.ts @@ -1,9 +1,8 @@ import { type NextRequest, NextResponse } from 'next/server' import { getSession } from '@/lib/auth' import { createLogger } from '@/lib/logs/console/logger' -import { getPresignedUrlWithConfig, USE_BLOB_STORAGE, USE_S3_STORAGE } from '@/lib/uploads' -import { BLOB_CONFIG, S3_CONFIG } from '@/lib/uploads/setup' -import { getWorkspaceFile } from '@/lib/uploads/workspace-files' +import { StorageService } from '@/lib/uploads' +import { getWorkspaceFile } from '@/lib/uploads/contexts/workspace' import { generateRequestId } from '@/lib/utils' import { verifyWorkspaceMembership } from '@/app/api/workflows/utils' @@ -43,32 +42,12 @@ export async function POST( return NextResponse.json({ error: 'File not found' }, { status: 404 }) } - // Generate 5-minute presigned URL (same pattern as execution files) - let downloadUrl: string - - if (USE_S3_STORAGE) { - downloadUrl = await getPresignedUrlWithConfig( - fileRecord.key, - { - bucket: S3_CONFIG.bucket, - region: S3_CONFIG.region, - }, - 5 * 60 // 5 minutes - ) - } else if (USE_BLOB_STORAGE) { - downloadUrl = await getPresignedUrlWithConfig( - fileRecord.key, - { - accountName: BLOB_CONFIG.accountName, - accountKey: BLOB_CONFIG.accountKey, - connectionString: BLOB_CONFIG.connectionString, - containerName: BLOB_CONFIG.containerName, - }, - 5 * 60 // 5 minutes - ) - } else { - throw new Error('No cloud storage configured') - } + // Generate 5-minute presigned URL using unified storage service + const downloadUrl = await StorageService.generatePresignedDownloadUrl( + fileRecord.key, + 'workspace', + 5 * 60 // 5 minutes + ) logger.info(`[${requestId}] Generated download URL for workspace file: ${fileRecord.name}`) diff --git a/apps/sim/app/api/workspaces/[id]/files/[fileId]/route.ts b/apps/sim/app/api/workspaces/[id]/files/[fileId]/route.ts index be0e3e71e..7b0d40940 100644 --- a/apps/sim/app/api/workspaces/[id]/files/[fileId]/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/[fileId]/route.ts @@ -2,7 +2,7 @@ import { type NextRequest, NextResponse } from 'next/server' import { getSession } from '@/lib/auth' import { createLogger } from '@/lib/logs/console/logger' import { getUserEntityPermissions } from '@/lib/permissions/utils' -import { deleteWorkspaceFile } from '@/lib/uploads/workspace-files' +import { deleteWorkspaceFile } from '@/lib/uploads/contexts/workspace' import { generateRequestId } from '@/lib/utils' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/workspaces/[id]/files/route.ts b/apps/sim/app/api/workspaces/[id]/files/route.ts index ba8d6448c..6c971246f 100644 --- a/apps/sim/app/api/workspaces/[id]/files/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/route.ts @@ -2,7 +2,7 @@ import { type NextRequest, NextResponse } from 'next/server' import { getSession } from '@/lib/auth' import { createLogger } from '@/lib/logs/console/logger' import { getUserEntityPermissions } from '@/lib/permissions/utils' -import { listWorkspaceFiles, uploadWorkspaceFile } from '@/lib/uploads/workspace-files' +import { listWorkspaceFiles, uploadWorkspaceFile } from '@/lib/uploads/contexts/workspace' import { generateRequestId } from '@/lib/utils' import { verifyWorkspaceMembership } from '@/app/api/workflows/utils' diff --git a/apps/sim/app/chat/components/input/input.tsx b/apps/sim/app/chat/components/input/input.tsx index 7b2affd41..9cce11a17 100644 --- a/apps/sim/app/chat/components/input/input.tsx +++ b/apps/sim/app/chat/components/input/input.tsx @@ -357,6 +357,7 @@ export const ChatInput: React.FC<{ ref={fileInputRef} type='file' multiple + accept='.pdf,.csv,.doc,.docx,.txt,.md,.xlsx,.xls,.html,.htm,.pptx,.ppt,.json,.xml,.rtf,image/*' onChange={(e) => { handleFileSelect(e.target.files) if (fileInputRef.current) { diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/upload-modal/upload-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/upload-modal/upload-modal.tsx index d1e3455f7..5047de9d4 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/upload-modal/upload-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/upload-modal/upload-modal.tsx @@ -7,7 +7,11 @@ import { Dialog, DialogContent, DialogHeader, DialogTitle } from '@/components/u import { Label } from '@/components/ui/label' import { Progress } from '@/components/ui/progress' import { createLogger } from '@/lib/logs/console/logger' -import { ACCEPT_ATTRIBUTE, ACCEPTED_FILE_TYPES, MAX_FILE_SIZE } from '@/lib/uploads/validation' +import { + ACCEPT_ATTRIBUTE, + ACCEPTED_FILE_TYPES, + MAX_FILE_SIZE, +} from '@/lib/uploads/utils/validation' import { getDocumentIcon } from '@/app/workspace/[workspaceId]/knowledge/components' import { useKnowledgeUpload } from '@/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload' diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/components/create-modal/create-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/components/create-modal/create-modal.tsx index 67ccaa0e4..98db10458 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/components/create-modal/create-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/components/create-modal/create-modal.tsx @@ -14,7 +14,11 @@ import { Label } from '@/components/ui/label' import { Progress } from '@/components/ui/progress' import { Textarea } from '@/components/ui/textarea' import { createLogger } from '@/lib/logs/console/logger' -import { ACCEPT_ATTRIBUTE, ACCEPTED_FILE_TYPES, MAX_FILE_SIZE } from '@/lib/uploads/validation' +import { + ACCEPT_ATTRIBUTE, + ACCEPTED_FILE_TYPES, + MAX_FILE_SIZE, +} from '@/lib/uploads/utils/validation' import { getDocumentIcon } from '@/app/workspace/[workspaceId]/knowledge/components' import { useKnowledgeUpload } from '@/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload' import type { KnowledgeBaseData } from '@/stores/knowledge/store' diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/control-bar/components/api-key-selector/api-key-selector.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/control-bar/components/api-key-selector/api-key-selector.tsx index fb18b657e..185d45a7f 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/control-bar/components/api-key-selector/api-key-selector.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/control-bar/components/api-key-selector/api-key-selector.tsx @@ -415,53 +415,47 @@ export function ApiKeySelector({ {/* New Key Dialog */} - + { + setShowNewKeyDialog(open) + if (!open) { + setNewKey(null) + setCopySuccess(false) + if (justCreatedKeyId) { + onChange(justCreatedKeyId) + setJustCreatedKeyId(null) + } + } + }} + > - API Key Created Successfully + Your API key has been created - Your new API key has been created. Make sure to copy it now as you won't be able to - see it again. + This is the only time you will see your API key.{' '} + Copy it now and store it securely. -
- -
- + {newKey && ( +
+
+ + {newKey.key} + +
-
- - - { - setShowNewKeyDialog(false) - setNewKey(null) - setCopySuccess(false) - // Auto-select the newly created key - if (justCreatedKeyId) { - onChange(justCreatedKeyId) - setJustCreatedKeyId(null) - } - }} - > - Done - - + )} diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/chat/chat.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/chat/chat.tsx index 5ec720eae..437a6c2f7 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/chat/chat.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/chat/chat.tsx @@ -794,6 +794,7 @@ export function Chat({ chatMessage, setChatMessage }: ChatProps) { id='chat-file-input' type='file' multiple + accept='.pdf,.csv,.doc,.docx,.txt,.md,.xlsx,.xls,.html,.htm,.pptx,.ppt,.json,.xml,.rtf,image/*' onChange={(e) => { const files = e.target.files if (!files) return diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/components/file-upload.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/components/file-upload.tsx index 3e1bd9122..593749474 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/components/file-upload.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/components/file-upload.tsx @@ -17,7 +17,7 @@ import { import { Button } from '@/components/ui/button' import { Progress } from '@/components/ui/progress' import { createLogger } from '@/lib/logs/console/logger' -import type { WorkspaceFileRecord } from '@/lib/uploads/workspace-files' +import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { useSubBlockValue } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/hooks/use-sub-block-value' import { useWorkflowRegistry } from '@/stores/workflows/registry/store' import { useWorkflowStore } from '@/stores/workflows/workflow/store' diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/settings-modal/components/account/account.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/settings-modal/components/account/account.tsx index 69b38d430..9f2b8cfa9 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/settings-modal/components/account/account.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/settings-modal/components/account/account.tsx @@ -305,7 +305,9 @@ export function Account(_props: AccountProps) { alt={name || 'User'} width={48} height={48} - className='h-full w-full object-cover' + className={`h-full w-full object-cover transition-opacity duration-300 ${ + isUploadingProfilePicture ? 'opacity-50' : 'opacity-100' + }`} /> ) : ( @@ -313,7 +315,13 @@ export function Account(_props: AccountProps) { })()} {/* Upload overlay */} -
+
{isUploadingProfilePicture ? (
) : ( diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/settings-modal/components/account/hooks/use-profile-picture-upload.ts b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/settings-modal/components/account/hooks/use-profile-picture-upload.ts index c53a55481..52414e374 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/settings-modal/components/account/hooks/use-profile-picture-upload.ts +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/settings-modal/components/account/hooks/use-profile-picture-upload.ts @@ -65,31 +65,33 @@ export function useProfilePictureUpload({ logger.info('Presigned URL response:', presignedData) - const uploadHeaders: Record = { - 'Content-Type': file.type, + if (presignedData.directUploadSupported && presignedData.presignedUrl) { + const uploadHeaders: Record = { + 'Content-Type': file.type, + } + + if (presignedData.uploadHeaders) { + Object.assign(uploadHeaders, presignedData.uploadHeaders) + } + + const uploadResponse = await fetch(presignedData.presignedUrl, { + method: 'PUT', + body: file, + headers: uploadHeaders, + }) + + logger.info(`Upload response status: ${uploadResponse.status}`) + + if (!uploadResponse.ok) { + const responseText = await uploadResponse.text() + logger.error(`Direct upload failed: ${uploadResponse.status} - ${responseText}`) + throw new Error(`Direct upload failed: ${uploadResponse.status} - ${responseText}`) + } + + const publicUrl = presignedData.fileInfo.path + logger.info(`Profile picture uploaded successfully via direct upload: ${publicUrl}`) + return publicUrl } - - if (presignedData.uploadHeaders) { - Object.assign(uploadHeaders, presignedData.uploadHeaders) - } - - const uploadResponse = await fetch(presignedData.uploadUrl, { - method: 'PUT', - body: file, - headers: uploadHeaders, - }) - - logger.info(`Upload response status: ${uploadResponse.status}`) - - if (!uploadResponse.ok) { - const responseText = await uploadResponse.text() - logger.error(`Direct upload failed: ${uploadResponse.status} - ${responseText}`) - throw new Error(`Direct upload failed: ${uploadResponse.status} - ${responseText}`) - } - - const publicUrl = presignedData.fileInfo.path - logger.info(`Profile picture uploaded successfully via direct upload: ${publicUrl}`) - return publicUrl } const formData = new FormData() diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/settings-modal/components/api-keys/api-keys.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/settings-modal/components/api-keys/api-keys.tsx index 38e076d98..27cbeeefb 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/settings-modal/components/api-keys/api-keys.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/settings-modal/components/api-keys/api-keys.tsx @@ -233,21 +233,29 @@ export function ApiKeys({ onOpenChange, registerCloseHandler }: ApiKeysProps) { ? `/api/workspaces/${workspaceId}/api-keys/${deleteKey.id}` : `/api/users/me/api-keys/${deleteKey.id}` + if (isWorkspaceKey) { + setWorkspaceKeys((prev) => prev.filter((k) => k.id !== deleteKey.id)) + } else { + setPersonalKeys((prev) => prev.filter((k) => k.id !== deleteKey.id)) + setConflicts((prev) => prev.filter((name) => name !== deleteKey.name)) + } + + setShowDeleteDialog(false) + setDeleteKey(null) + setDeleteConfirmationName('') + const response = await fetch(url, { method: 'DELETE', }) - if (response.ok) { - fetchApiKeys() - setShowDeleteDialog(false) - setDeleteKey(null) - setDeleteConfirmationName('') - } else { + if (!response.ok) { const errorData = await response.json() logger.error('Failed to delete API key:', errorData) + fetchApiKeys() } } catch (error) { logger.error('Error deleting API key:', { error }) + fetchApiKeys() } } diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/settings-modal/components/file-uploads/file-uploads.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/settings-modal/components/file-uploads/file-uploads.tsx index d0dd9e257..af0935c3a 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/settings-modal/components/file-uploads/file-uploads.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/settings-modal/components/file-uploads/file-uploads.tsx @@ -14,8 +14,8 @@ import { TableRow, } from '@/components/ui/table' import { createLogger } from '@/lib/logs/console/logger' -import { getFileExtension } from '@/lib/uploads/file-utils' -import type { WorkspaceFileRecord } from '@/lib/uploads/workspace-files' +import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' +import { getFileExtension } from '@/lib/uploads/utils/file-utils' import { cn } from '@/lib/utils' import { getDocumentIcon } from '@/app/workspace/[workspaceId]/knowledge/components' import { useUserPermissions } from '@/hooks/use-user-permissions' @@ -199,18 +199,36 @@ export function FileUploads() { try { setDeletingFileId(file.id) + const previousFiles = files + const previousStorageInfo = storageInfo + + setFiles((prev) => prev.filter((f) => f.id !== file.id)) + + if (storageInfo) { + const newUsedBytes = Math.max(0, storageInfo.usedBytes - file.size) + const newPercentUsed = (newUsedBytes / storageInfo.limitBytes) * 100 + setStorageInfo({ + ...storageInfo, + usedBytes: newUsedBytes, + percentUsed: newPercentUsed, + }) + } + const response = await fetch(`/api/workspaces/${workspaceId}/files/${file.id}`, { method: 'DELETE', }) const data = await response.json() - if (data.success) { - await loadFiles() - await loadStorageInfo() + if (!data.success) { + setFiles(previousFiles) + setStorageInfo(previousStorageInfo) + logger.error('Failed to delete file:', data.error) } } catch (error) { logger.error('Error deleting file:', error) + await loadFiles() + await loadStorageInfo() } finally { setDeletingFileId(null) } diff --git a/apps/sim/blocks/blocks/file.ts b/apps/sim/blocks/blocks/file.ts index 9cba3db63..4a1711071 100644 --- a/apps/sim/blocks/blocks/file.ts +++ b/apps/sim/blocks/blocks/file.ts @@ -45,7 +45,8 @@ export const FileBlock: BlockConfig = { title: 'Process Files', type: 'file-upload' as SubBlockType, layout: 'full' as SubBlockLayout, - acceptedTypes: '.pdf,.csv,.doc,.docx,.txt,.md,.xlsx,.xls,.html,.htm,.pptx,.ppt', + acceptedTypes: + '.pdf,.csv,.doc,.docx,.txt,.md,.xlsx,.xls,.html,.htm,.pptx,.ppt,.json,.xml,.rtf', multiple: true, condition: { field: 'inputMethod', diff --git a/apps/sim/executor/types.ts b/apps/sim/executor/types.ts index efbcddd50..ea7c46e51 100644 --- a/apps/sim/executor/types.ts +++ b/apps/sim/executor/types.ts @@ -14,6 +14,7 @@ export interface UserFile { key: string uploadedAt: string expiresAt: string + context?: string } /** diff --git a/apps/sim/executor/utils/file-tool-processor.ts b/apps/sim/executor/utils/file-tool-processor.ts index b6d4d2735..f910e69f1 100644 --- a/apps/sim/executor/utils/file-tool-processor.ts +++ b/apps/sim/executor/utils/file-tool-processor.ts @@ -1,5 +1,5 @@ import { createLogger } from '@/lib/logs/console/logger' -import { uploadExecutionFile } from '@/lib/workflows/execution-file-storage' +import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' import type { ExecutionContext, UserFile } from '@/executor/types' import type { ToolConfig, ToolFileData } from '@/tools/types' diff --git a/apps/sim/lib/execution/files.ts b/apps/sim/lib/execution/files.ts index adb0c013c..815bbdc89 100644 --- a/apps/sim/lib/execution/files.ts +++ b/apps/sim/lib/execution/files.ts @@ -1,6 +1,6 @@ import { v4 as uuidv4 } from 'uuid' import { createLogger } from '@/lib/logs/console/logger' -import { uploadExecutionFile } from '@/lib/workflows/execution-file-storage' +import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' import type { UserFile } from '@/executor/types' const logger = createLogger('ExecutionFiles') diff --git a/apps/sim/lib/knowledge/documents/document-processor.ts b/apps/sim/lib/knowledge/documents/document-processor.ts index 3ad948f45..438b9a053 100644 --- a/apps/sim/lib/knowledge/documents/document-processor.ts +++ b/apps/sim/lib/knowledge/documents/document-processor.ts @@ -3,13 +3,7 @@ import { env } from '@/lib/env' import { parseBuffer, parseFile } from '@/lib/file-parsers' import { retryWithExponentialBackoff } from '@/lib/knowledge/documents/utils' import { createLogger } from '@/lib/logs/console/logger' -import { - type CustomStorageConfig, - getPresignedUrlWithConfig, - getStorageProvider, - uploadFile, -} from '@/lib/uploads' -import { BLOB_KB_CONFIG, S3_KB_CONFIG } from '@/lib/uploads/setup' +import { StorageService } from '@/lib/uploads' import { mistralParserTool } from '@/tools/mistral/parser' const logger = createLogger('DocumentProcessor') @@ -45,21 +39,6 @@ type AzureOCRResponse = { [key: string]: unknown } -const getKBConfig = (): CustomStorageConfig => { - const provider = getStorageProvider() - return provider === 'blob' - ? { - containerName: BLOB_KB_CONFIG.containerName, - accountName: BLOB_KB_CONFIG.accountName, - accountKey: BLOB_KB_CONFIG.accountKey, - connectionString: BLOB_KB_CONFIG.connectionString, - } - : { - bucket: S3_KB_CONFIG.bucket, - region: S3_KB_CONFIG.region, - } -} - class APIError extends Error { public status: number @@ -189,13 +168,21 @@ async function handleFileForOCR(fileUrl: string, filename: string, mimeType: str logger.info(`Uploading "${filename}" to cloud storage for OCR`) const buffer = await downloadFileWithTimeout(fileUrl) - const kbConfig = getKBConfig() - - validateCloudConfig(kbConfig) try { - const cloudResult = await uploadFile(buffer, filename, mimeType, kbConfig) - const httpsUrl = await getPresignedUrlWithConfig(cloudResult.key, kbConfig, 900) + const cloudResult = await StorageService.uploadFile({ + file: buffer, + fileName: filename, + contentType: mimeType, + context: 'knowledge-base', + }) + + const httpsUrl = await StorageService.generatePresignedDownloadUrl( + cloudResult.key, + 'knowledge-base', + 900 // 15 minutes + ) + logger.info(`Successfully uploaded for OCR: ${cloudResult.key}`) return { httpsUrl, cloudUrl: httpsUrl } } catch (uploadError) { @@ -250,25 +237,6 @@ async function downloadFileForBase64(fileUrl: string): Promise { return fs.readFile(fileUrl) } -function validateCloudConfig(kbConfig: CustomStorageConfig) { - const provider = getStorageProvider() - - if (provider === 'blob') { - if ( - !kbConfig.containerName || - (!kbConfig.connectionString && (!kbConfig.accountName || !kbConfig.accountKey)) - ) { - throw new Error( - 'Azure Blob configuration missing. Set AZURE_CONNECTION_STRING or AZURE_ACCOUNT_NAME + AZURE_ACCOUNT_KEY + AZURE_KB_CONTAINER_NAME' - ) - } - } else { - if (!kbConfig.bucket || !kbConfig.region) { - throw new Error('S3 configuration missing. Set AWS_REGION and S3_KB_BUCKET_NAME') - } - } -} - function processOCRContent(result: OCRResult, filename: string): string { if (!result.success) { throw new Error(`OCR processing failed: ${result.error || 'Unknown error'}`) diff --git a/apps/sim/lib/uploads/contexts/chat/chat-file-manager.ts b/apps/sim/lib/uploads/contexts/chat/chat-file-manager.ts new file mode 100644 index 000000000..e19d52aed --- /dev/null +++ b/apps/sim/lib/uploads/contexts/chat/chat-file-manager.ts @@ -0,0 +1,82 @@ +import { processExecutionFiles } from '@/lib/execution/files' +import { createLogger } from '@/lib/logs/console/logger' +import type { UserFile } from '@/executor/types' + +const logger = createLogger('ChatFileManager') + +export interface ChatFile { + dataUrl?: string // Base64-encoded file data (data:mime;base64,...) + url?: string // Direct URL to existing file + name: string // Original filename + type: string // MIME type +} + +export interface ChatExecutionContext { + workspaceId: string + workflowId: string + executionId: string +} + +/** + * Process and upload chat files to temporary execution storage + * + * Handles two input formats: + * 1. Base64 dataUrl - File content encoded as data URL (uploaded from client) + * 2. Direct URL - Pass-through URL to existing file (already uploaded) + * + * Files are stored in the execution context with 5-10 minute expiry. + * + * @param files Array of chat file attachments + * @param executionContext Execution context for temporary storage + * @param requestId Unique request identifier for logging/tracing + * @returns Array of UserFile objects with upload results + */ +export async function processChatFiles( + files: ChatFile[], + executionContext: ChatExecutionContext, + requestId: string +): Promise { + logger.info( + `Processing ${files.length} chat files for execution ${executionContext.executionId}`, + { + requestId, + executionContext, + } + ) + + const transformedFiles = files.map((file) => ({ + type: file.dataUrl ? ('file' as const) : ('url' as const), + data: file.dataUrl || file.url || '', + name: file.name, + mime: file.type, + })) + + const userFiles = await processExecutionFiles(transformedFiles, executionContext, requestId) + + logger.info(`Successfully processed ${userFiles.length} chat files`, { + requestId, + executionId: executionContext.executionId, + }) + + return userFiles +} + +/** + * Upload a single chat file to temporary execution storage + * + * This is a convenience function for uploading individual files. + * For batch uploads, use processChatFiles() for better performance. + * + * @param file Chat file to upload + * @param executionContext Execution context for temporary storage + * @param requestId Unique request identifier + * @returns UserFile object with upload result + */ +export async function uploadChatFile( + file: ChatFile, + executionContext: ChatExecutionContext, + requestId: string +): Promise { + const [userFile] = await processChatFiles([file], executionContext, requestId) + return userFile +} diff --git a/apps/sim/lib/uploads/contexts/chat/index.ts b/apps/sim/lib/uploads/contexts/chat/index.ts new file mode 100644 index 000000000..7a317a565 --- /dev/null +++ b/apps/sim/lib/uploads/contexts/chat/index.ts @@ -0,0 +1,6 @@ +export { + type ChatExecutionContext, + type ChatFile, + processChatFiles, + uploadChatFile, +} from './chat-file-manager' diff --git a/apps/sim/lib/uploads/contexts/copilot/copilot-file-manager.ts b/apps/sim/lib/uploads/contexts/copilot/copilot-file-manager.ts new file mode 100644 index 000000000..d1f1db030 --- /dev/null +++ b/apps/sim/lib/uploads/contexts/copilot/copilot-file-manager.ts @@ -0,0 +1,199 @@ +import { createLogger } from '@/lib/logs/console/logger' +import { + deleteFile, + downloadFile, + generatePresignedDownloadUrl, + generatePresignedUploadUrl, + type PresignedUrlResponse, +} from '@/lib/uploads/core/storage-service' + +const logger = createLogger('CopilotFileManager') + +const SUPPORTED_IMAGE_TYPES = [ + 'image/jpeg', + 'image/jpg', + 'image/png', + 'image/gif', + 'image/webp', + 'image/svg+xml', +] + +/** + * Check if a file type is a supported image format for copilot + */ +export function isSupportedFileType(mimeType: string): boolean { + return SUPPORTED_IMAGE_TYPES.includes(mimeType.toLowerCase()) +} + +/** + * Check if a content type is an image + */ +export function isImageFileType(contentType: string): boolean { + return contentType.toLowerCase().startsWith('image/') +} + +export interface CopilotFileAttachment { + key: string + filename: string + media_type: string +} + +export interface GenerateCopilotUploadUrlOptions { + fileName: string + contentType: string + fileSize: number + userId: string + expirationSeconds?: number +} + +/** + * Generate a presigned URL for copilot file upload + * + * Only image files are allowed for copilot uploads. + * Requires authenticated user session. + * + * @param options Upload URL generation options + * @returns Presigned URL response with upload URL and file key + * @throws Error if file type is not an image or user is not authenticated + */ +export async function generateCopilotUploadUrl( + options: GenerateCopilotUploadUrlOptions +): Promise { + const { fileName, contentType, fileSize, userId, expirationSeconds = 3600 } = options + + logger.info(`Generating copilot upload URL for: ${fileName}`, { + userId, + contentType, + fileSize, + }) + + if (!userId?.trim()) { + throw new Error('Authenticated user session is required for copilot uploads') + } + + if (!isImageFileType(contentType)) { + throw new Error('Only image files (JPEG, PNG, GIF, WebP, SVG) are allowed for copilot uploads') + } + + const presignedUrlResponse = await generatePresignedUploadUrl({ + fileName, + contentType, + fileSize, + context: 'copilot', + userId, + expirationSeconds, + }) + + logger.info(`Generated copilot upload URL for: ${fileName}`, { + key: presignedUrlResponse.key, + userId, + }) + + return presignedUrlResponse +} + +/** + * Download a copilot file from storage + * + * Uses the unified storage service with explicit copilot context. + * Handles S3, Azure Blob, and local storage automatically. + * + * @param key File storage key + * @returns File buffer + * @throws Error if file not found or download fails + */ +export async function downloadCopilotFile(key: string): Promise { + logger.info(`Downloading copilot file: ${key}`) + + try { + const fileBuffer = await downloadFile({ + key, + context: 'copilot', + }) + + logger.info(`Successfully downloaded copilot file: ${key}`, { + size: fileBuffer.length, + }) + + return fileBuffer + } catch (error) { + logger.error(`Failed to download copilot file: ${key}`, error) + throw error + } +} + +/** + * Process copilot file attachments for chat messages + * + * Downloads files from storage and validates they are supported types. + * Skips unsupported files with a warning. + * + * @param attachments Array of file attachments + * @param requestId Request identifier for logging + * @returns Array of buffers for successfully downloaded files + */ +export async function processCopilotAttachments( + attachments: CopilotFileAttachment[], + requestId: string +): Promise> { + logger.info(`Processing ${attachments.length} copilot attachments`, { requestId }) + + const results: Array<{ buffer: Buffer; attachment: CopilotFileAttachment }> = [] + + for (const attachment of attachments) { + try { + if (!isSupportedFileType(attachment.media_type)) { + logger.warn(`[${requestId}] Unsupported file type: ${attachment.media_type}`) + continue + } + + const buffer = await downloadCopilotFile(attachment.key) + + results.push({ buffer, attachment }) + } catch (error) { + logger.error(`[${requestId}] Failed to process file ${attachment.filename}:`, error) + } + } + + logger.info(`Successfully processed ${results.length}/${attachments.length} attachments`, { + requestId, + }) + + return results +} + +/** + * Generate a presigned download URL for a copilot file + * + * @param key File storage key + * @param expirationSeconds Time in seconds until URL expires (default: 1 hour) + * @returns Presigned download URL + */ +export async function generateCopilotDownloadUrl( + key: string, + expirationSeconds = 3600 +): Promise { + logger.info(`Generating copilot download URL for: ${key}`) + + const downloadUrl = await generatePresignedDownloadUrl(key, 'copilot', expirationSeconds) + + logger.info(`Generated copilot download URL for: ${key}`) + + return downloadUrl +} + +/** + * Delete a copilot file from storage + * + * @param key File storage key + */ +export async function deleteCopilotFile(key: string): Promise { + logger.info(`Deleting copilot file: ${key}`) + + await deleteFile({ + key, + context: 'copilot', + }) + + logger.info(`Successfully deleted copilot file: ${key}`) +} diff --git a/apps/sim/lib/uploads/contexts/copilot/index.ts b/apps/sim/lib/uploads/contexts/copilot/index.ts new file mode 100644 index 000000000..bee1981b5 --- /dev/null +++ b/apps/sim/lib/uploads/contexts/copilot/index.ts @@ -0,0 +1,11 @@ +export { + type CopilotFileAttachment, + deleteCopilotFile, + downloadCopilotFile, + type GenerateCopilotUploadUrlOptions, + generateCopilotDownloadUrl, + generateCopilotUploadUrl, + isImageFileType, + isSupportedFileType, + processCopilotAttachments, +} from './copilot-file-manager' diff --git a/apps/sim/lib/workflows/execution-files.ts b/apps/sim/lib/uploads/contexts/execution/execution-file-helpers.ts similarity index 93% rename from apps/sim/lib/workflows/execution-files.ts rename to apps/sim/lib/uploads/contexts/execution/execution-file-helpers.ts index ecfc97f58..c5b674c93 100644 --- a/apps/sim/lib/workflows/execution-files.ts +++ b/apps/sim/lib/uploads/contexts/execution/execution-file-helpers.ts @@ -1,8 +1,3 @@ -/** - * Execution file management system for binary data transfer between blocks - * This handles file storage, retrieval, and cleanup for workflow executions - */ - import type { UserFile } from '@/executor/types' /** diff --git a/apps/sim/lib/uploads/contexts/execution/execution-file-manager.ts b/apps/sim/lib/uploads/contexts/execution/execution-file-manager.ts new file mode 100644 index 000000000..45bf36997 --- /dev/null +++ b/apps/sim/lib/uploads/contexts/execution/execution-file-manager.ts @@ -0,0 +1,163 @@ +import { createLogger } from '@/lib/logs/console/logger' +import { + deleteFile, + downloadFile, + generatePresignedDownloadUrl, + uploadFile, +} from '@/lib/uploads/core/storage-service' +import type { UserFile } from '@/executor/types' +import type { ExecutionContext } from './execution-file-helpers' +import { + generateExecutionFileKey, + generateFileId, + getFileExpirationDate, +} from './execution-file-helpers' + +const logger = createLogger('ExecutionFileStorage') + +/** + * Upload a file to execution-scoped storage + */ +export async function uploadExecutionFile( + context: ExecutionContext, + fileBuffer: Buffer, + fileName: string, + contentType: string, + isAsync?: boolean +): Promise { + logger.info(`Uploading execution file: ${fileName} for execution ${context.executionId}`) + logger.debug(`File upload context:`, { + workspaceId: context.workspaceId, + workflowId: context.workflowId, + executionId: context.executionId, + fileName, + bufferSize: fileBuffer.length, + }) + + const storageKey = generateExecutionFileKey(context, fileName) + const fileId = generateFileId() + + logger.info(`Generated storage key: "${storageKey}" for file: ${fileName}`) + + const urlExpirationSeconds = isAsync ? 10 * 60 : 5 * 60 + + try { + const fileInfo = await uploadFile({ + file: fileBuffer, + fileName: storageKey, + contentType, + context: 'execution', + preserveKey: true, // Don't add timestamp prefix + customKey: storageKey, // Use exact execution-scoped key + }) + + logger.info(`Upload returned key: "${fileInfo.key}" for file: ${fileName}`) + logger.info(`Original storage key was: "${storageKey}"`) + logger.info(`Keys match: ${fileInfo.key === storageKey}`) + + let directUrl: string | undefined + + try { + logger.info( + `Generating presigned URL with key: "${fileInfo.key}" (expiration: ${urlExpirationSeconds / 60} minutes)` + ) + directUrl = await generatePresignedDownloadUrl( + fileInfo.key, + 'execution', + urlExpirationSeconds + ) + logger.info(`Generated presigned URL for execution file`) + } catch (error) { + logger.warn(`Failed to generate presigned URL for ${fileName}:`, error) + } + + const userFile: UserFile = { + id: fileId, + name: fileName, + size: fileBuffer.length, + type: contentType, + url: directUrl || `/api/files/serve/${fileInfo.key}`, // Use presigned URL (5 or 10 min), fallback to serve path + key: fileInfo.key, + uploadedAt: new Date().toISOString(), + expiresAt: getFileExpirationDate(), + context: 'execution', // Preserve context in file object + } + + logger.info(`Successfully uploaded execution file: ${fileName} (${fileBuffer.length} bytes)`) + return userFile + } catch (error) { + logger.error(`Failed to upload execution file ${fileName}:`, error) + throw new Error( + `Failed to upload file: ${error instanceof Error ? error.message : 'Unknown error'}` + ) + } +} + +/** + * Download a file from execution-scoped storage + */ +export async function downloadExecutionFile(userFile: UserFile): Promise { + logger.info(`Downloading execution file: ${userFile.name}`) + + try { + const fileBuffer = await downloadFile({ + key: userFile.key, + context: 'execution', + }) + + logger.info( + `Successfully downloaded execution file: ${userFile.name} (${fileBuffer.length} bytes)` + ) + return fileBuffer + } catch (error) { + logger.error(`Failed to download execution file ${userFile.name}:`, error) + throw new Error( + `Failed to download file: ${error instanceof Error ? error.message : 'Unknown error'}` + ) + } +} + +/** + * Generate a short-lived presigned URL for file download (5 minutes) + */ +export async function generateExecutionFileDownloadUrl(userFile: UserFile): Promise { + logger.info(`Generating download URL for execution file: ${userFile.name}`) + logger.info(`File key: "${userFile.key}"`) + + try { + const downloadUrl = await generatePresignedDownloadUrl( + userFile.key, + 'execution', + 5 * 60 // 5 minutes + ) + + logger.info(`Generated download URL for execution file: ${userFile.name}`) + return downloadUrl + } catch (error) { + logger.error(`Failed to generate download URL for ${userFile.name}:`, error) + throw new Error( + `Failed to generate download URL: ${error instanceof Error ? error.message : 'Unknown error'}` + ) + } +} + +/** + * Delete a file from execution-scoped storage + */ +export async function deleteExecutionFile(userFile: UserFile): Promise { + logger.info(`Deleting execution file: ${userFile.name}`) + + try { + await deleteFile({ + key: userFile.key, + context: 'execution', + }) + + logger.info(`Successfully deleted execution file: ${userFile.name}`) + } catch (error) { + logger.error(`Failed to delete execution file ${userFile.name}:`, error) + throw new Error( + `Failed to delete file: ${error instanceof Error ? error.message : 'Unknown error'}` + ) + } +} diff --git a/apps/sim/lib/workflows/execution-files-server.ts b/apps/sim/lib/uploads/contexts/execution/execution-file-server.ts similarity index 86% rename from apps/sim/lib/workflows/execution-files-server.ts rename to apps/sim/lib/uploads/contexts/execution/execution-file-server.ts index fe63f0868..fc206ff2c 100644 --- a/apps/sim/lib/workflows/execution-files-server.ts +++ b/apps/sim/lib/uploads/contexts/execution/execution-file-server.ts @@ -1,13 +1,8 @@ -/** - * Server-only execution file metadata management - * This file contains database operations and should only be imported by server-side code - */ - import { db } from '@sim/db' import { workflowExecutionLogs } from '@sim/db/schema' import { eq } from 'drizzle-orm' import { createLogger } from '@/lib/logs/console/logger' -import type { ExecutionFileMetadata } from './execution-files' +import type { ExecutionFileMetadata } from './execution-file-helpers' const logger = createLogger('ExecutionFilesServer') @@ -26,7 +21,6 @@ export async function getExecutionFiles(executionId: string): Promise { try { - // Get existing files const existingFiles = await getExecutionFiles(executionId) - // Add new file const updatedFiles = [...existingFiles, fileMetadata] - // Store updated files await storeExecutionFiles(executionId, updatedFiles) logger.info(`Added file ${fileMetadata.name} to execution ${executionId}`) @@ -87,11 +78,10 @@ export async function getExpiredFiles(): Promise { try { const now = new Date().toISOString() - // Query all execution logs that have files const logs = await db .select() .from(workflowExecutionLogs) - .where(eq(workflowExecutionLogs.level, 'info')) // Only get successful executions + .where(eq(workflowExecutionLogs.level, 'info')) const expiredFiles: ExecutionFileMetadata[] = [] @@ -118,7 +108,6 @@ export async function cleanupExpiredFileMetadata(): Promise { const now = new Date().toISOString() let cleanedCount = 0 - // Get all execution logs const logs = await db.select().from(workflowExecutionLogs) for (const log of logs) { @@ -127,7 +116,6 @@ export async function cleanupExpiredFileMetadata(): Promise { const nonExpiredFiles = files.filter((file) => file.expiresAt >= now) if (nonExpiredFiles.length !== files.length) { - // Some files expired, update the files column await db .update(workflowExecutionLogs) .set({ files: nonExpiredFiles.length > 0 ? nonExpiredFiles : null }) diff --git a/apps/sim/lib/uploads/contexts/execution/index.ts b/apps/sim/lib/uploads/contexts/execution/index.ts new file mode 100644 index 000000000..3b0da0c96 --- /dev/null +++ b/apps/sim/lib/uploads/contexts/execution/index.ts @@ -0,0 +1,3 @@ +export * from './execution-file-helpers' +export * from './execution-file-manager' +export * from './execution-file-server' diff --git a/apps/sim/lib/uploads/contexts/workspace/index.ts b/apps/sim/lib/uploads/contexts/workspace/index.ts new file mode 100644 index 000000000..4d2d50a4a --- /dev/null +++ b/apps/sim/lib/uploads/contexts/workspace/index.ts @@ -0,0 +1 @@ +export * from './workspace-file-manager' diff --git a/apps/sim/lib/uploads/workspace-files.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts similarity index 71% rename from apps/sim/lib/uploads/workspace-files.ts rename to apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts index 2f3145e60..0a0319cd0 100644 --- a/apps/sim/lib/uploads/workspace-files.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts @@ -12,7 +12,13 @@ import { incrementStorageUsage, } from '@/lib/billing/storage' import { createLogger } from '@/lib/logs/console/logger' -import { deleteFile, downloadFile } from '@/lib/uploads/storage-client' +import { + deleteFile, + downloadFile, + generatePresignedDownloadUrl, + hasCloudStorage, + uploadFile, +} from '@/lib/uploads/core/storage-service' import type { UserFile } from '@/executor/types' const logger = createLogger('WorkspaceFileStorage') @@ -53,70 +59,34 @@ export async function uploadWorkspaceFile( ): Promise { logger.info(`Uploading workspace file: ${fileName} for workspace ${workspaceId}`) - // Check for duplicates const exists = await fileExistsInWorkspace(workspaceId, fileName) if (exists) { throw new Error(`A file named "${fileName}" already exists in this workspace`) } - // Check storage quota const quotaCheck = await checkStorageQuota(userId, fileBuffer.length) if (!quotaCheck.allowed) { throw new Error(quotaCheck.error || 'Storage limit exceeded') } - // Generate workspace-scoped storage key const storageKey = generateWorkspaceFileKey(workspaceId, fileName) const fileId = `wf_${Date.now()}_${Math.random().toString(36).substring(2, 9)}` try { - let uploadResult: any - logger.info(`Generated storage key: ${storageKey}`) - // Upload to storage with skipTimestampPrefix to use exact key - const { USE_S3_STORAGE, USE_BLOB_STORAGE, S3_CONFIG, BLOB_CONFIG } = await import( - '@/lib/uploads/setup' - ) + const uploadResult = await uploadFile({ + file: fileBuffer, + fileName: storageKey, // Use the full storageKey as fileName + contentType, + context: 'workspace', + preserveKey: true, // Don't add timestamp prefix + customKey: storageKey, // Explicitly set the key + }) - if (USE_S3_STORAGE) { - const { uploadToS3 } = await import('@/lib/uploads/s3/s3-client') - // Use custom config overload with skipTimestampPrefix - uploadResult = await uploadToS3( - fileBuffer, - storageKey, - contentType, - { - bucket: S3_CONFIG.bucket, - region: S3_CONFIG.region, - }, - fileBuffer.length, - true // skipTimestampPrefix = true - ) - } else if (USE_BLOB_STORAGE) { - const { uploadToBlob } = await import('@/lib/uploads/blob/blob-client') - // Blob doesn't have skipTimestampPrefix, but we pass the full key - uploadResult = await uploadToBlob( - fileBuffer, - storageKey, - contentType, - { - accountName: BLOB_CONFIG.accountName, - accountKey: BLOB_CONFIG.accountKey, - connectionString: BLOB_CONFIG.connectionString, - containerName: BLOB_CONFIG.containerName, - }, - fileBuffer.length - ) - } else { - throw new Error('No storage provider configured') - } + logger.info(`Upload returned key: ${uploadResult.key}`) - logger.info(`S3/Blob upload returned key: ${uploadResult.key}`) - logger.info(`Keys match: ${uploadResult.key === storageKey}`) - - // Store metadata in database - use the EXACT key from upload result await db.insert(workspaceFile).values({ id: fileId, workspaceId, @@ -130,25 +100,26 @@ export async function uploadWorkspaceFile( logger.info(`Successfully uploaded workspace file: ${fileName} with key: ${uploadResult.key}`) - // Increment storage usage tracking try { await incrementStorageUsage(userId, fileBuffer.length) } catch (storageError) { logger.error(`Failed to update storage tracking:`, storageError) - // Continue - don't fail upload if tracking fails } - // Generate presigned URL (valid for 24 hours for initial access) - const { getPresignedUrl } = await import('@/lib/uploads') let presignedUrl: string | undefined - try { - presignedUrl = await getPresignedUrl(uploadResult.key, 24 * 60 * 60) // 24 hours - } catch (error) { - logger.warn(`Failed to generate presigned URL for ${fileName}:`, error) + if (hasCloudStorage()) { + try { + presignedUrl = await generatePresignedDownloadUrl( + uploadResult.key, + 'workspace', + 24 * 60 * 60 // 24 hours + ) + } catch (error) { + logger.warn(`Failed to generate presigned URL for ${fileName}:`, error) + } } - // Return UserFile format (no expiry for workspace files) return { id: fileId, name: fileName, @@ -158,6 +129,7 @@ export async function uploadWorkspaceFile( key: uploadResult.key, uploadedAt: new Date().toISOString(), expiresAt: new Date(Date.now() + 365 * 24 * 60 * 60 * 1000).toISOString(), // 1 year + context: 'workspace', } } catch (error) { logger.error(`Failed to upload workspace file ${fileName}:`, error) @@ -199,14 +171,12 @@ export async function listWorkspaceFiles(workspaceId: string): Promise ({ ...file, - path: `${pathPrefix}${encodeURIComponent(file.key)}`, - // url will be generated on-demand during execution for external APIs + path: `${pathPrefix}${encodeURIComponent(file.key)}?context=workspace`, })) } catch (error) { logger.error(`Failed to list workspace files for ${workspaceId}:`, error) @@ -230,13 +200,12 @@ export async function getWorkspaceFile( if (files.length === 0) return null - // Add full serve path const { getServePathPrefix } = await import('@/lib/uploads') const pathPrefix = getServePathPrefix() return { ...files[0], - path: `${pathPrefix}${encodeURIComponent(files[0].key)}`, + path: `${pathPrefix}${encodeURIComponent(files[0].key)}?context=workspace`, } } catch (error) { logger.error(`Failed to get workspace file ${fileId}:`, error) @@ -251,7 +220,10 @@ export async function downloadWorkspaceFile(fileRecord: WorkspaceFileRecord): Pr logger.info(`Downloading workspace file: ${fileRecord.name}`) try { - const buffer = await downloadFile(fileRecord.key) + const buffer = await downloadFile({ + key: fileRecord.key, + context: 'workspace', + }) logger.info( `Successfully downloaded workspace file: ${fileRecord.name} (${buffer.length} bytes)` ) @@ -271,26 +243,24 @@ export async function deleteWorkspaceFile(workspaceId: string, fileId: string): logger.info(`Deleting workspace file: ${fileId}`) try { - // Get file record first const fileRecord = await getWorkspaceFile(workspaceId, fileId) if (!fileRecord) { throw new Error('File not found') } - // Delete from storage - await deleteFile(fileRecord.key) + await deleteFile({ + key: fileRecord.key, + context: 'workspace', + }) - // Delete from database await db .delete(workspaceFile) .where(and(eq(workspaceFile.id, fileId), eq(workspaceFile.workspaceId, workspaceId))) - // Decrement storage usage tracking try { await decrementStorageUsage(fileRecord.uploadedBy, fileRecord.size) } catch (storageError) { logger.error(`Failed to update storage tracking:`, storageError) - // Continue - don't fail deletion if tracking fails } logger.info(`Successfully deleted workspace file: ${fileRecord.name}`) diff --git a/apps/sim/lib/uploads/core/config-resolver.ts b/apps/sim/lib/uploads/core/config-resolver.ts new file mode 100644 index 000000000..3294693f8 --- /dev/null +++ b/apps/sim/lib/uploads/core/config-resolver.ts @@ -0,0 +1,177 @@ +import { + BLOB_CHAT_CONFIG, + BLOB_CONFIG, + BLOB_COPILOT_CONFIG, + BLOB_EXECUTION_FILES_CONFIG, + BLOB_KB_CONFIG, + BLOB_PROFILE_PICTURES_CONFIG, + S3_CHAT_CONFIG, + S3_CONFIG, + S3_COPILOT_CONFIG, + S3_EXECUTION_FILES_CONFIG, + S3_KB_CONFIG, + S3_PROFILE_PICTURES_CONFIG, + USE_BLOB_STORAGE, + USE_S3_STORAGE, +} from '@/lib/uploads/core/setup' + +export type StorageContext = + | 'general' + | 'knowledge-base' + | 'chat' + | 'copilot' + | 'execution' + | 'workspace' + | 'profile-pictures' + +export interface StorageConfig { + // S3 config + bucket?: string + region?: string + // Blob config + containerName?: string + accountName?: string + accountKey?: string + connectionString?: string +} + +/** + * Get the appropriate storage configuration for a given context + * Automatically selects between S3 and Blob based on USE_BLOB_STORAGE/USE_S3_STORAGE flags + */ +export function getStorageConfig(context: StorageContext): StorageConfig { + if (USE_BLOB_STORAGE) { + return getBlobConfig(context) + } + + if (USE_S3_STORAGE) { + return getS3Config(context) + } + + // Local storage doesn't need config + return {} +} + +/** + * Get S3 configuration for a given context + */ +function getS3Config(context: StorageContext): StorageConfig { + switch (context) { + case 'knowledge-base': + return { + bucket: S3_KB_CONFIG.bucket, + region: S3_KB_CONFIG.region, + } + case 'chat': + return { + bucket: S3_CHAT_CONFIG.bucket, + region: S3_CHAT_CONFIG.region, + } + case 'copilot': + return { + bucket: S3_COPILOT_CONFIG.bucket, + region: S3_COPILOT_CONFIG.region, + } + case 'execution': + return { + bucket: S3_EXECUTION_FILES_CONFIG.bucket, + region: S3_EXECUTION_FILES_CONFIG.region, + } + case 'workspace': + // Workspace files use general bucket but with custom key structure + return { + bucket: S3_CONFIG.bucket, + region: S3_CONFIG.region, + } + case 'profile-pictures': + return { + bucket: S3_PROFILE_PICTURES_CONFIG.bucket, + region: S3_PROFILE_PICTURES_CONFIG.region, + } + default: + return { + bucket: S3_CONFIG.bucket, + region: S3_CONFIG.region, + } + } +} + +/** + * Get Azure Blob configuration for a given context + */ +function getBlobConfig(context: StorageContext): StorageConfig { + switch (context) { + case 'knowledge-base': + return { + accountName: BLOB_KB_CONFIG.accountName, + accountKey: BLOB_KB_CONFIG.accountKey, + connectionString: BLOB_KB_CONFIG.connectionString, + containerName: BLOB_KB_CONFIG.containerName, + } + case 'chat': + return { + accountName: BLOB_CHAT_CONFIG.accountName, + accountKey: BLOB_CHAT_CONFIG.accountKey, + connectionString: BLOB_CHAT_CONFIG.connectionString, + containerName: BLOB_CHAT_CONFIG.containerName, + } + case 'copilot': + return { + accountName: BLOB_COPILOT_CONFIG.accountName, + accountKey: BLOB_COPILOT_CONFIG.accountKey, + connectionString: BLOB_COPILOT_CONFIG.connectionString, + containerName: BLOB_COPILOT_CONFIG.containerName, + } + case 'execution': + return { + accountName: BLOB_EXECUTION_FILES_CONFIG.accountName, + accountKey: BLOB_EXECUTION_FILES_CONFIG.accountKey, + connectionString: BLOB_EXECUTION_FILES_CONFIG.connectionString, + containerName: BLOB_EXECUTION_FILES_CONFIG.containerName, + } + case 'workspace': + // Workspace files use general container but with custom key structure + return { + accountName: BLOB_CONFIG.accountName, + accountKey: BLOB_CONFIG.accountKey, + connectionString: BLOB_CONFIG.connectionString, + containerName: BLOB_CONFIG.containerName, + } + case 'profile-pictures': + return { + accountName: BLOB_PROFILE_PICTURES_CONFIG.accountName, + accountKey: BLOB_PROFILE_PICTURES_CONFIG.accountKey, + connectionString: BLOB_PROFILE_PICTURES_CONFIG.connectionString, + containerName: BLOB_PROFILE_PICTURES_CONFIG.containerName, + } + default: + return { + accountName: BLOB_CONFIG.accountName, + accountKey: BLOB_CONFIG.accountKey, + connectionString: BLOB_CONFIG.connectionString, + containerName: BLOB_CONFIG.containerName, + } + } +} + +/** + * Check if a specific storage context is configured + * Returns false if the context would fall back to general config but general isn't configured + */ +export function isStorageContextConfigured(context: StorageContext): boolean { + const config = getStorageConfig(context) + + if (USE_BLOB_STORAGE) { + return !!( + config.containerName && + (config.connectionString || (config.accountName && config.accountKey)) + ) + } + + if (USE_S3_STORAGE) { + return !!(config.bucket && config.region) + } + + // Local storage is always available + return true +} diff --git a/apps/sim/lib/uploads/setup.server.ts b/apps/sim/lib/uploads/core/setup.server.ts similarity index 99% rename from apps/sim/lib/uploads/setup.server.ts rename to apps/sim/lib/uploads/core/setup.server.ts index fb65de1ff..aef5f08a0 100644 --- a/apps/sim/lib/uploads/setup.server.ts +++ b/apps/sim/lib/uploads/core/setup.server.ts @@ -3,7 +3,7 @@ import { mkdir } from 'fs/promises' import path, { join } from 'path' import { env } from '@/lib/env' import { createLogger } from '@/lib/logs/console/logger' -import { getStorageProvider, USE_BLOB_STORAGE, USE_S3_STORAGE } from '@/lib/uploads/setup' +import { getStorageProvider, USE_BLOB_STORAGE, USE_S3_STORAGE } from '@/lib/uploads/core/setup' const logger = createLogger('UploadsSetup') diff --git a/apps/sim/lib/uploads/setup.ts b/apps/sim/lib/uploads/core/setup.ts similarity index 100% rename from apps/sim/lib/uploads/setup.ts rename to apps/sim/lib/uploads/core/setup.ts diff --git a/apps/sim/lib/uploads/storage-client.ts b/apps/sim/lib/uploads/core/storage-client.ts similarity index 68% rename from apps/sim/lib/uploads/storage-client.ts rename to apps/sim/lib/uploads/core/storage-client.ts index 72607d934..5b48d7a5f 100644 --- a/apps/sim/lib/uploads/storage-client.ts +++ b/apps/sim/lib/uploads/core/storage-client.ts @@ -1,7 +1,7 @@ import { createLogger } from '@/lib/logs/console/logger' -import type { CustomBlobConfig } from '@/lib/uploads/blob/blob-client' -import type { CustomS3Config } from '@/lib/uploads/s3/s3-client' -import { USE_BLOB_STORAGE, USE_S3_STORAGE } from '@/lib/uploads/setup' +import { USE_BLOB_STORAGE, USE_S3_STORAGE } from '@/lib/uploads/core/setup' +import type { CustomBlobConfig } from '@/lib/uploads/providers/blob/blob-client' +import type { CustomS3Config } from '@/lib/uploads/providers/s3/s3-client' const logger = createLogger('StorageClient') @@ -66,7 +66,7 @@ export async function uploadFile( ): Promise { if (USE_BLOB_STORAGE) { logger.info(`Uploading file to Azure Blob Storage: ${fileName}`) - const { uploadToBlob } = await import('@/lib/uploads/blob/blob-client') + const { uploadToBlob } = await import('@/lib/uploads/providers/blob/blob-client') if (typeof configOrSize === 'object') { const blobConfig: CustomBlobConfig = { containerName: configOrSize.containerName!, @@ -81,7 +81,7 @@ export async function uploadFile( if (USE_S3_STORAGE) { logger.info(`Uploading file to S3: ${fileName}`) - const { uploadToS3 } = await import('@/lib/uploads/s3/s3-client') + const { uploadToS3 } = await import('@/lib/uploads/providers/s3/s3-client') if (typeof configOrSize === 'object') { const s3Config: CustomS3Config = { bucket: configOrSize.bucket!, @@ -96,7 +96,7 @@ export async function uploadFile( const { writeFile } = await import('fs/promises') const { join } = await import('path') const { v4: uuidv4 } = await import('uuid') - const { UPLOAD_DIR_SERVER } = await import('@/lib/uploads/setup.server') + const { UPLOAD_DIR_SERVER } = await import('@/lib/uploads/core/setup.server') const safeFileName = fileName.replace(/[^a-zA-Z0-9.-]/g, '_').replace(/\.\./g, '') const uniqueKey = `${uuidv4()}-${safeFileName}` @@ -143,7 +143,7 @@ export async function downloadFile( ): Promise { if (USE_BLOB_STORAGE) { logger.info(`Downloading file from Azure Blob Storage: ${key}`) - const { downloadFromBlob } = await import('@/lib/uploads/blob/blob-client') + const { downloadFromBlob } = await import('@/lib/uploads/providers/blob/blob-client') if (customConfig) { const blobConfig: CustomBlobConfig = { containerName: customConfig.containerName!, @@ -158,7 +158,7 @@ export async function downloadFile( if (USE_S3_STORAGE) { logger.info(`Downloading file from S3: ${key}`) - const { downloadFromS3 } = await import('@/lib/uploads/s3/s3-client') + const { downloadFromS3 } = await import('@/lib/uploads/providers/s3/s3-client') if (customConfig) { const s3Config: CustomS3Config = { bucket: customConfig.bucket!, @@ -172,7 +172,7 @@ export async function downloadFile( logger.info(`Downloading file from local storage: ${key}`) const { readFile } = await import('fs/promises') const { join, resolve, sep } = await import('path') - const { UPLOAD_DIR_SERVER } = await import('@/lib/uploads/setup.server') + const { UPLOAD_DIR_SERVER } = await import('@/lib/uploads/core/setup.server') const safeKey = key.replace(/\.\./g, '').replace(/[/\\]/g, '') const filePath = join(UPLOAD_DIR_SERVER, safeKey) @@ -200,20 +200,20 @@ export async function downloadFile( export async function deleteFile(key: string): Promise { if (USE_BLOB_STORAGE) { logger.info(`Deleting file from Azure Blob Storage: ${key}`) - const { deleteFromBlob } = await import('@/lib/uploads/blob/blob-client') + const { deleteFromBlob } = await import('@/lib/uploads/providers/blob/blob-client') return deleteFromBlob(key) } if (USE_S3_STORAGE) { logger.info(`Deleting file from S3: ${key}`) - const { deleteFromS3 } = await import('@/lib/uploads/s3/s3-client') + const { deleteFromS3 } = await import('@/lib/uploads/providers/s3/s3-client') return deleteFromS3(key) } logger.info(`Deleting file from local storage: ${key}`) const { unlink } = await import('fs/promises') const { join, resolve, sep } = await import('path') - const { UPLOAD_DIR_SERVER } = await import('@/lib/uploads/setup.server') + const { UPLOAD_DIR_SERVER } = await import('@/lib/uploads/core/setup.server') const safeKey = key.replace(/\.\./g, '').replace(/[/\\]/g, '') const filePath = join(UPLOAD_DIR_SERVER, safeKey) @@ -235,74 +235,6 @@ export async function deleteFile(key: string): Promise { } } -/** - * Generate a presigned URL for direct file access - * @param key File key/name - * @param expiresIn Time in seconds until URL expires - * @returns Presigned URL - */ -export async function getPresignedUrl(key: string, expiresIn = 3600): Promise { - if (USE_BLOB_STORAGE) { - logger.info(`Generating presigned URL for Azure Blob Storage: ${key}`) - const { getPresignedUrl: getBlobPresignedUrl } = await import('@/lib/uploads/blob/blob-client') - return getBlobPresignedUrl(key, expiresIn) - } - - if (USE_S3_STORAGE) { - logger.info(`Generating presigned URL for S3: ${key}`) - const { getPresignedUrl: getS3PresignedUrl } = await import('@/lib/uploads/s3/s3-client') - return getS3PresignedUrl(key, expiresIn) - } - - logger.info(`Generating serve path for local storage: ${key}`) - return `/api/files/serve/${encodeURIComponent(key)}` -} - -/** - * Generate a presigned URL for direct file access with custom configuration - * @param key File key/name - * @param customConfig Custom storage configuration - * @param expiresIn Time in seconds until URL expires - * @returns Presigned URL - */ -export async function getPresignedUrlWithConfig( - key: string, - customConfig: CustomStorageConfig, - expiresIn = 3600 -): Promise { - if (USE_BLOB_STORAGE) { - logger.info(`Generating presigned URL for Azure Blob Storage with custom config: ${key}`) - const { getPresignedUrlWithConfig: getBlobPresignedUrlWithConfig } = await import( - '@/lib/uploads/blob/blob-client' - ) - // Convert CustomStorageConfig to CustomBlobConfig - const blobConfig: CustomBlobConfig = { - containerName: customConfig.containerName!, - accountName: customConfig.accountName!, - accountKey: customConfig.accountKey, - connectionString: customConfig.connectionString, - } - return getBlobPresignedUrlWithConfig(key, blobConfig, expiresIn) - } - - if (USE_S3_STORAGE) { - logger.info(`Generating presigned URL for S3 with custom config: ${key}`) - const { getPresignedUrlWithConfig: getS3PresignedUrlWithConfig } = await import( - '@/lib/uploads/s3/s3-client' - ) - // Convert CustomStorageConfig to CustomS3Config - const s3Config: CustomS3Config = { - bucket: customConfig.bucket!, - region: customConfig.region!, - } - return getS3PresignedUrlWithConfig(key, s3Config, expiresIn) - } - - throw new Error( - 'No storage provider configured. Set Azure credentials (AZURE_CONNECTION_STRING or AZURE_ACCOUNT_NAME + AZURE_ACCOUNT_KEY) or configure AWS credentials for S3.' - ) -} - /** * Get the current storage provider name */ diff --git a/apps/sim/lib/uploads/core/storage-service.ts b/apps/sim/lib/uploads/core/storage-service.ts new file mode 100644 index 000000000..a6f5bc72c --- /dev/null +++ b/apps/sim/lib/uploads/core/storage-service.ts @@ -0,0 +1,428 @@ +import { createLogger } from '@/lib/logs/console/logger' +import { USE_BLOB_STORAGE, USE_S3_STORAGE } from '@/lib/uploads/core/setup' +import { getStorageConfig, type StorageContext } from './config-resolver' +import type { FileInfo } from './storage-client' + +const logger = createLogger('StorageService') + +export interface UploadFileOptions { + file: Buffer + fileName: string + contentType: string + context: StorageContext + preserveKey?: boolean // Skip timestamp prefix (for workspace/execution files) + customKey?: string // Provide exact key to use (overrides fileName) + metadata?: Record +} + +export interface DownloadFileOptions { + key: string + context?: StorageContext +} + +export interface DeleteFileOptions { + key: string + context?: StorageContext +} + +export interface GeneratePresignedUrlOptions { + fileName: string + contentType: string + fileSize: number + context: StorageContext + userId?: string + expirationSeconds?: number + metadata?: Record +} + +export interface PresignedUrlResponse { + url: string + key: string + uploadHeaders?: Record +} + +/** + * Upload a file to the configured storage provider with context-aware configuration + */ +export async function uploadFile(options: UploadFileOptions): Promise { + const { file, fileName, contentType, context, preserveKey, customKey, metadata } = options + + logger.info(`Uploading file to ${context} storage: ${fileName}`) + + const config = getStorageConfig(context) + + const keyToUse = customKey || fileName + + if (USE_BLOB_STORAGE) { + const { uploadToBlob } = await import('../providers/blob/blob-client') + const blobConfig = { + containerName: config.containerName!, + accountName: config.accountName!, + accountKey: config.accountKey, + connectionString: config.connectionString, + } + + return uploadToBlob(file, keyToUse, contentType, blobConfig, file.length) + } + + if (USE_S3_STORAGE) { + const { uploadToS3 } = await import('../providers/s3/s3-client') + const s3Config = { + bucket: config.bucket!, + region: config.region!, + } + + return uploadToS3(file, keyToUse, contentType, s3Config, file.length, preserveKey) + } + + logger.info('Using local file storage') + const { writeFile } = await import('fs/promises') + const { join } = await import('path') + const { v4: uuidv4 } = await import('uuid') + const { UPLOAD_DIR_SERVER } = await import('./setup.server') + + const safeKey = keyToUse.replace(/[^a-zA-Z0-9.-]/g, '_').replace(/\.\./g, '') + const uniqueKey = `${uuidv4()}-${safeKey}` + const filePath = join(UPLOAD_DIR_SERVER, uniqueKey) + + try { + await writeFile(filePath, file) + } catch (error) { + logger.error(`Failed to write file to local storage: ${fileName}`, error) + throw new Error( + `Failed to write file to local storage: ${error instanceof Error ? error.message : 'Unknown error'}` + ) + } + + return { + path: `/api/files/serve/${uniqueKey}`, + key: uniqueKey, + name: fileName, + size: file.length, + type: contentType, + } +} + +/** + * Download a file from the configured storage provider + */ +export async function downloadFile(options: DownloadFileOptions): Promise { + const { key, context } = options + + logger.info(`Downloading file: ${key}${context ? ` (context: ${context})` : ''}`) + + if (context) { + const config = getStorageConfig(context) + + if (USE_BLOB_STORAGE) { + const { downloadFromBlob } = await import('../providers/blob/blob-client') + const blobConfig = { + containerName: config.containerName!, + accountName: config.accountName!, + accountKey: config.accountKey, + connectionString: config.connectionString, + } + return downloadFromBlob(key, blobConfig) + } + + if (USE_S3_STORAGE) { + const { downloadFromS3 } = await import('../providers/s3/s3-client') + const s3Config = { + bucket: config.bucket!, + region: config.region!, + } + return downloadFromS3(key, s3Config) + } + } + + const { downloadFile: defaultDownload } = await import('./storage-client') + return defaultDownload(key) +} + +/** + * Delete a file from the configured storage provider + */ +export async function deleteFile(options: DeleteFileOptions): Promise { + const { key, context } = options + + logger.info(`Deleting file: ${key}${context ? ` (context: ${context})` : ''}`) + + if (context) { + const config = getStorageConfig(context) + + if (USE_BLOB_STORAGE) { + const { deleteFromBlob } = await import('../providers/blob/blob-client') + const blobConfig = { + containerName: config.containerName!, + accountName: config.accountName!, + accountKey: config.accountKey, + connectionString: config.connectionString, + } + return deleteFromBlob(key, blobConfig) + } + + if (USE_S3_STORAGE) { + const { deleteFromS3 } = await import('../providers/s3/s3-client') + const s3Config = { + bucket: config.bucket!, + region: config.region!, + } + return deleteFromS3(key, s3Config) + } + } + + const { deleteFile: defaultDelete } = await import('./storage-client') + return defaultDelete(key) +} + +/** + * Generate a presigned URL for direct file upload + */ +export async function generatePresignedUploadUrl( + options: GeneratePresignedUrlOptions +): Promise { + const { + fileName, + contentType, + fileSize, + context, + userId, + expirationSeconds = 3600, + metadata = {}, + } = options + + logger.info(`Generating presigned upload URL for ${context}: ${fileName}`) + + const allMetadata = { + ...metadata, + originalname: fileName, + uploadedat: new Date().toISOString(), + purpose: context, + ...(userId && { userid: userId }), + } + + const config = getStorageConfig(context) + + const timestamp = Date.now() + const uniqueId = Math.random().toString(36).substring(2, 9) + const safeFileName = fileName.replace(/[^a-zA-Z0-9.-]/g, '_') + const key = `${timestamp}-${uniqueId}-${safeFileName}` + + if (USE_S3_STORAGE) { + return generateS3PresignedUrl( + key, + contentType, + fileSize, + allMetadata, + config, + expirationSeconds + ) + } + + if (USE_BLOB_STORAGE) { + return generateBlobPresignedUrl(key, contentType, allMetadata, config, expirationSeconds) + } + + throw new Error('Cloud storage not configured. Cannot generate presigned URL for local storage.') +} + +/** + * Generate presigned URL for S3 + */ +async function generateS3PresignedUrl( + key: string, + contentType: string, + fileSize: number, + metadata: Record, + config: { bucket?: string; region?: string }, + expirationSeconds: number +): Promise { + const { getS3Client, sanitizeFilenameForMetadata } = await import('../providers/s3/s3-client') + const { PutObjectCommand } = await import('@aws-sdk/client-s3') + const { getSignedUrl } = await import('@aws-sdk/s3-request-presigner') + + if (!config.bucket || !config.region) { + throw new Error('S3 configuration missing bucket or region') + } + + const sanitizedMetadata: Record = {} + for (const [key, value] of Object.entries(metadata)) { + if (key === 'originalname') { + sanitizedMetadata[key] = sanitizeFilenameForMetadata(value) + } else { + sanitizedMetadata[key] = value + } + } + + const command = new PutObjectCommand({ + Bucket: config.bucket, + Key: key, + ContentType: contentType, + ContentLength: fileSize, + Metadata: sanitizedMetadata, + }) + + const presignedUrl = await getSignedUrl(getS3Client(), command, { expiresIn: expirationSeconds }) + + return { + url: presignedUrl, + key, + } +} + +/** + * Generate presigned URL for Azure Blob + */ +async function generateBlobPresignedUrl( + key: string, + contentType: string, + metadata: Record, + config: { + containerName?: string + accountName?: string + accountKey?: string + connectionString?: string + }, + expirationSeconds: number +): Promise { + const { getBlobServiceClient } = await import('../providers/blob/blob-client') + const { BlobSASPermissions, generateBlobSASQueryParameters, StorageSharedKeyCredential } = + await import('@azure/storage-blob') + + if (!config.containerName) { + throw new Error('Blob configuration missing container name') + } + + const blobServiceClient = getBlobServiceClient() + const containerClient = blobServiceClient.getContainerClient(config.containerName) + const blobClient = containerClient.getBlockBlobClient(key) + + const startsOn = new Date() + const expiresOn = new Date(startsOn.getTime() + expirationSeconds * 1000) + + let sasToken: string + + if (config.accountName && config.accountKey) { + const sharedKeyCredential = new StorageSharedKeyCredential( + config.accountName, + config.accountKey + ) + sasToken = generateBlobSASQueryParameters( + { + containerName: config.containerName, + blobName: key, + permissions: BlobSASPermissions.parse('w'), // write permission for upload + startsOn, + expiresOn, + }, + sharedKeyCredential + ).toString() + } else { + throw new Error('Azure Blob SAS generation requires accountName and accountKey') + } + + return { + url: `${blobClient.url}?${sasToken}`, + key, + uploadHeaders: { + 'x-ms-blob-type': 'BlockBlob', + 'x-ms-blob-content-type': contentType, + ...Object.entries(metadata).reduce( + (acc, [k, v]) => { + acc[`x-ms-meta-${k}`] = encodeURIComponent(v) + return acc + }, + {} as Record + ), + }, + } +} + +/** + * Generate multiple presigned URLs at once (batch operation) + */ +export async function generateBatchPresignedUploadUrls( + files: Array<{ + fileName: string + contentType: string + fileSize: number + }>, + context: StorageContext, + userId?: string, + expirationSeconds?: number +): Promise { + logger.info(`Generating ${files.length} presigned upload URLs for ${context}`) + + const results: PresignedUrlResponse[] = [] + + for (const file of files) { + const result = await generatePresignedUploadUrl({ + fileName: file.fileName, + contentType: file.contentType, + fileSize: file.fileSize, + context, + userId, + expirationSeconds, + }) + results.push(result) + } + + return results +} + +/** + * Generate a presigned URL for downloading/accessing an existing file + */ +export async function generatePresignedDownloadUrl( + key: string, + context: StorageContext, + expirationSeconds = 3600 +): Promise { + logger.info(`Generating presigned download URL for ${context}: ${key}`) + + const config = getStorageConfig(context) + + if (USE_S3_STORAGE) { + const { getPresignedUrlWithConfig } = await import('../providers/s3/s3-client') + return getPresignedUrlWithConfig( + key, + { + bucket: config.bucket!, + region: config.region!, + }, + expirationSeconds + ) + } + + if (USE_BLOB_STORAGE) { + const { getPresignedUrlWithConfig } = await import('../providers/blob/blob-client') + return getPresignedUrlWithConfig( + key, + { + containerName: config.containerName!, + accountName: config.accountName!, + accountKey: config.accountKey, + connectionString: config.connectionString, + }, + expirationSeconds + ) + } + + return `/api/files/serve/${encodeURIComponent(key)}` +} + +/** + * Check if cloud storage is available + */ +export function hasCloudStorage(): boolean { + return USE_BLOB_STORAGE || USE_S3_STORAGE +} + +/** + * Get the current storage provider name + */ +export function getStorageProviderName(): 'Azure Blob' | 'S3' | 'Local' { + if (USE_BLOB_STORAGE) return 'Azure Blob' + if (USE_S3_STORAGE) return 'S3' + return 'Local' +} diff --git a/apps/sim/lib/uploads/index.ts b/apps/sim/lib/uploads/index.ts index 93c57feca..79a91d175 100644 --- a/apps/sim/lib/uploads/index.ts +++ b/apps/sim/lib/uploads/index.ts @@ -1,7 +1,21 @@ -// BlobClient and S3Client are server-only - import from specific files when needed -// export * as BlobClient from '@/lib/uploads/blob/blob-client' -// export * as S3Client from '@/lib/uploads/s3/s3-client' - +export * as ChatFiles from '@/lib/uploads/contexts/chat' +export * as CopilotFiles from '@/lib/uploads/contexts/copilot' +export * as ExecutionFiles from '@/lib/uploads/contexts/execution' +export * as WorkspaceFiles from '@/lib/uploads/contexts/workspace' +export { getStorageConfig, type StorageContext } from '@/lib/uploads/core/config-resolver' +export { + UPLOAD_DIR, + USE_BLOB_STORAGE, + USE_S3_STORAGE, +} from '@/lib/uploads/core/setup' +export { + type CustomStorageConfig, + type FileInfo, + getServePathPrefix, + getStorageProvider, + isUsingCloudStorage, +} from '@/lib/uploads/core/storage-client' +export * as StorageService from '@/lib/uploads/core/storage-service' export { bufferToBase64, createFileContent as createAnthropicFileContent, @@ -12,27 +26,4 @@ export { isSupportedFileType, type MessageContent as AnthropicMessageContent, MIME_TYPE_MAPPING, -} from '@/lib/uploads/file-utils' -export { - BLOB_CHAT_CONFIG, - BLOB_CONFIG, - BLOB_KB_CONFIG, - S3_CHAT_CONFIG, - S3_CONFIG, - S3_KB_CONFIG, - UPLOAD_DIR, - USE_BLOB_STORAGE, - USE_S3_STORAGE, -} from '@/lib/uploads/setup' -export { - type CustomStorageConfig, - deleteFile, - downloadFile, - type FileInfo, - getPresignedUrl, - getPresignedUrlWithConfig, - getServePathPrefix, - getStorageProvider, - isUsingCloudStorage, - uploadFile, -} from '@/lib/uploads/storage-client' +} from '@/lib/uploads/utils/file-utils' diff --git a/apps/sim/lib/uploads/blob/blob-client.test.ts b/apps/sim/lib/uploads/providers/blob/blob-client.test.ts similarity index 91% rename from apps/sim/lib/uploads/blob/blob-client.test.ts rename to apps/sim/lib/uploads/providers/blob/blob-client.test.ts index c82f89f46..451947308 100644 --- a/apps/sim/lib/uploads/blob/blob-client.test.ts +++ b/apps/sim/lib/uploads/providers/blob/blob-client.test.ts @@ -90,7 +90,7 @@ describe('Azure Blob Storage Client', () => { describe('uploadToBlob', () => { it('should upload a file to Azure Blob Storage', async () => { - const { uploadToBlob } = await import('@/lib/uploads/blob/blob-client') + const { uploadToBlob } = await import('@/lib/uploads/providers/blob/blob-client') const testBuffer = Buffer.from('test file content') const fileName = 'test-file.txt' @@ -120,7 +120,7 @@ describe('Azure Blob Storage Client', () => { }) it('should handle custom blob configuration', async () => { - const { uploadToBlob } = await import('@/lib/uploads/blob/blob-client') + const { uploadToBlob } = await import('@/lib/uploads/providers/blob/blob-client') const testBuffer = Buffer.from('test file content') const fileName = 'test-file.txt' @@ -143,7 +143,7 @@ describe('Azure Blob Storage Client', () => { describe('downloadFromBlob', () => { it('should download a file from Azure Blob Storage', async () => { - const { downloadFromBlob } = await import('@/lib/uploads/blob/blob-client') + const { downloadFromBlob } = await import('@/lib/uploads/providers/blob/blob-client') const testKey = 'test-file-key' const testContent = Buffer.from('downloaded content') @@ -172,7 +172,7 @@ describe('Azure Blob Storage Client', () => { describe('deleteFromBlob', () => { it('should delete a file from Azure Blob Storage', async () => { - const { deleteFromBlob } = await import('@/lib/uploads/blob/blob-client') + const { deleteFromBlob } = await import('@/lib/uploads/providers/blob/blob-client') const testKey = 'test-file-key' @@ -187,7 +187,7 @@ describe('Azure Blob Storage Client', () => { describe('getPresignedUrl', () => { it('should generate a presigned URL for Azure Blob Storage', async () => { - const { getPresignedUrl } = await import('@/lib/uploads/blob/blob-client') + const { getPresignedUrl } = await import('@/lib/uploads/providers/blob/blob-client') const testKey = 'test-file-key' const expiresIn = 3600 @@ -211,7 +211,9 @@ describe('Azure Blob Storage Client', () => { ] it.each(testCases)('should sanitize "$input" to "$expected"', async ({ input, expected }) => { - const { sanitizeFilenameForMetadata } = await import('@/lib/uploads/blob/blob-client') + const { sanitizeFilenameForMetadata } = await import( + '@/lib/uploads/providers/blob/blob-client' + ) expect(sanitizeFilenameForMetadata(input)).toBe(expected) }) }) diff --git a/apps/sim/lib/uploads/blob/blob-client.ts b/apps/sim/lib/uploads/providers/blob/blob-client.ts similarity index 99% rename from apps/sim/lib/uploads/blob/blob-client.ts rename to apps/sim/lib/uploads/providers/blob/blob-client.ts index 18b7c1a9e..bf2420ffe 100644 --- a/apps/sim/lib/uploads/blob/blob-client.ts +++ b/apps/sim/lib/uploads/providers/blob/blob-client.ts @@ -6,7 +6,7 @@ import { StorageSharedKeyCredential, } from '@azure/storage-blob' import { createLogger } from '@/lib/logs/console/logger' -import { BLOB_CONFIG } from '@/lib/uploads/setup' +import { BLOB_CONFIG } from '@/lib/uploads/core/setup' const logger = createLogger('BlobClient') diff --git a/apps/sim/lib/uploads/blob/index.ts b/apps/sim/lib/uploads/providers/blob/index.ts similarity index 80% rename from apps/sim/lib/uploads/blob/index.ts rename to apps/sim/lib/uploads/providers/blob/index.ts index f4be6ae11..6f1a03c14 100644 --- a/apps/sim/lib/uploads/blob/index.ts +++ b/apps/sim/lib/uploads/providers/blob/index.ts @@ -8,4 +8,4 @@ export { getPresignedUrlWithConfig, sanitizeFilenameForMetadata, uploadToBlob, -} from '@/lib/uploads/blob/blob-client' +} from '@/lib/uploads/providers/blob/blob-client' diff --git a/apps/sim/lib/uploads/s3/index.ts b/apps/sim/lib/uploads/providers/s3/index.ts similarity index 80% rename from apps/sim/lib/uploads/s3/index.ts rename to apps/sim/lib/uploads/providers/s3/index.ts index d39af344c..b26661933 100644 --- a/apps/sim/lib/uploads/s3/index.ts +++ b/apps/sim/lib/uploads/providers/s3/index.ts @@ -8,4 +8,4 @@ export { getS3Client, sanitizeFilenameForMetadata, uploadToS3, -} from '@/lib/uploads/s3/s3-client' +} from '@/lib/uploads/providers/s3/s3-client' diff --git a/apps/sim/lib/uploads/s3/s3-client.test.ts b/apps/sim/lib/uploads/providers/s3/s3-client.test.ts similarity index 88% rename from apps/sim/lib/uploads/s3/s3-client.test.ts rename to apps/sim/lib/uploads/providers/s3/s3-client.test.ts index 7188f11aa..08662c5f0 100644 --- a/apps/sim/lib/uploads/s3/s3-client.test.ts +++ b/apps/sim/lib/uploads/providers/s3/s3-client.test.ts @@ -68,7 +68,7 @@ describe('S3 Client', () => { it('should upload a file to S3 and return file info', async () => { mockSend.mockResolvedValueOnce({}) - const { uploadToS3 } = await import('@/lib/uploads/s3/s3-client') + const { uploadToS3 } = await import('@/lib/uploads/providers/s3/s3-client') const file = Buffer.from('test content') const fileName = 'test-file.txt' @@ -101,7 +101,7 @@ describe('S3 Client', () => { it('should handle spaces in filenames', async () => { mockSend.mockResolvedValueOnce({}) - const { uploadToS3 } = await import('@/lib/uploads/s3/s3-client') + const { uploadToS3 } = await import('@/lib/uploads/providers/s3/s3-client') const testFile = Buffer.from('test file content') const fileName = 'test file with spaces.txt' @@ -121,7 +121,7 @@ describe('S3 Client', () => { it('should use provided size if available', async () => { mockSend.mockResolvedValueOnce({}) - const { uploadToS3 } = await import('@/lib/uploads/s3/s3-client') + const { uploadToS3 } = await import('@/lib/uploads/providers/s3/s3-client') const testFile = Buffer.from('test file content') const fileName = 'test-file.txt' @@ -137,7 +137,7 @@ describe('S3 Client', () => { const error = new Error('Upload failed') mockSend.mockRejectedValueOnce(error) - const { uploadToS3 } = await import('@/lib/uploads/s3/s3-client') + const { uploadToS3 } = await import('@/lib/uploads/providers/s3/s3-client') const testFile = Buffer.from('test file content') const fileName = 'test-file.txt' @@ -151,7 +151,7 @@ describe('S3 Client', () => { it('should generate a presigned URL for a file', async () => { mockGetSignedUrl.mockResolvedValueOnce('https://example.com/presigned-url') - const { getPresignedUrl } = await import('@/lib/uploads/s3/s3-client') + const { getPresignedUrl } = await import('@/lib/uploads/providers/s3/s3-client') const key = 'test-file.txt' const expiresIn = 1800 @@ -171,7 +171,7 @@ describe('S3 Client', () => { it('should use default expiration if not provided', async () => { mockGetSignedUrl.mockResolvedValueOnce('https://example.com/presigned-url') - const { getPresignedUrl } = await import('@/lib/uploads/s3/s3-client') + const { getPresignedUrl } = await import('@/lib/uploads/providers/s3/s3-client') const key = 'test-file.txt' @@ -188,7 +188,7 @@ describe('S3 Client', () => { const error = new Error('Presigned URL generation failed') mockGetSignedUrl.mockRejectedValueOnce(error) - const { getPresignedUrl } = await import('@/lib/uploads/s3/s3-client') + const { getPresignedUrl } = await import('@/lib/uploads/providers/s3/s3-client') const key = 'test-file.txt' @@ -216,7 +216,7 @@ describe('S3 Client', () => { $metadata: { httpStatusCode: 200 }, }) - const { downloadFromS3 } = await import('@/lib/uploads/s3/s3-client') + const { downloadFromS3 } = await import('@/lib/uploads/providers/s3/s3-client') const key = 'test-file.txt' @@ -247,7 +247,7 @@ describe('S3 Client', () => { $metadata: { httpStatusCode: 200 }, }) - const { downloadFromS3 } = await import('@/lib/uploads/s3/s3-client') + const { downloadFromS3 } = await import('@/lib/uploads/providers/s3/s3-client') const key = 'test-file.txt' @@ -258,7 +258,7 @@ describe('S3 Client', () => { const error = new Error('Download failed') mockSend.mockRejectedValueOnce(error) - const { downloadFromS3 } = await import('@/lib/uploads/s3/s3-client') + const { downloadFromS3 } = await import('@/lib/uploads/providers/s3/s3-client') const key = 'test-file.txt' @@ -270,7 +270,7 @@ describe('S3 Client', () => { it('should delete a file from S3', async () => { mockSend.mockResolvedValueOnce({}) - const { deleteFromS3 } = await import('@/lib/uploads/s3/s3-client') + const { deleteFromS3 } = await import('@/lib/uploads/providers/s3/s3-client') const key = 'test-file.txt' @@ -288,7 +288,7 @@ describe('S3 Client', () => { const error = new Error('Delete failed') mockSend.mockRejectedValueOnce(error) - const { deleteFromS3 } = await import('@/lib/uploads/s3/s3-client') + const { deleteFromS3 } = await import('@/lib/uploads/providers/s3/s3-client') const key = 'test-file.txt' @@ -315,7 +315,7 @@ describe('S3 Client', () => { })) vi.resetModules() - const { getS3Client } = await import('@/lib/uploads/s3/s3-client') + const { getS3Client } = await import('@/lib/uploads/providers/s3/s3-client') const { S3Client } = await import('@aws-sdk/client-s3') const client = getS3Client() @@ -348,7 +348,7 @@ describe('S3 Client', () => { })) vi.resetModules() - const { getS3Client } = await import('@/lib/uploads/s3/s3-client') + const { getS3Client } = await import('@/lib/uploads/providers/s3/s3-client') const { S3Client } = await import('@aws-sdk/client-s3') const client = getS3Client() diff --git a/apps/sim/lib/uploads/s3/s3-client.ts b/apps/sim/lib/uploads/providers/s3/s3-client.ts similarity index 99% rename from apps/sim/lib/uploads/s3/s3-client.ts rename to apps/sim/lib/uploads/providers/s3/s3-client.ts index 11c411633..fd90adb4f 100644 --- a/apps/sim/lib/uploads/s3/s3-client.ts +++ b/apps/sim/lib/uploads/providers/s3/s3-client.ts @@ -10,7 +10,7 @@ import { } from '@aws-sdk/client-s3' import { getSignedUrl } from '@aws-sdk/s3-request-presigner' import { env } from '@/lib/env' -import { S3_CONFIG, S3_KB_CONFIG } from '@/lib/uploads/setup' +import { S3_CONFIG, S3_KB_CONFIG } from '@/lib/uploads/core/setup' // Lazily create a single S3 client instance. let _s3Client: S3Client | null = null diff --git a/apps/sim/lib/uploads/file-processing.ts b/apps/sim/lib/uploads/utils/file-processing.ts similarity index 66% rename from apps/sim/lib/uploads/file-processing.ts rename to apps/sim/lib/uploads/utils/file-processing.ts index 46e680a9f..18d95190d 100644 --- a/apps/sim/lib/uploads/file-processing.ts +++ b/apps/sim/lib/uploads/utils/file-processing.ts @@ -1,8 +1,7 @@ import type { Logger } from '@/lib/logs/console/logger' -import { extractStorageKey } from '@/lib/uploads/file-utils' -import { downloadFile } from '@/lib/uploads/storage-client' -import { downloadExecutionFile } from '@/lib/workflows/execution-file-storage' -import { isExecutionFile } from '@/lib/workflows/execution-files' +import { type StorageContext, StorageService } from '@/lib/uploads' +import { downloadExecutionFile, isExecutionFile } from '@/lib/uploads/contexts/execution' +import { extractStorageKey } from '@/lib/uploads/utils/file-utils' import type { UserFile } from '@/executor/types' /** @@ -75,6 +74,34 @@ export function processFilesToUserFiles( return userFiles } +/** + * Infer storage context from file key pattern + * @param key - File storage key + * @returns Inferred storage context + */ +function inferContextFromKey(key: string): StorageContext { + // KB files always start with 'kb/' prefix + if (key.startsWith('kb/')) { + return 'knowledge-base' + } + + // Execution files: three or more UUID segments + // Pattern: {uuid}/{uuid}/{uuid}/{filename} + const segments = key.split('/') + if (segments.length >= 4 && segments[0].match(/^[a-f0-9-]{36}$/)) { + return 'execution' + } + + // Workspace files: UUID-like ID followed by timestamp pattern + // Pattern: {uuid}/{timestamp}-{random}-{filename} + if (key.match(/^[a-f0-9-]{36}\/\d+-[a-z0-9]+-/)) { + return 'workspace' + } + + // Default to general for all other patterns + return 'general' +} + /** * Downloads a file from storage (execution or regular) * @param userFile - UserFile object @@ -93,8 +120,16 @@ export async function downloadFileFromStorage( logger.info(`[${requestId}] Downloading from execution storage: ${userFile.key}`) buffer = await downloadExecutionFile(userFile) } else if (userFile.key) { - logger.info(`[${requestId}] Downloading from regular storage: ${userFile.key}`) - buffer = await downloadFile(userFile.key) + // Use explicit context from file if available, otherwise infer from key pattern (fallback) + const context = (userFile.context as StorageContext) || inferContextFromKey(userFile.key) + logger.info( + `[${requestId}] Downloading from ${context} storage (${userFile.context ? 'explicit' : 'inferred'}): ${userFile.key}` + ) + + buffer = await StorageService.downloadFile({ + key: userFile.key, + context, + }) } else { throw new Error('File has no key - cannot download') } diff --git a/apps/sim/lib/uploads/file-utils.ts b/apps/sim/lib/uploads/utils/file-utils.ts similarity index 87% rename from apps/sim/lib/uploads/file-utils.ts rename to apps/sim/lib/uploads/utils/file-utils.ts index 2c02ba134..7a002a854 100644 --- a/apps/sim/lib/uploads/file-utils.ts +++ b/apps/sim/lib/uploads/utils/file-utils.ts @@ -146,16 +146,19 @@ export function getMimeTypeFromExtension(extension: string): string { /** * Extract storage key from a file path * Handles various path formats: /api/files/serve/xyz, /api/files/serve/s3/xyz, etc. + * Strips query parameters from the path before extracting the key. */ export function extractStorageKey(filePath: string): string { - if (filePath.includes('/api/files/serve/s3/')) { - return decodeURIComponent(filePath.split('/api/files/serve/s3/')[1]) + const pathWithoutQuery = filePath.split('?')[0] + + if (pathWithoutQuery.includes('/api/files/serve/s3/')) { + return decodeURIComponent(pathWithoutQuery.split('/api/files/serve/s3/')[1]) } - if (filePath.includes('/api/files/serve/blob/')) { - return decodeURIComponent(filePath.split('/api/files/serve/blob/')[1]) + if (pathWithoutQuery.includes('/api/files/serve/blob/')) { + return decodeURIComponent(pathWithoutQuery.split('/api/files/serve/blob/')[1]) } - if (filePath.startsWith('/api/files/serve/')) { - return decodeURIComponent(filePath.substring('/api/files/serve/'.length)) + if (pathWithoutQuery.startsWith('/api/files/serve/')) { + return decodeURIComponent(pathWithoutQuery.substring('/api/files/serve/'.length)) } - return filePath + return pathWithoutQuery } diff --git a/apps/sim/lib/uploads/utils/index.ts b/apps/sim/lib/uploads/utils/index.ts new file mode 100644 index 000000000..fa1977c7f --- /dev/null +++ b/apps/sim/lib/uploads/utils/index.ts @@ -0,0 +1,3 @@ +export * from './file-processing' +export * from './file-utils' +export * from './validation' diff --git a/apps/sim/lib/uploads/validation.ts b/apps/sim/lib/uploads/utils/validation.ts similarity index 100% rename from apps/sim/lib/uploads/validation.ts rename to apps/sim/lib/uploads/utils/validation.ts diff --git a/apps/sim/lib/webhooks/attachment-processor.ts b/apps/sim/lib/webhooks/attachment-processor.ts index 028a36287..b12fd28ed 100644 --- a/apps/sim/lib/webhooks/attachment-processor.ts +++ b/apps/sim/lib/webhooks/attachment-processor.ts @@ -1,5 +1,5 @@ import { createLogger } from '@/lib/logs/console/logger' -import { uploadExecutionFile } from '@/lib/workflows/execution-file-storage' +import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' import type { UserFile } from '@/executor/types' const logger = createLogger('WebhookAttachmentProcessor') diff --git a/apps/sim/lib/workflows/execution-file-storage.ts b/apps/sim/lib/workflows/execution-file-storage.ts deleted file mode 100644 index 0c478e50d..000000000 --- a/apps/sim/lib/workflows/execution-file-storage.ts +++ /dev/null @@ -1,263 +0,0 @@ -/** - * Specialized storage client for workflow execution files - * Uses dedicated S3 bucket: sim-execution-files - * Directory structure: workspace_id/workflow_id/execution_id/filename - */ - -import { createLogger } from '@/lib/logs/console/logger' -import { - deleteFromBlob, - downloadFromBlob, - getPresignedUrlWithConfig as getBlobPresignedUrlWithConfig, - uploadToBlob, -} from '@/lib/uploads/blob/blob-client' -import { - deleteFromS3, - downloadFromS3, - getPresignedUrlWithConfig, - uploadToS3, -} from '@/lib/uploads/s3/s3-client' -import { - BLOB_EXECUTION_FILES_CONFIG, - S3_EXECUTION_FILES_CONFIG, - USE_BLOB_STORAGE, - USE_S3_STORAGE, -} from '@/lib/uploads/setup' -import type { UserFile } from '@/executor/types' -import type { ExecutionContext } from './execution-files' -import { generateExecutionFileKey, generateFileId, getFileExpirationDate } from './execution-files' - -const logger = createLogger('ExecutionFileStorage') - -/** - * Upload a file to execution-scoped storage - */ -export async function uploadExecutionFile( - context: ExecutionContext, - fileBuffer: Buffer, - fileName: string, - contentType: string, - isAsync?: boolean -): Promise { - logger.info(`Uploading execution file: ${fileName} for execution ${context.executionId}`) - logger.debug(`File upload context:`, { - workspaceId: context.workspaceId, - workflowId: context.workflowId, - executionId: context.executionId, - fileName, - bufferSize: fileBuffer.length, - }) - - // Generate execution-scoped storage key - const storageKey = generateExecutionFileKey(context, fileName) - const fileId = generateFileId() - - logger.info(`Generated storage key: "${storageKey}" for file: ${fileName}`) - - // Use 10-minute expiration for async executions, 5 minutes for sync - const urlExpirationSeconds = isAsync ? 10 * 60 : 5 * 60 - - try { - let fileInfo: any - let directUrl: string | undefined - - if (USE_S3_STORAGE) { - // Upload to S3 execution files bucket with exact key (no timestamp prefix) - logger.debug( - `Uploading to S3 with key: ${storageKey}, bucket: ${S3_EXECUTION_FILES_CONFIG.bucket}` - ) - fileInfo = await uploadToS3( - fileBuffer, - storageKey, // Use storageKey as fileName - contentType, - { - bucket: S3_EXECUTION_FILES_CONFIG.bucket, - region: S3_EXECUTION_FILES_CONFIG.region, - }, - undefined, // size (will use buffer length) - true // skipTimestampPrefix = true - ) - - logger.info(`S3 upload returned key: "${fileInfo.key}" for file: ${fileName}`) - logger.info(`Original storage key was: "${storageKey}"`) - logger.info(`Keys match: ${fileInfo.key === storageKey}`) - - // Generate presigned URL for execution (5 or 10 minutes) - try { - logger.info( - `Generating presigned URL with key: "${fileInfo.key}" (expiration: ${urlExpirationSeconds / 60} minutes)` - ) - directUrl = await getPresignedUrlWithConfig( - fileInfo.key, // Use the actual uploaded key - { - bucket: S3_EXECUTION_FILES_CONFIG.bucket, - region: S3_EXECUTION_FILES_CONFIG.region, - }, - urlExpirationSeconds - ) - logger.info(`Generated presigned URL: ${directUrl}`) - } catch (error) { - logger.warn(`Failed to generate S3 presigned URL for ${fileName}:`, error) - } - } else if (USE_BLOB_STORAGE) { - // Upload to Azure Blob execution files container - fileInfo = await uploadToBlob(fileBuffer, storageKey, contentType, { - accountName: BLOB_EXECUTION_FILES_CONFIG.accountName, - accountKey: BLOB_EXECUTION_FILES_CONFIG.accountKey, - connectionString: BLOB_EXECUTION_FILES_CONFIG.connectionString, - containerName: BLOB_EXECUTION_FILES_CONFIG.containerName, - }) - - // Generate presigned URL for execution (5 or 10 minutes) - try { - directUrl = await getBlobPresignedUrlWithConfig( - fileInfo.key, // Use the actual uploaded key - { - accountName: BLOB_EXECUTION_FILES_CONFIG.accountName, - accountKey: BLOB_EXECUTION_FILES_CONFIG.accountKey, - connectionString: BLOB_EXECUTION_FILES_CONFIG.connectionString, - containerName: BLOB_EXECUTION_FILES_CONFIG.containerName, - }, - urlExpirationSeconds - ) - } catch (error) { - logger.warn(`Failed to generate Blob presigned URL for ${fileName}:`, error) - } - } else { - throw new Error('No cloud storage configured for execution files') - } - - const userFile: UserFile = { - id: fileId, - name: fileName, - size: fileBuffer.length, - type: contentType, - url: directUrl || `/api/files/serve/${fileInfo.key}`, // Use presigned URL (5 or 10 min), fallback to serve path - key: fileInfo.key, // Use the actual uploaded key from S3/Blob - uploadedAt: new Date().toISOString(), - expiresAt: getFileExpirationDate(), - } - - logger.info(`Successfully uploaded execution file: ${fileName} (${fileBuffer.length} bytes)`) - return userFile - } catch (error) { - logger.error(`Failed to upload execution file ${fileName}:`, error) - throw new Error( - `Failed to upload file: ${error instanceof Error ? error.message : 'Unknown error'}` - ) - } -} - -/** - * Download a file from execution-scoped storage - */ -export async function downloadExecutionFile(userFile: UserFile): Promise { - logger.info(`Downloading execution file: ${userFile.name}`) - - try { - let fileBuffer: Buffer - - if (USE_S3_STORAGE) { - fileBuffer = await downloadFromS3(userFile.key, { - bucket: S3_EXECUTION_FILES_CONFIG.bucket, - region: S3_EXECUTION_FILES_CONFIG.region, - }) - } else if (USE_BLOB_STORAGE) { - fileBuffer = await downloadFromBlob(userFile.key, { - accountName: BLOB_EXECUTION_FILES_CONFIG.accountName, - accountKey: BLOB_EXECUTION_FILES_CONFIG.accountKey, - connectionString: BLOB_EXECUTION_FILES_CONFIG.connectionString, - containerName: BLOB_EXECUTION_FILES_CONFIG.containerName, - }) - } else { - throw new Error('No cloud storage configured for execution files') - } - - logger.info( - `Successfully downloaded execution file: ${userFile.name} (${fileBuffer.length} bytes)` - ) - return fileBuffer - } catch (error) { - logger.error(`Failed to download execution file ${userFile.name}:`, error) - throw new Error( - `Failed to download file: ${error instanceof Error ? error.message : 'Unknown error'}` - ) - } -} - -/** - * Generate a short-lived presigned URL for file download (5 minutes) - */ -export async function generateExecutionFileDownloadUrl(userFile: UserFile): Promise { - logger.info(`Generating download URL for execution file: ${userFile.name}`) - logger.info(`File key: "${userFile.key}"`) - logger.info(`S3 bucket: ${S3_EXECUTION_FILES_CONFIG.bucket}`) - - try { - let downloadUrl: string - - if (USE_S3_STORAGE) { - downloadUrl = await getPresignedUrlWithConfig( - userFile.key, - { - bucket: S3_EXECUTION_FILES_CONFIG.bucket, - region: S3_EXECUTION_FILES_CONFIG.region, - }, - 5 * 60 // 5 minutes - ) - } else if (USE_BLOB_STORAGE) { - downloadUrl = await getBlobPresignedUrlWithConfig( - userFile.key, - { - accountName: BLOB_EXECUTION_FILES_CONFIG.accountName, - accountKey: BLOB_EXECUTION_FILES_CONFIG.accountKey, - connectionString: BLOB_EXECUTION_FILES_CONFIG.connectionString, - containerName: BLOB_EXECUTION_FILES_CONFIG.containerName, - }, - 5 * 60 // 5 minutes - ) - } else { - throw new Error('No cloud storage configured for execution files') - } - - logger.info(`Generated download URL for execution file: ${userFile.name}`) - return downloadUrl - } catch (error) { - logger.error(`Failed to generate download URL for ${userFile.name}:`, error) - throw new Error( - `Failed to generate download URL: ${error instanceof Error ? error.message : 'Unknown error'}` - ) - } -} - -/** - * Delete a file from execution-scoped storage - */ -export async function deleteExecutionFile(userFile: UserFile): Promise { - logger.info(`Deleting execution file: ${userFile.name}`) - - try { - if (USE_S3_STORAGE) { - await deleteFromS3(userFile.key, { - bucket: S3_EXECUTION_FILES_CONFIG.bucket, - region: S3_EXECUTION_FILES_CONFIG.region, - }) - } else if (USE_BLOB_STORAGE) { - await deleteFromBlob(userFile.key, { - accountName: BLOB_EXECUTION_FILES_CONFIG.accountName, - accountKey: BLOB_EXECUTION_FILES_CONFIG.accountKey, - connectionString: BLOB_EXECUTION_FILES_CONFIG.connectionString, - containerName: BLOB_EXECUTION_FILES_CONFIG.containerName, - }) - } else { - throw new Error('No cloud storage configured for execution files') - } - - logger.info(`Successfully deleted execution file: ${userFile.name}`) - } catch (error) { - logger.error(`Failed to delete execution file ${userFile.name}:`, error) - throw new Error( - `Failed to delete file: ${error instanceof Error ? error.message : 'Unknown error'}` - ) - } -}