mirror of
https://github.com/simstudioai/sim.git
synced 2026-04-06 03:00:16 -04:00
fix(files): fix json uploads, disable storage metering when billing is disabled, exclude kb uploads from storage metering, simplify serve path route (#1850)
* fix(files): fix local kb files storage to have parity with cloud storage providers * fix(files): fix json uploads, disable storage metering when billing is disabled, exclude kb uploads from storage metering, simplify serve path route * cleanup
This commit is contained in:
@@ -11,7 +11,6 @@ export const dynamic = 'force-dynamic'
|
||||
|
||||
const logger = createLogger('CareersAPI')
|
||||
|
||||
// Max file size: 10MB
|
||||
const MAX_FILE_SIZE = 10 * 1024 * 1024
|
||||
const ALLOWED_FILE_TYPES = [
|
||||
'application/pdf',
|
||||
@@ -37,7 +36,6 @@ export async function POST(request: NextRequest) {
|
||||
try {
|
||||
const formData = await request.formData()
|
||||
|
||||
// Extract form fields
|
||||
const data = {
|
||||
name: formData.get('name') as string,
|
||||
email: formData.get('email') as string,
|
||||
@@ -50,7 +48,6 @@ export async function POST(request: NextRequest) {
|
||||
message: formData.get('message') as string,
|
||||
}
|
||||
|
||||
// Extract and validate resume file
|
||||
const resumeFile = formData.get('resume') as File | null
|
||||
if (!resumeFile) {
|
||||
return NextResponse.json(
|
||||
@@ -63,7 +60,6 @@ export async function POST(request: NextRequest) {
|
||||
)
|
||||
}
|
||||
|
||||
// Validate file size
|
||||
if (resumeFile.size > MAX_FILE_SIZE) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
@@ -75,7 +71,6 @@ export async function POST(request: NextRequest) {
|
||||
)
|
||||
}
|
||||
|
||||
// Validate file type
|
||||
if (!ALLOWED_FILE_TYPES.includes(resumeFile.type)) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
@@ -87,7 +82,6 @@ export async function POST(request: NextRequest) {
|
||||
)
|
||||
}
|
||||
|
||||
// Convert file to base64 for email attachment
|
||||
const resumeBuffer = await resumeFile.arrayBuffer()
|
||||
const resumeBase64 = Buffer.from(resumeBuffer).toString('base64')
|
||||
|
||||
@@ -126,7 +120,6 @@ export async function POST(request: NextRequest) {
|
||||
})
|
||||
)
|
||||
|
||||
// Send email with resume attachment
|
||||
const careersEmailResult = await sendEmail({
|
||||
to: 'careers@sim.ai',
|
||||
subject: `New Career Application: ${validatedData.name} - ${validatedData.position}`,
|
||||
|
||||
@@ -98,7 +98,6 @@ function extractWorkspaceIdFromKey(key: string): string | null {
|
||||
* Verify file access based on file path patterns and metadata
|
||||
* @param cloudKey The file key/path (e.g., "workspace_id/workflow_id/execution_id/filename" or "kb/filename")
|
||||
* @param userId The authenticated user ID
|
||||
* @param bucketType Optional bucket type (e.g., 'copilot', 'execution-files')
|
||||
* @param customConfig Optional custom storage configuration
|
||||
* @param context Optional explicit storage context
|
||||
* @param isLocal Optional flag indicating if this is local storage
|
||||
@@ -107,7 +106,6 @@ function extractWorkspaceIdFromKey(key: string): string | null {
|
||||
export async function verifyFileAccess(
|
||||
cloudKey: string,
|
||||
userId: string,
|
||||
bucketType?: string | null,
|
||||
customConfig?: StorageConfig,
|
||||
context?: StorageContext,
|
||||
isLocal?: boolean
|
||||
@@ -128,12 +126,12 @@ export async function verifyFileAccess(
|
||||
}
|
||||
|
||||
// 2. Execution files: workspace_id/workflow_id/execution_id/filename
|
||||
if (inferredContext === 'execution' || (!context && isExecutionFile(cloudKey, bucketType))) {
|
||||
if (inferredContext === 'execution') {
|
||||
return await verifyExecutionFileAccess(cloudKey, userId, customConfig)
|
||||
}
|
||||
|
||||
// 3. Copilot files: Check database first, then metadata, then path pattern (legacy)
|
||||
if (inferredContext === 'copilot' || bucketType === 'copilot') {
|
||||
if (inferredContext === 'copilot') {
|
||||
return await verifyCopilotFileAccess(cloudKey, userId, customConfig)
|
||||
}
|
||||
|
||||
@@ -223,18 +221,6 @@ async function verifyWorkspaceFileAccess(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if file is an execution file based on path pattern
|
||||
* Execution files have format: workspace_id/workflow_id/execution_id/filename
|
||||
*/
|
||||
function isExecutionFile(cloudKey: string, bucketType?: string | null): boolean {
|
||||
if (bucketType === 'execution-files' || bucketType === 'execution') {
|
||||
return true
|
||||
}
|
||||
|
||||
return inferContextFromKey(cloudKey) === 'execution'
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify access to execution files
|
||||
* Modern format: execution/workspace_id/workflow_id/execution_id/filename
|
||||
@@ -590,7 +576,7 @@ export async function authorizeFileAccess(
|
||||
storageConfig?: StorageConfig,
|
||||
isLocal?: boolean
|
||||
): Promise<AuthorizationResult> {
|
||||
const granted = await verifyFileAccess(key, userId, null, storageConfig, context, isLocal)
|
||||
const granted = await verifyFileAccess(key, userId, storageConfig, context, isLocal)
|
||||
|
||||
if (granted) {
|
||||
let workspaceId: string | undefined
|
||||
|
||||
@@ -59,7 +59,7 @@ describe('File Delete API Route', () => {
|
||||
})
|
||||
|
||||
const req = createMockRequest('POST', {
|
||||
filePath: '/api/files/serve/s3/workspace/test-workspace-id/1234567890-test-file.txt',
|
||||
filePath: '/api/files/serve/workspace/test-workspace-id/1234567890-test-file.txt',
|
||||
})
|
||||
|
||||
const { POST } = await import('@/app/api/files/delete/route')
|
||||
@@ -85,7 +85,7 @@ describe('File Delete API Route', () => {
|
||||
})
|
||||
|
||||
const req = createMockRequest('POST', {
|
||||
filePath: '/api/files/serve/blob/workspace/test-workspace-id/1234567890-test-document.pdf',
|
||||
filePath: '/api/files/serve/workspace/test-workspace-id/1234567890-test-document.pdf',
|
||||
})
|
||||
|
||||
const { POST } = await import('@/app/api/files/delete/route')
|
||||
|
||||
@@ -13,9 +13,6 @@ import {
|
||||
extractFilename,
|
||||
FileNotFoundError,
|
||||
InvalidRequestError,
|
||||
isBlobPath,
|
||||
isCloudPath,
|
||||
isS3Path,
|
||||
} from '@/app/api/files/utils'
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
@@ -54,9 +51,8 @@ export async function POST(request: NextRequest) {
|
||||
const hasAccess = await verifyFileAccess(
|
||||
key,
|
||||
userId,
|
||||
null,
|
||||
undefined,
|
||||
storageContext,
|
||||
undefined, // customConfig
|
||||
storageContext, // context
|
||||
!hasCloudStorage() // isLocal
|
||||
)
|
||||
|
||||
@@ -99,15 +95,11 @@ export async function POST(request: NextRequest) {
|
||||
* Extract storage key from file path
|
||||
*/
|
||||
function extractStorageKeyFromPath(filePath: string): string {
|
||||
if (isS3Path(filePath) || isBlobPath(filePath) || filePath.startsWith('/api/files/serve/')) {
|
||||
if (filePath.startsWith('/api/files/serve/')) {
|
||||
return extractStorageKey(filePath)
|
||||
}
|
||||
|
||||
if (!isCloudPath(filePath)) {
|
||||
return extractFilename(filePath)
|
||||
}
|
||||
|
||||
return filePath
|
||||
return extractFilename(filePath)
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -51,10 +51,9 @@ export async function POST(request: NextRequest) {
|
||||
const hasAccess = await verifyFileAccess(
|
||||
key,
|
||||
userId,
|
||||
isExecutionFile ? 'execution' : null,
|
||||
undefined,
|
||||
storageContext,
|
||||
!hasCloudStorage()
|
||||
undefined, // customConfig
|
||||
storageContext, // context
|
||||
!hasCloudStorage() // isLocal
|
||||
)
|
||||
|
||||
if (!hasAccess) {
|
||||
|
||||
@@ -427,9 +427,8 @@ async function handleCloudFile(
|
||||
const hasAccess = await verifyFileAccess(
|
||||
cloudKey,
|
||||
userId,
|
||||
null,
|
||||
undefined,
|
||||
context,
|
||||
undefined, // customConfig
|
||||
context, // context
|
||||
false // isLocal
|
||||
)
|
||||
|
||||
@@ -534,9 +533,8 @@ async function handleLocalFile(
|
||||
const hasAccess = await verifyFileAccess(
|
||||
filename,
|
||||
userId,
|
||||
null,
|
||||
undefined,
|
||||
context,
|
||||
undefined, // customConfig
|
||||
context, // context
|
||||
true // isLocal
|
||||
)
|
||||
|
||||
@@ -812,11 +810,7 @@ function prettySize(bytes: number): string {
|
||||
* Create a formatted message for PDF content
|
||||
*/
|
||||
function createPdfFallbackMessage(pageCount: number, size: number, path?: string): string {
|
||||
const formattedPath = path
|
||||
? path.includes('/api/files/serve/s3/')
|
||||
? `S3 path: ${decodeURIComponent(path.split('/api/files/serve/s3/')[1])}`
|
||||
: `Local path: ${path}`
|
||||
: 'Unknown path'
|
||||
const formattedPath = path || 'Unknown path'
|
||||
|
||||
return `PDF document - ${pageCount} page(s), ${prettySize(size)}
|
||||
Path: ${formattedPath}
|
||||
@@ -834,12 +828,8 @@ function createPdfFailureMessage(
|
||||
path: string,
|
||||
error: string
|
||||
): string {
|
||||
const formattedPath = path.includes('/api/files/serve/s3/')
|
||||
? `S3 path: ${decodeURIComponent(path.split('/api/files/serve/s3/')[1])}`
|
||||
: `Local path: ${path}`
|
||||
|
||||
return `PDF document - Processing failed, ${prettySize(size)}
|
||||
Path: ${formattedPath}
|
||||
Path: ${path}
|
||||
Error: ${error}
|
||||
|
||||
This file appears to be a PDF document that could not be processed.
|
||||
|
||||
@@ -54,8 +54,6 @@ describe('File Serve API Route', () => {
|
||||
})
|
||||
}),
|
||||
getContentType: vi.fn().mockReturnValue('text/plain'),
|
||||
isS3Path: vi.fn().mockReturnValue(false),
|
||||
isBlobPath: vi.fn().mockReturnValue(false),
|
||||
extractStorageKey: vi.fn().mockImplementation((path) => path.split('/').pop()),
|
||||
extractFilename: vi.fn().mockImplementation((path) => path.split('/').pop()),
|
||||
findLocalFile: vi.fn().mockReturnValue('/test/uploads/test-file.txt'),
|
||||
@@ -112,8 +110,6 @@ describe('File Serve API Route', () => {
|
||||
})
|
||||
}),
|
||||
getContentType: vi.fn().mockReturnValue('text/plain'),
|
||||
isS3Path: vi.fn().mockReturnValue(false),
|
||||
isBlobPath: vi.fn().mockReturnValue(false),
|
||||
extractStorageKey: vi.fn().mockImplementation((path) => path.split('/').pop()),
|
||||
extractFilename: vi.fn().mockImplementation((path) => path.split('/').pop()),
|
||||
findLocalFile: vi.fn().mockReturnValue('/test/uploads/nested/path/file.txt'),
|
||||
@@ -203,17 +199,15 @@ describe('File Serve API Route', () => {
|
||||
})
|
||||
}),
|
||||
getContentType: vi.fn().mockReturnValue('image/png'),
|
||||
isS3Path: vi.fn().mockReturnValue(false),
|
||||
isBlobPath: vi.fn().mockReturnValue(false),
|
||||
extractStorageKey: vi.fn().mockImplementation((path) => path.split('/').pop()),
|
||||
extractFilename: vi.fn().mockImplementation((path) => path.split('/').pop()),
|
||||
findLocalFile: vi.fn().mockReturnValue('/test/uploads/test-file.txt'),
|
||||
}))
|
||||
|
||||
const req = new NextRequest(
|
||||
'http://localhost:3000/api/files/serve/s3/workspace/test-workspace-id/1234567890-image.png'
|
||||
'http://localhost:3000/api/files/serve/workspace/test-workspace-id/1234567890-image.png'
|
||||
)
|
||||
const params = { path: ['s3', 'workspace', 'test-workspace-id', '1234567890-image.png'] }
|
||||
const params = { path: ['workspace', 'test-workspace-id', '1234567890-image.png'] }
|
||||
const { GET } = await import('@/app/api/files/serve/[...path]/route')
|
||||
|
||||
const response = await GET(req, { params: Promise.resolve(params) })
|
||||
@@ -262,8 +256,6 @@ describe('File Serve API Route', () => {
|
||||
})
|
||||
}),
|
||||
getContentType: vi.fn().mockReturnValue('text/plain'),
|
||||
isS3Path: vi.fn().mockReturnValue(false),
|
||||
isBlobPath: vi.fn().mockReturnValue(false),
|
||||
extractStorageKey: vi.fn(),
|
||||
extractFilename: vi.fn(),
|
||||
findLocalFile: vi.fn().mockReturnValue(null),
|
||||
|
||||
@@ -62,11 +62,11 @@ export async function GET(
|
||||
|
||||
const userId = authResult.userId
|
||||
|
||||
if (isUsingCloudStorage() || isCloudPath) {
|
||||
return await handleCloudProxy(cloudKey, userId, contextParam, legacyBucketType)
|
||||
if (isUsingCloudStorage()) {
|
||||
return await handleCloudProxy(cloudKey, userId, contextParam)
|
||||
}
|
||||
|
||||
return await handleLocalFile(fullPath, userId)
|
||||
return await handleLocalFile(cloudKey, userId)
|
||||
} catch (error) {
|
||||
logger.error('Error serving file:', error)
|
||||
|
||||
@@ -87,10 +87,9 @@ async function handleLocalFile(filename: string, userId: string): Promise<NextRe
|
||||
const hasAccess = await verifyFileAccess(
|
||||
filename,
|
||||
userId,
|
||||
null,
|
||||
undefined,
|
||||
contextParam,
|
||||
true // isLocal = true
|
||||
undefined, // customConfig
|
||||
contextParam, // context
|
||||
true // isLocal
|
||||
)
|
||||
|
||||
if (!hasAccess) {
|
||||
@@ -123,8 +122,7 @@ async function handleLocalFile(filename: string, userId: string): Promise<NextRe
|
||||
async function handleCloudProxy(
|
||||
cloudKey: string,
|
||||
userId: string,
|
||||
contextParam?: string | null,
|
||||
legacyBucketType?: string | null
|
||||
contextParam?: string | null
|
||||
): Promise<NextResponse> {
|
||||
try {
|
||||
let context: StorageContext
|
||||
@@ -132,9 +130,6 @@ async function handleCloudProxy(
|
||||
if (contextParam) {
|
||||
context = contextParam as StorageContext
|
||||
logger.info(`Using explicit context: ${context} for key: ${cloudKey}`)
|
||||
} else if (legacyBucketType === 'copilot') {
|
||||
context = 'copilot'
|
||||
logger.info(`Using legacy bucket parameter for copilot context: ${cloudKey}`)
|
||||
} else {
|
||||
context = inferContextFromKey(cloudKey)
|
||||
logger.info(`Inferred context: ${context} from key pattern: ${cloudKey}`)
|
||||
@@ -143,10 +138,9 @@ async function handleCloudProxy(
|
||||
const hasAccess = await verifyFileAccess(
|
||||
cloudKey,
|
||||
userId,
|
||||
legacyBucketType || null,
|
||||
undefined,
|
||||
context,
|
||||
false // isLocal = false
|
||||
undefined, // customConfig
|
||||
context, // context
|
||||
false // isLocal
|
||||
)
|
||||
|
||||
if (!hasAccess) {
|
||||
|
||||
@@ -137,6 +137,10 @@ export async function POST(request: NextRequest) {
|
||||
|
||||
logger.info(`Uploading knowledge-base file: ${originalName}`)
|
||||
|
||||
const timestamp = Date.now()
|
||||
const safeFileName = originalName.replace(/\s+/g, '-')
|
||||
const storageKey = `kb/${timestamp}-${safeFileName}`
|
||||
|
||||
const metadata: Record<string, string> = {
|
||||
originalName: originalName,
|
||||
uploadedAt: new Date().toISOString(),
|
||||
@@ -150,9 +154,11 @@ export async function POST(request: NextRequest) {
|
||||
|
||||
const fileInfo = await storageService.uploadFile({
|
||||
file: buffer,
|
||||
fileName: originalName,
|
||||
fileName: storageKey,
|
||||
contentType: file.type,
|
||||
context: 'knowledge-base',
|
||||
preserveKey: true,
|
||||
customKey: storageKey,
|
||||
metadata,
|
||||
})
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ import { join, resolve, sep } from 'path'
|
||||
import { NextResponse } from 'next/server'
|
||||
import { createLogger } from '@/lib/logs/console/logger'
|
||||
import { UPLOAD_DIR } from '@/lib/uploads/config'
|
||||
import { sanitizeFileKey } from '@/lib/uploads/utils/file-utils'
|
||||
|
||||
const logger = createLogger('FilesUtils')
|
||||
|
||||
@@ -37,7 +38,6 @@ export class InvalidRequestError extends Error {
|
||||
}
|
||||
|
||||
export const contentTypeMap: Record<string, string> = {
|
||||
// Text formats
|
||||
txt: 'text/plain',
|
||||
csv: 'text/csv',
|
||||
json: 'application/json',
|
||||
@@ -47,26 +47,20 @@ export const contentTypeMap: Record<string, string> = {
|
||||
css: 'text/css',
|
||||
js: 'application/javascript',
|
||||
ts: 'application/typescript',
|
||||
// Document formats
|
||||
pdf: 'application/pdf',
|
||||
googleDoc: 'application/vnd.google-apps.document',
|
||||
doc: 'application/msword',
|
||||
docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
||||
// Spreadsheet formats
|
||||
xls: 'application/vnd.ms-excel',
|
||||
xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
||||
googleSheet: 'application/vnd.google-apps.spreadsheet',
|
||||
// Presentation formats
|
||||
ppt: 'application/vnd.ms-powerpoint',
|
||||
pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
||||
// Image formats
|
||||
png: 'image/png',
|
||||
jpg: 'image/jpeg',
|
||||
jpeg: 'image/jpeg',
|
||||
gif: 'image/gif',
|
||||
// Archive formats
|
||||
zip: 'application/zip',
|
||||
// Folder format
|
||||
googleFolder: 'application/vnd.google-apps.folder',
|
||||
}
|
||||
|
||||
@@ -90,18 +84,6 @@ export function getContentType(filename: string): string {
|
||||
return contentTypeMap[extension] || 'application/octet-stream'
|
||||
}
|
||||
|
||||
export function isS3Path(path: string): boolean {
|
||||
return path.includes('/api/files/serve/s3/')
|
||||
}
|
||||
|
||||
export function isBlobPath(path: string): boolean {
|
||||
return path.includes('/api/files/serve/blob/')
|
||||
}
|
||||
|
||||
export function isCloudPath(path: string): boolean {
|
||||
return isS3Path(path) || isBlobPath(path)
|
||||
}
|
||||
|
||||
export function extractFilename(path: string): string {
|
||||
let filename: string
|
||||
|
||||
@@ -142,29 +124,48 @@ function sanitizeFilename(filename: string): string {
|
||||
throw new Error('Invalid filename provided')
|
||||
}
|
||||
|
||||
const sanitized = filename.replace(/\.\./g, '').replace(/[/\\]/g, '').replace(/^\./g, '').trim()
|
||||
|
||||
if (!sanitized || sanitized.length === 0) {
|
||||
throw new Error('Invalid or empty filename after sanitization')
|
||||
if (!filename.includes('/')) {
|
||||
throw new Error('File key must include a context prefix (e.g., kb/, workspace/, execution/)')
|
||||
}
|
||||
|
||||
if (
|
||||
sanitized.includes(':') ||
|
||||
sanitized.includes('|') ||
|
||||
sanitized.includes('?') ||
|
||||
sanitized.includes('*') ||
|
||||
sanitized.includes('\x00') ||
|
||||
/[\x00-\x1F\x7F]/.test(sanitized)
|
||||
) {
|
||||
throw new Error('Filename contains invalid characters')
|
||||
}
|
||||
const segments = filename.split('/')
|
||||
|
||||
return sanitized
|
||||
const sanitizedSegments = segments.map((segment) => {
|
||||
if (segment === '..' || segment === '.') {
|
||||
throw new Error('Path traversal detected')
|
||||
}
|
||||
|
||||
const sanitized = segment.replace(/\.\./g, '').replace(/[\\]/g, '').replace(/^\./g, '').trim()
|
||||
|
||||
if (!sanitized) {
|
||||
throw new Error('Invalid or empty path segment after sanitization')
|
||||
}
|
||||
|
||||
if (
|
||||
sanitized.includes(':') ||
|
||||
sanitized.includes('|') ||
|
||||
sanitized.includes('?') ||
|
||||
sanitized.includes('*') ||
|
||||
sanitized.includes('\x00') ||
|
||||
/[\x00-\x1F\x7F]/.test(sanitized)
|
||||
) {
|
||||
throw new Error('Path segment contains invalid characters')
|
||||
}
|
||||
|
||||
return sanitized
|
||||
})
|
||||
|
||||
return sanitizedSegments.join(sep)
|
||||
}
|
||||
|
||||
export function findLocalFile(filename: string): string | null {
|
||||
try {
|
||||
const sanitizedFilename = sanitizeFilename(filename)
|
||||
const sanitizedFilename = sanitizeFileKey(filename)
|
||||
|
||||
// Reject if sanitized filename is empty or only contains path separators/dots
|
||||
if (!sanitizedFilename || !sanitizedFilename.trim() || /^[/\\.\s]+$/.test(sanitizedFilename)) {
|
||||
return null
|
||||
}
|
||||
|
||||
const possiblePaths = [
|
||||
join(UPLOAD_DIR, sanitizedFilename),
|
||||
@@ -175,8 +176,9 @@ export function findLocalFile(filename: string): string | null {
|
||||
const resolvedPath = resolve(path)
|
||||
const allowedDirs = [resolve(UPLOAD_DIR), resolve(process.cwd(), 'uploads')]
|
||||
|
||||
// Must be within allowed directory but NOT the directory itself
|
||||
const isWithinAllowedDir = allowedDirs.some(
|
||||
(allowedDir) => resolvedPath.startsWith(allowedDir + sep) || resolvedPath === allowedDir
|
||||
(allowedDir) => resolvedPath.startsWith(allowedDir + sep) && resolvedPath !== allowedDir
|
||||
)
|
||||
|
||||
if (!isWithinAllowedDir) {
|
||||
@@ -233,7 +235,6 @@ function getSecureFileHeaders(filename: string, originalContentType: string) {
|
||||
}
|
||||
|
||||
function encodeFilenameForHeader(storageKey: string): string {
|
||||
// Extract just the filename from the storage key (last segment after /)
|
||||
const filename = storageKey.split('/').pop() || storageKey
|
||||
|
||||
const hasNonAscii = /[^\x00-\x7F]/.test(filename)
|
||||
|
||||
@@ -123,6 +123,8 @@ export async function GET(request: NextRequest) {
|
||||
fileName: enhancedLogKey,
|
||||
contentType: 'application/json',
|
||||
context: 'logs',
|
||||
preserveKey: true,
|
||||
customKey: enhancedLogKey,
|
||||
metadata: {
|
||||
logId: String(log.id),
|
||||
workflowId: String(log.workflowId),
|
||||
|
||||
@@ -62,9 +62,8 @@ export async function POST(request: NextRequest) {
|
||||
const hasAccess = await verifyFileAccess(
|
||||
storageKey,
|
||||
userId,
|
||||
null,
|
||||
undefined,
|
||||
context,
|
||||
undefined, // customConfig
|
||||
context, // context
|
||||
false // isLocal
|
||||
)
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
TableHeader,
|
||||
TableRow,
|
||||
} from '@/components/ui/table'
|
||||
import { getEnv, isTruthy } from '@/lib/env'
|
||||
import { createLogger } from '@/lib/logs/console/logger'
|
||||
import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace'
|
||||
import { getFileExtension } from '@/lib/uploads/utils/file-utils'
|
||||
@@ -22,6 +23,7 @@ import { useUserPermissions } from '@/hooks/use-user-permissions'
|
||||
import { useWorkspacePermissions } from '@/hooks/use-workspace-permissions'
|
||||
|
||||
const logger = createLogger('FileUploadsSettings')
|
||||
const isBillingEnabled = isTruthy(getEnv('NEXT_PUBLIC_BILLING_ENABLED'))
|
||||
|
||||
const SUPPORTED_EXTENSIONS = [
|
||||
'pdf',
|
||||
@@ -36,8 +38,12 @@ const SUPPORTED_EXTENSIONS = [
|
||||
'htm',
|
||||
'pptx',
|
||||
'ppt',
|
||||
'json',
|
||||
'yaml',
|
||||
'yml',
|
||||
] as const
|
||||
const ACCEPT_ATTR = '.pdf,.csv,.doc,.docx,.txt,.md,.xlsx,.xls,.html,.htm,.pptx,.ppt'
|
||||
const ACCEPT_ATTR =
|
||||
'.pdf,.csv,.doc,.docx,.txt,.md,.xlsx,.xls,.html,.htm,.pptx,.ppt,.json,.yaml,.yml'
|
||||
|
||||
interface StorageInfo {
|
||||
usedBytes: number
|
||||
@@ -45,11 +51,6 @@ interface StorageInfo {
|
||||
percentUsed: number
|
||||
}
|
||||
|
||||
interface UsageData {
|
||||
plan: string
|
||||
storage: StorageInfo
|
||||
}
|
||||
|
||||
export function FileUploads() {
|
||||
const params = useParams()
|
||||
const workspaceId = params?.workspaceId as string
|
||||
@@ -87,6 +88,11 @@ export function FileUploads() {
|
||||
}
|
||||
|
||||
const loadStorageInfo = async () => {
|
||||
if (!isBillingEnabled) {
|
||||
setStorageLoading(false)
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
setStorageLoading(true)
|
||||
const response = await fetch('/api/users/me/usage-limits')
|
||||
@@ -158,7 +164,9 @@ export function FileUploads() {
|
||||
}
|
||||
|
||||
await loadFiles()
|
||||
await loadStorageInfo()
|
||||
if (isBillingEnabled) {
|
||||
await loadStorageInfo()
|
||||
}
|
||||
if (unsupported.length) {
|
||||
lastError = `Unsupported file type: ${unsupported.join(', ')}`
|
||||
}
|
||||
@@ -193,7 +201,7 @@ export function FileUploads() {
|
||||
|
||||
setFiles((prev) => prev.filter((f) => f.id !== file.id))
|
||||
|
||||
if (storageInfo) {
|
||||
if (isBillingEnabled && storageInfo) {
|
||||
const newUsedBytes = Math.max(0, storageInfo.usedBytes - file.size)
|
||||
const newPercentUsed = (newUsedBytes / storageInfo.limitBytes) * 100
|
||||
setStorageInfo({
|
||||
@@ -217,7 +225,9 @@ export function FileUploads() {
|
||||
} catch (error) {
|
||||
logger.error('Error deleting file:', error)
|
||||
await loadFiles()
|
||||
await loadStorageInfo()
|
||||
if (isBillingEnabled) {
|
||||
await loadStorageInfo()
|
||||
}
|
||||
} finally {
|
||||
setDeletingFileId(null)
|
||||
}
|
||||
@@ -283,31 +293,35 @@ export function FileUploads() {
|
||||
/>
|
||||
</div>
|
||||
<div className='flex items-center gap-3'>
|
||||
{storageLoading ? (
|
||||
<Skeleton className='h-4 w-32' />
|
||||
) : storageInfo ? (
|
||||
<div className='flex flex-col items-end gap-1'>
|
||||
<div className='flex items-center gap-2 text-sm'>
|
||||
<span
|
||||
className={cn(
|
||||
'font-medium',
|
||||
planName === 'free' ? 'text-foreground' : GRADIENT_TEXT_STYLES
|
||||
)}
|
||||
>
|
||||
{displayPlanName}
|
||||
</span>
|
||||
<span className='text-muted-foreground tabular-nums'>
|
||||
{formatStorageSize(storageInfo.usedBytes)} /{' '}
|
||||
{formatStorageSize(storageInfo.limitBytes)}
|
||||
</span>
|
||||
</div>
|
||||
<Progress
|
||||
value={Math.min(storageInfo.percentUsed, 100)}
|
||||
className='h-1 w-full'
|
||||
indicatorClassName='bg-black dark:bg-white'
|
||||
/>
|
||||
</div>
|
||||
) : null}
|
||||
{isBillingEnabled && (
|
||||
<>
|
||||
{storageLoading ? (
|
||||
<Skeleton className='h-4 w-32' />
|
||||
) : storageInfo ? (
|
||||
<div className='flex flex-col items-end gap-1'>
|
||||
<div className='flex items-center gap-2 text-sm'>
|
||||
<span
|
||||
className={cn(
|
||||
'font-medium',
|
||||
planName === 'free' ? 'text-foreground' : GRADIENT_TEXT_STYLES
|
||||
)}
|
||||
>
|
||||
{displayPlanName}
|
||||
</span>
|
||||
<span className='text-muted-foreground tabular-nums'>
|
||||
{formatStorageSize(storageInfo.usedBytes)} /{' '}
|
||||
{formatStorageSize(storageInfo.limitBytes)}
|
||||
</span>
|
||||
</div>
|
||||
<Progress
|
||||
value={Math.min(storageInfo.percentUsed, 100)}
|
||||
className='h-1 w-full'
|
||||
indicatorClassName='bg-black dark:bg-white'
|
||||
/>
|
||||
</div>
|
||||
) : null}
|
||||
</>
|
||||
)}
|
||||
{userPermissions.canEdit && (
|
||||
<div className='flex items-center'>
|
||||
<input
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
import { organization, subscription, userStats } from '@sim/db/schema'
|
||||
import { eq } from 'drizzle-orm'
|
||||
import { getEnv } from '@/lib/env'
|
||||
import { isBillingEnabled } from '@/lib/environment'
|
||||
import { createLogger } from '@/lib/logs/console/logger'
|
||||
|
||||
const logger = createLogger('StorageLimits')
|
||||
@@ -156,11 +157,20 @@ export async function getUserStorageUsage(userId: string): Promise<number> {
|
||||
|
||||
/**
|
||||
* Check if user has storage quota available
|
||||
* Always allows uploads when billing is disabled
|
||||
*/
|
||||
export async function checkStorageQuota(
|
||||
userId: string,
|
||||
additionalBytes: number
|
||||
): Promise<{ allowed: boolean; currentUsage: number; limit: number; error?: string }> {
|
||||
if (!isBillingEnabled) {
|
||||
return {
|
||||
allowed: true,
|
||||
currentUsage: 0,
|
||||
limit: Number.MAX_SAFE_INTEGER,
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const [currentUsage, limit] = await Promise.all([
|
||||
getUserStorageUsage(userId),
|
||||
|
||||
@@ -1,19 +1,27 @@
|
||||
/**
|
||||
* Storage usage tracking
|
||||
* Updates storage_used_bytes for users and organizations
|
||||
* Only tracks when billing is enabled
|
||||
*/
|
||||
|
||||
import { db } from '@sim/db'
|
||||
import { organization, userStats } from '@sim/db/schema'
|
||||
import { eq, sql } from 'drizzle-orm'
|
||||
import { isBillingEnabled } from '@/lib/environment'
|
||||
import { createLogger } from '@/lib/logs/console/logger'
|
||||
|
||||
const logger = createLogger('StorageTracking')
|
||||
|
||||
/**
|
||||
* Increment storage usage after successful file upload
|
||||
* Only tracks if billing is enabled
|
||||
*/
|
||||
export async function incrementStorageUsage(userId: string, bytes: number): Promise<void> {
|
||||
if (!isBillingEnabled) {
|
||||
logger.debug('Billing disabled, skipping storage increment')
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
// Check if user is in a team/enterprise org
|
||||
const { getHighestPrioritySubscription } = await import('@/lib/billing/core/subscription')
|
||||
@@ -48,8 +56,14 @@ export async function incrementStorageUsage(userId: string, bytes: number): Prom
|
||||
|
||||
/**
|
||||
* Decrement storage usage after file deletion
|
||||
* Only tracks if billing is enabled
|
||||
*/
|
||||
export async function decrementStorageUsage(userId: string, bytes: number): Promise<void> {
|
||||
if (!isBillingEnabled) {
|
||||
logger.debug('Billing disabled, skipping storage decrement')
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
// Check if user is in a team/enterprise org
|
||||
const { getHighestPrioritySubscription } = await import('@/lib/billing/core/subscription')
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import { readFile } from 'fs/promises'
|
||||
import { PDFParse } from 'pdf-parse'
|
||||
import type { FileParseResult, FileParser } from '@/lib/file-parsers/types'
|
||||
import { createLogger } from '@/lib/logs/console/logger'
|
||||
|
||||
@@ -29,6 +28,8 @@ export class PdfParser implements FileParser {
|
||||
try {
|
||||
logger.info('Starting to parse buffer, size:', dataBuffer.length)
|
||||
|
||||
const { PDFParse } = await import('pdf-parse')
|
||||
|
||||
const parser = new PDFParse({ data: dataBuffer })
|
||||
const textResult = await parser.getText()
|
||||
const infoResult = await parser.getInfo()
|
||||
@@ -41,7 +42,6 @@ export class PdfParser implements FileParser {
|
||||
textResult.text.length
|
||||
)
|
||||
|
||||
// Remove null bytes from content (PostgreSQL JSONB doesn't allow them)
|
||||
const cleanContent = textResult.text.replace(/\u0000/g, '')
|
||||
|
||||
return {
|
||||
|
||||
@@ -189,11 +189,17 @@ async function handleFileForOCR(
|
||||
...(workspaceId && { workspaceId }),
|
||||
}
|
||||
|
||||
const timestamp = Date.now()
|
||||
const uniqueId = Math.random().toString(36).substring(2, 9)
|
||||
const safeFileName = filename.replace(/[^a-zA-Z0-9.-]/g, '_')
|
||||
const customKey = `kb/${timestamp}-${uniqueId}-${safeFileName}`
|
||||
|
||||
const cloudResult = await StorageService.uploadFile({
|
||||
file: buffer,
|
||||
fileName: filename,
|
||||
contentType: mimeType,
|
||||
context: 'knowledge-base',
|
||||
customKey,
|
||||
metadata,
|
||||
})
|
||||
|
||||
|
||||
@@ -3,11 +3,6 @@ import { db } from '@sim/db'
|
||||
import { document, embedding, knowledgeBase, knowledgeBaseTagDefinitions } from '@sim/db/schema'
|
||||
import { tasks } from '@trigger.dev/sdk'
|
||||
import { and, asc, desc, eq, inArray, isNull, sql } from 'drizzle-orm'
|
||||
import {
|
||||
checkStorageQuota,
|
||||
decrementStorageUsage,
|
||||
incrementStorageUsage,
|
||||
} from '@/lib/billing/storage'
|
||||
import { generateEmbeddings } from '@/lib/embeddings/utils'
|
||||
import { env } from '@/lib/env'
|
||||
import { getSlotsForFieldType, type TAG_SLOT_CONFIG } from '@/lib/knowledge/consts'
|
||||
@@ -696,13 +691,6 @@ export async function createDocumentRecords(
|
||||
if (kb.length === 0) {
|
||||
throw new Error('Knowledge base not found')
|
||||
}
|
||||
|
||||
// Always meter the knowledge base owner
|
||||
const quotaCheck = await checkStorageQuota(kb[0].userId, totalSize)
|
||||
|
||||
if (!quotaCheck.allowed) {
|
||||
throw new Error(quotaCheck.error || 'Storage limit exceeded')
|
||||
}
|
||||
}
|
||||
|
||||
return await db.transaction(async (tx) => {
|
||||
@@ -787,21 +775,6 @@ export async function createDocumentRecords(
|
||||
.from(knowledgeBase)
|
||||
.where(eq(knowledgeBase.id, knowledgeBaseId))
|
||||
.limit(1)
|
||||
|
||||
if (kb.length > 0) {
|
||||
// Always meter the knowledge base owner
|
||||
try {
|
||||
await incrementStorageUsage(kb[0].userId, totalSize)
|
||||
logger.info(
|
||||
`[${requestId}] Updated knowledge base owner storage usage for ${totalSize} bytes`
|
||||
)
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
`[${requestId}] Failed to update knowledge base owner storage usage:`,
|
||||
error
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1037,13 +1010,6 @@ export async function createSingleDocument(
|
||||
if (kb.length === 0) {
|
||||
throw new Error('Knowledge base not found')
|
||||
}
|
||||
|
||||
// Always meter the knowledge base owner
|
||||
const quotaCheck = await checkStorageQuota(kb[0].userId, documentData.fileSize)
|
||||
|
||||
if (!quotaCheck.allowed) {
|
||||
throw new Error(quotaCheck.error || 'Storage limit exceeded')
|
||||
}
|
||||
}
|
||||
|
||||
const documentId = randomUUID()
|
||||
@@ -1103,18 +1069,6 @@ export async function createSingleDocument(
|
||||
.from(knowledgeBase)
|
||||
.where(eq(knowledgeBase.id, knowledgeBaseId))
|
||||
.limit(1)
|
||||
|
||||
if (kb.length > 0) {
|
||||
// Always meter the knowledge base owner
|
||||
try {
|
||||
await incrementStorageUsage(kb[0].userId, documentData.fileSize)
|
||||
logger.info(
|
||||
`[${requestId}] Updated knowledge base owner storage usage for ${documentData.fileSize} bytes`
|
||||
)
|
||||
} catch (error) {
|
||||
logger.error(`[${requestId}] Failed to update knowledge base owner storage usage:`, error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return newDocument as {
|
||||
@@ -1226,28 +1180,6 @@ export async function bulkDocumentOperation(
|
||||
)
|
||||
)
|
||||
.returning({ id: document.id, deletedAt: document.deletedAt })
|
||||
|
||||
// Decrement storage usage tracking
|
||||
if (userId && totalSize > 0) {
|
||||
// Get knowledge base owner
|
||||
const kb = await db
|
||||
.select({ userId: knowledgeBase.userId })
|
||||
.from(knowledgeBase)
|
||||
.where(eq(knowledgeBase.id, knowledgeBaseId))
|
||||
.limit(1)
|
||||
|
||||
if (kb.length > 0) {
|
||||
// Always meter the knowledge base owner
|
||||
try {
|
||||
await decrementStorageUsage(kb[0].userId, totalSize)
|
||||
logger.info(
|
||||
`[${requestId}] Updated knowledge base owner storage usage for -${totalSize} bytes`
|
||||
)
|
||||
} catch (error) {
|
||||
logger.error(`[${requestId}] Failed to update knowledge base owner storage usage:`, error)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Handle bulk enable/disable
|
||||
const enabled = operation === 'enable'
|
||||
|
||||
@@ -7,7 +7,6 @@ import { getStorageProvider, USE_BLOB_STORAGE, USE_S3_STORAGE } from '@/lib/uplo
|
||||
|
||||
const logger = createLogger('UploadsSetup')
|
||||
|
||||
// Server-only upload directory path
|
||||
const PROJECT_ROOT = path.resolve(process.cwd())
|
||||
export const UPLOAD_DIR_SERVER = join(PROJECT_ROOT, 'uploads')
|
||||
|
||||
|
||||
@@ -1,244 +1,7 @@
|
||||
import { createLogger } from '@/lib/logs/console/logger'
|
||||
import { USE_BLOB_STORAGE, USE_S3_STORAGE } from '@/lib/uploads/config'
|
||||
import type { BlobConfig } from '@/lib/uploads/providers/blob/types'
|
||||
import type { S3Config } from '@/lib/uploads/providers/s3/types'
|
||||
import type { FileInfo, StorageConfig } from '@/lib/uploads/shared/types'
|
||||
import { sanitizeFileKey } from '@/lib/uploads/utils/file-utils'
|
||||
import type { StorageConfig } from '@/lib/uploads/shared/types'
|
||||
|
||||
const logger = createLogger('StorageClient')
|
||||
|
||||
export type { FileInfo, StorageConfig } from '@/lib/uploads/shared/types'
|
||||
|
||||
/**
|
||||
* Validate and resolve local file path ensuring it's within the allowed directory
|
||||
* @param key File key/name
|
||||
* @param uploadDir Upload directory path
|
||||
* @returns Resolved file path
|
||||
* @throws Error if path is invalid or outside allowed directory
|
||||
*/
|
||||
async function validateLocalFilePath(key: string, uploadDir: string): Promise<string> {
|
||||
const { join, resolve, sep } = await import('path')
|
||||
|
||||
const safeKey = sanitizeFileKey(key)
|
||||
const filePath = join(uploadDir, safeKey)
|
||||
|
||||
const resolvedPath = resolve(filePath)
|
||||
const allowedDir = resolve(uploadDir)
|
||||
|
||||
if (!resolvedPath.startsWith(allowedDir + sep) && resolvedPath !== allowedDir) {
|
||||
throw new Error('Invalid file path')
|
||||
}
|
||||
|
||||
return filePath
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload a file to the configured storage provider
|
||||
* @param file Buffer containing file data
|
||||
* @param fileName Original file name
|
||||
* @param contentType MIME type of the file
|
||||
* @param size File size in bytes (optional, will use buffer length if not provided)
|
||||
* @returns Object with file information
|
||||
*/
|
||||
export async function uploadFile(
|
||||
file: Buffer,
|
||||
fileName: string,
|
||||
contentType: string,
|
||||
size?: number
|
||||
): Promise<FileInfo>
|
||||
|
||||
/**
|
||||
* Upload a file to the configured storage provider with custom configuration
|
||||
* @param file Buffer containing file data
|
||||
* @param fileName Original file name
|
||||
* @param contentType MIME type of the file
|
||||
* @param customConfig Custom storage configuration
|
||||
* @param size File size in bytes (optional, will use buffer length if not provided)
|
||||
* @returns Object with file information
|
||||
*/
|
||||
export async function uploadFile(
|
||||
file: Buffer,
|
||||
fileName: string,
|
||||
contentType: string,
|
||||
customConfig: StorageConfig,
|
||||
size?: number
|
||||
): Promise<FileInfo>
|
||||
|
||||
export async function uploadFile(
|
||||
file: Buffer,
|
||||
fileName: string,
|
||||
contentType: string,
|
||||
configOrSize?: StorageConfig | number,
|
||||
size?: number
|
||||
): Promise<FileInfo> {
|
||||
if (USE_BLOB_STORAGE) {
|
||||
const { uploadToBlob } = await import('@/lib/uploads/providers/blob/client')
|
||||
if (typeof configOrSize === 'object') {
|
||||
if (!configOrSize.containerName || !configOrSize.accountName) {
|
||||
throw new Error(
|
||||
'Blob configuration missing required properties: containerName and accountName'
|
||||
)
|
||||
}
|
||||
if (!configOrSize.connectionString && !configOrSize.accountKey) {
|
||||
throw new Error(
|
||||
'Blob configuration missing authentication: either connectionString or accountKey must be provided'
|
||||
)
|
||||
}
|
||||
const blobConfig: BlobConfig = {
|
||||
containerName: configOrSize.containerName,
|
||||
accountName: configOrSize.accountName,
|
||||
accountKey: configOrSize.accountKey,
|
||||
connectionString: configOrSize.connectionString,
|
||||
}
|
||||
return uploadToBlob(file, fileName, contentType, blobConfig, size)
|
||||
}
|
||||
return uploadToBlob(file, fileName, contentType, configOrSize)
|
||||
}
|
||||
|
||||
if (USE_S3_STORAGE) {
|
||||
const { uploadToS3 } = await import('@/lib/uploads/providers/s3/client')
|
||||
if (typeof configOrSize === 'object') {
|
||||
if (!configOrSize.bucket || !configOrSize.region) {
|
||||
throw new Error('S3 configuration missing required properties: bucket and region')
|
||||
}
|
||||
const s3Config: S3Config = {
|
||||
bucket: configOrSize.bucket,
|
||||
region: configOrSize.region,
|
||||
}
|
||||
return uploadToS3(file, fileName, contentType, s3Config, size)
|
||||
}
|
||||
return uploadToS3(file, fileName, contentType, configOrSize)
|
||||
}
|
||||
|
||||
const { writeFile } = await import('fs/promises')
|
||||
const { join } = await import('path')
|
||||
const { v4: uuidv4 } = await import('uuid')
|
||||
const { UPLOAD_DIR_SERVER } = await import('@/lib/uploads/core/setup.server')
|
||||
|
||||
const safeFileName = sanitizeFileKey(fileName)
|
||||
const uniqueKey = `${uuidv4()}-${safeFileName}`
|
||||
const filePath = join(UPLOAD_DIR_SERVER, uniqueKey)
|
||||
|
||||
try {
|
||||
await writeFile(filePath, file)
|
||||
} catch (error) {
|
||||
logger.error(`Failed to write file to local storage: ${fileName}`, error)
|
||||
throw error
|
||||
}
|
||||
|
||||
const fileSize = typeof configOrSize === 'number' ? configOrSize : size || file.length
|
||||
|
||||
return {
|
||||
path: `/api/files/serve/${uniqueKey}`,
|
||||
key: uniqueKey,
|
||||
name: fileName,
|
||||
size: fileSize,
|
||||
type: contentType,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Download a file from the configured storage provider
|
||||
* @param key File key/name
|
||||
* @returns File buffer
|
||||
*/
|
||||
export async function downloadFile(key: string): Promise<Buffer>
|
||||
|
||||
/**
|
||||
* Download a file from the configured storage provider with custom configuration
|
||||
* @param key File key/name
|
||||
* @param customConfig Custom storage configuration
|
||||
* @returns File buffer
|
||||
*/
|
||||
export async function downloadFile(key: string, customConfig: StorageConfig): Promise<Buffer>
|
||||
|
||||
export async function downloadFile(key: string, customConfig?: StorageConfig): Promise<Buffer> {
|
||||
if (USE_BLOB_STORAGE) {
|
||||
const { downloadFromBlob } = await import('@/lib/uploads/providers/blob/client')
|
||||
if (customConfig) {
|
||||
if (!customConfig.containerName || !customConfig.accountName) {
|
||||
throw new Error(
|
||||
'Blob configuration missing required properties: containerName and accountName'
|
||||
)
|
||||
}
|
||||
if (!customConfig.connectionString && !customConfig.accountKey) {
|
||||
throw new Error(
|
||||
'Blob configuration missing authentication: either connectionString or accountKey must be provided'
|
||||
)
|
||||
}
|
||||
const blobConfig: BlobConfig = {
|
||||
containerName: customConfig.containerName,
|
||||
accountName: customConfig.accountName,
|
||||
accountKey: customConfig.accountKey,
|
||||
connectionString: customConfig.connectionString,
|
||||
}
|
||||
return downloadFromBlob(key, blobConfig)
|
||||
}
|
||||
return downloadFromBlob(key)
|
||||
}
|
||||
|
||||
if (USE_S3_STORAGE) {
|
||||
const { downloadFromS3 } = await import('@/lib/uploads/providers/s3/client')
|
||||
if (customConfig) {
|
||||
if (!customConfig.bucket || !customConfig.region) {
|
||||
throw new Error('S3 configuration missing required properties: bucket and region')
|
||||
}
|
||||
const s3Config: S3Config = {
|
||||
bucket: customConfig.bucket,
|
||||
region: customConfig.region,
|
||||
}
|
||||
return downloadFromS3(key, s3Config)
|
||||
}
|
||||
return downloadFromS3(key)
|
||||
}
|
||||
|
||||
const { readFile } = await import('fs/promises')
|
||||
const { UPLOAD_DIR_SERVER } = await import('@/lib/uploads/core/setup.server')
|
||||
|
||||
const filePath = await validateLocalFilePath(key, UPLOAD_DIR_SERVER)
|
||||
|
||||
try {
|
||||
return await readFile(filePath)
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
|
||||
throw new Error(`File not found: ${key}`)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a file from the configured storage provider
|
||||
* @param key File key/name
|
||||
*/
|
||||
export async function deleteFile(key: string): Promise<void> {
|
||||
if (USE_BLOB_STORAGE) {
|
||||
const { deleteFromBlob } = await import('@/lib/uploads/providers/blob/client')
|
||||
return deleteFromBlob(key)
|
||||
}
|
||||
|
||||
if (USE_S3_STORAGE) {
|
||||
const { deleteFromS3 } = await import('@/lib/uploads/providers/s3/client')
|
||||
return deleteFromS3(key)
|
||||
}
|
||||
|
||||
const { unlink } = await import('fs/promises')
|
||||
const { UPLOAD_DIR_SERVER } = await import('@/lib/uploads/core/setup.server')
|
||||
|
||||
const filePath = await validateLocalFilePath(key, UPLOAD_DIR_SERVER)
|
||||
|
||||
try {
|
||||
await unlink(filePath)
|
||||
} catch (error) {
|
||||
// File deletion is idempotent - if file doesn't exist, that's fine
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
const { deleteFileMetadata } = await import('../server/metadata')
|
||||
await deleteFileMetadata(key)
|
||||
}
|
||||
export type { StorageConfig } from '@/lib/uploads/shared/types'
|
||||
|
||||
/**
|
||||
* Get the current storage provider name
|
||||
@@ -250,11 +13,9 @@ export function getStorageProvider(): 'blob' | 's3' | 'local' {
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the appropriate serve path prefix based on storage provider
|
||||
* Get the serve path prefix (unified across all storage providers)
|
||||
*/
|
||||
export function getServePathPrefix(): string {
|
||||
if (USE_BLOB_STORAGE) return '/api/files/serve/blob/'
|
||||
if (USE_S3_STORAGE) return '/api/files/serve/s3/'
|
||||
return '/api/files/serve/'
|
||||
}
|
||||
|
||||
|
||||
@@ -101,6 +101,7 @@ export async function uploadFile(options: UploadFileOptions): Promise<FileInfo>
|
||||
contentType,
|
||||
createBlobConfig(config),
|
||||
file.length,
|
||||
preserveKey,
|
||||
metadata
|
||||
)
|
||||
|
||||
@@ -144,24 +145,32 @@ export async function uploadFile(options: UploadFileOptions): Promise<FileInfo>
|
||||
return uploadResult
|
||||
}
|
||||
|
||||
const { writeFile } = await import('fs/promises')
|
||||
const { join } = await import('path')
|
||||
const { v4: uuidv4 } = await import('uuid')
|
||||
const { writeFile, mkdir } = await import('fs/promises')
|
||||
const { join, dirname } = await import('path')
|
||||
const { UPLOAD_DIR_SERVER } = await import('./setup.server')
|
||||
|
||||
const safeKey = sanitizeFileKey(keyToUse)
|
||||
const uniqueKey = `${uuidv4()}-${safeKey}`
|
||||
const filePath = join(UPLOAD_DIR_SERVER, uniqueKey)
|
||||
const storageKey = keyToUse
|
||||
const safeKey = sanitizeFileKey(keyToUse) // Validates and preserves path structure
|
||||
const filesystemPath = join(UPLOAD_DIR_SERVER, safeKey)
|
||||
|
||||
await writeFile(filePath, file)
|
||||
await mkdir(dirname(filesystemPath), { recursive: true })
|
||||
|
||||
await writeFile(filesystemPath, file)
|
||||
|
||||
if (metadata) {
|
||||
await insertFileMetadataHelper(uniqueKey, metadata, context, fileName, contentType, file.length)
|
||||
await insertFileMetadataHelper(
|
||||
storageKey,
|
||||
metadata,
|
||||
context,
|
||||
fileName,
|
||||
contentType,
|
||||
file.length
|
||||
)
|
||||
}
|
||||
|
||||
return {
|
||||
path: `/api/files/serve/${uniqueKey}`,
|
||||
key: uniqueKey,
|
||||
path: `/api/files/serve/${storageKey}`,
|
||||
key: storageKey,
|
||||
name: fileName,
|
||||
size: file.length,
|
||||
type: contentType,
|
||||
@@ -188,8 +197,14 @@ export async function downloadFile(options: DownloadFileOptions): Promise<Buffer
|
||||
}
|
||||
}
|
||||
|
||||
const { downloadFile: defaultDownload } = await import('./storage-client')
|
||||
return defaultDownload(key)
|
||||
const { readFile } = await import('fs/promises')
|
||||
const { join } = await import('path')
|
||||
const { UPLOAD_DIR_SERVER } = await import('./setup.server')
|
||||
|
||||
const safeKey = sanitizeFileKey(key)
|
||||
const filePath = join(UPLOAD_DIR_SERVER, safeKey)
|
||||
|
||||
return readFile(filePath)
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -212,8 +227,14 @@ export async function deleteFile(options: DeleteFileOptions): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
const { deleteFile: defaultDelete } = await import('@/lib/uploads/core/storage-client')
|
||||
return defaultDelete(key)
|
||||
const { unlink } = await import('fs/promises')
|
||||
const { join } = await import('path')
|
||||
const { UPLOAD_DIR_SERVER } = await import('./setup.server')
|
||||
|
||||
const safeKey = sanitizeFileKey(key)
|
||||
const filePath = join(UPLOAD_DIR_SERVER, safeKey)
|
||||
|
||||
await unlink(filePath)
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -423,7 +444,9 @@ export async function generatePresignedDownloadUrl(
|
||||
return getPresignedUrlWithConfig(key, createBlobConfig(config), expirationSeconds)
|
||||
}
|
||||
|
||||
return `/api/files/serve/${encodeURIComponent(key)}`
|
||||
const { getBaseUrl } = await import('@/lib/urls/utils')
|
||||
const baseUrl = getBaseUrl()
|
||||
return `${baseUrl}/api/files/serve/${encodeURIComponent(key)}`
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -432,12 +455,3 @@ export async function generatePresignedDownloadUrl(
|
||||
export function hasCloudStorage(): boolean {
|
||||
return USE_BLOB_STORAGE || USE_S3_STORAGE
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current storage provider name
|
||||
*/
|
||||
export function getStorageProviderName(): 'Azure Blob' | 'S3' | 'Local' {
|
||||
if (USE_BLOB_STORAGE) return 'Azure Blob'
|
||||
if (USE_S3_STORAGE) return 'S3'
|
||||
return 'Local'
|
||||
}
|
||||
|
||||
@@ -12,7 +12,6 @@ export * as CopilotFiles from '@/lib/uploads/contexts/copilot'
|
||||
export * as ExecutionFiles from '@/lib/uploads/contexts/execution'
|
||||
export * as WorkspaceFiles from '@/lib/uploads/contexts/workspace'
|
||||
export {
|
||||
type FileInfo,
|
||||
getFileMetadata,
|
||||
getServePathPrefix,
|
||||
getStorageProvider,
|
||||
|
||||
@@ -111,7 +111,7 @@ describe('Azure Blob Storage Client', () => {
|
||||
})
|
||||
|
||||
expect(result).toEqual({
|
||||
path: expect.stringContaining('/api/files/serve/blob/'),
|
||||
path: expect.stringContaining('/api/files/serve/'),
|
||||
key: expect.stringContaining(fileName.replace(/\s+/g, '-')),
|
||||
name: fileName,
|
||||
size: testBuffer.length,
|
||||
|
||||
@@ -47,6 +47,7 @@ export async function getBlobServiceClient(): Promise<BlobServiceClientInstance>
|
||||
* @param contentType MIME type of the file
|
||||
* @param configOrSize Custom Blob configuration OR file size in bytes (optional)
|
||||
* @param size File size in bytes (required if configOrSize is BlobConfig, optional otherwise)
|
||||
* @param preserveKey Preserve the fileName as the storage key without adding timestamp prefix (default: false)
|
||||
* @param metadata Optional metadata to store with the file
|
||||
* @returns Object with file information
|
||||
*/
|
||||
@@ -56,23 +57,17 @@ export async function uploadToBlob(
|
||||
contentType: string,
|
||||
configOrSize?: BlobConfig | number,
|
||||
size?: number,
|
||||
metadata?: Record<string, string>
|
||||
): Promise<FileInfo>
|
||||
|
||||
export async function uploadToBlob(
|
||||
file: Buffer,
|
||||
fileName: string,
|
||||
contentType: string,
|
||||
configOrSize?: BlobConfig | number,
|
||||
size?: number,
|
||||
preserveKey?: boolean,
|
||||
metadata?: Record<string, string>
|
||||
): Promise<FileInfo> {
|
||||
let config: BlobConfig
|
||||
let fileSize: number
|
||||
let shouldPreserveKey: boolean
|
||||
|
||||
if (typeof configOrSize === 'object') {
|
||||
config = configOrSize
|
||||
fileSize = size ?? file.length
|
||||
shouldPreserveKey = preserveKey ?? false
|
||||
} else {
|
||||
config = {
|
||||
containerName: BLOB_CONFIG.containerName,
|
||||
@@ -81,10 +76,11 @@ export async function uploadToBlob(
|
||||
connectionString: BLOB_CONFIG.connectionString,
|
||||
}
|
||||
fileSize = configOrSize ?? file.length
|
||||
shouldPreserveKey = preserveKey ?? false
|
||||
}
|
||||
|
||||
const safeFileName = fileName.replace(/\s+/g, '-') // Replace spaces with hyphens
|
||||
const uniqueKey = `${Date.now()}-${safeFileName}`
|
||||
const uniqueKey = shouldPreserveKey ? fileName : `${Date.now()}-${safeFileName}`
|
||||
|
||||
const blobServiceClient = await getBlobServiceClient()
|
||||
const containerClient = blobServiceClient.getContainerClient(config.containerName)
|
||||
@@ -106,7 +102,7 @@ export async function uploadToBlob(
|
||||
metadata: blobMetadata,
|
||||
})
|
||||
|
||||
const servePath = `/api/files/serve/blob/${encodeURIComponent(uniqueKey)}`
|
||||
const servePath = `/api/files/serve/${encodeURIComponent(uniqueKey)}`
|
||||
|
||||
return {
|
||||
path: servePath,
|
||||
@@ -518,7 +514,7 @@ export async function completeMultipartUpload(
|
||||
})
|
||||
|
||||
const location = blockBlobClient.url
|
||||
const path = `/api/files/serve/blob/${encodeURIComponent(key)}`
|
||||
const path = `/api/files/serve/${encodeURIComponent(key)}`
|
||||
|
||||
return {
|
||||
location,
|
||||
|
||||
@@ -90,7 +90,7 @@ describe('S3 Client', () => {
|
||||
expect(mockSend).toHaveBeenCalledWith(expect.any(Object))
|
||||
|
||||
expect(result).toEqual({
|
||||
path: expect.stringContaining('/api/files/serve/s3/'),
|
||||
path: expect.stringContaining('/api/files/serve/'),
|
||||
key: expect.stringContaining('test-file.txt'),
|
||||
name: 'test-file.txt',
|
||||
size: file.length,
|
||||
|
||||
@@ -61,16 +61,6 @@ export function getS3Client(): S3Client {
|
||||
* @param metadata Optional metadata to store with the file
|
||||
* @returns Object with file information
|
||||
*/
|
||||
export async function uploadToS3(
|
||||
file: Buffer,
|
||||
fileName: string,
|
||||
contentType: string,
|
||||
configOrSize?: S3Config | number,
|
||||
size?: number,
|
||||
skipTimestampPrefix?: boolean,
|
||||
metadata?: Record<string, string>
|
||||
): Promise<FileInfo>
|
||||
|
||||
export async function uploadToS3(
|
||||
file: Buffer,
|
||||
fileName: string,
|
||||
@@ -95,7 +85,7 @@ export async function uploadToS3(
|
||||
}
|
||||
|
||||
const safeFileName = fileName.replace(/\s+/g, '-') // Replace spaces with hyphens
|
||||
const uniqueKey = shouldSkipTimestamp ? safeFileName : `${Date.now()}-${safeFileName}`
|
||||
const uniqueKey = shouldSkipTimestamp ? fileName : `${Date.now()}-${safeFileName}`
|
||||
|
||||
const s3Client = getS3Client()
|
||||
|
||||
@@ -118,7 +108,7 @@ export async function uploadToS3(
|
||||
})
|
||||
)
|
||||
|
||||
const servePath = `/api/files/serve/s3/${encodeURIComponent(uniqueKey)}`
|
||||
const servePath = `/api/files/serve/${encodeURIComponent(uniqueKey)}`
|
||||
|
||||
return {
|
||||
path: servePath,
|
||||
@@ -313,7 +303,7 @@ export async function completeS3MultipartUpload(
|
||||
const response = await s3Client.send(command)
|
||||
const location =
|
||||
response.Location || `https://${config.bucket}.s3.${config.region}.amazonaws.com/${key}`
|
||||
const path = `/api/files/serve/s3/${encodeURIComponent(key)}`
|
||||
const path = `/api/files/serve/${encodeURIComponent(key)}`
|
||||
|
||||
return {
|
||||
location,
|
||||
|
||||
@@ -214,12 +214,6 @@ export function extractStorageKey(filePath: string): string {
|
||||
// If URL parsing fails, use the original path
|
||||
}
|
||||
|
||||
if (pathWithoutQuery.includes('/api/files/serve/s3/')) {
|
||||
return decodeURIComponent(pathWithoutQuery.split('/api/files/serve/s3/')[1])
|
||||
}
|
||||
if (pathWithoutQuery.includes('/api/files/serve/blob/')) {
|
||||
return decodeURIComponent(pathWithoutQuery.split('/api/files/serve/blob/')[1])
|
||||
}
|
||||
if (pathWithoutQuery.startsWith('/api/files/serve/')) {
|
||||
return decodeURIComponent(pathWithoutQuery.substring('/api/files/serve/'.length))
|
||||
}
|
||||
@@ -421,11 +415,30 @@ export function sanitizeStorageMetadata(
|
||||
/**
|
||||
* Sanitize a file key/path for local storage
|
||||
* Removes dangerous characters and prevents path traversal
|
||||
* Preserves forward slashes for structured paths (e.g., kb/file.json, workspace/id/file.json)
|
||||
* All keys must have a context prefix structure
|
||||
* @param key Original file key/path
|
||||
* @returns Sanitized key safe for filesystem use
|
||||
*/
|
||||
export function sanitizeFileKey(key: string): string {
|
||||
return key.replace(/[^a-zA-Z0-9.-]/g, '_').replace(/\.\./g, '')
|
||||
if (!key.includes('/')) {
|
||||
throw new Error('File key must include a context prefix (e.g., kb/, workspace/, execution/)')
|
||||
}
|
||||
|
||||
const segments = key.split('/')
|
||||
|
||||
const sanitizedSegments = segments.map((segment, index) => {
|
||||
if (segment === '..' || segment === '.') {
|
||||
throw new Error('Path traversal detected in file key')
|
||||
}
|
||||
|
||||
if (index === segments.length - 1) {
|
||||
return segment.replace(/[^a-zA-Z0-9.-]/g, '_')
|
||||
}
|
||||
return segment.replace(/[^a-zA-Z0-9-]/g, '_')
|
||||
})
|
||||
|
||||
return sanitizedSegments.join('/')
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -80,11 +80,13 @@ export function validateFileType(fileName: string, mimeType: string): FileValida
|
||||
}
|
||||
}
|
||||
|
||||
const baseMimeType = mimeType.split(';')[0].trim()
|
||||
|
||||
const allowedMimeTypes = SUPPORTED_MIME_TYPES[extension]
|
||||
if (!allowedMimeTypes.includes(mimeType)) {
|
||||
if (!allowedMimeTypes.includes(baseMimeType)) {
|
||||
return {
|
||||
code: 'MIME_TYPE_MISMATCH',
|
||||
message: `MIME type ${mimeType} does not match file extension ${extension}. Expected: ${allowedMimeTypes.join(', ')}`,
|
||||
message: `MIME type ${baseMimeType} does not match file extension ${extension}. Expected: ${allowedMimeTypes.join(', ')}`,
|
||||
supportedTypes: allowedMimeTypes,
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user