fix(files): fix json uploads, disable storage metering when billing is disabled, exclude kb uploads from storage metering, simplify serve path route (#1850)

* fix(files): fix local kb files storage to have parity with cloud storage providers

* fix(files): fix json uploads, disable storage metering when billing is disabled, exclude kb uploads from storage metering, simplify serve path route

* cleanup
This commit is contained in:
Waleed
2025-11-07 18:41:44 -08:00
committed by GitHub
parent d17c627064
commit e91a8af7cd
28 changed files with 237 additions and 533 deletions

View File

@@ -11,7 +11,6 @@ export const dynamic = 'force-dynamic'
const logger = createLogger('CareersAPI')
// Max file size: 10MB
const MAX_FILE_SIZE = 10 * 1024 * 1024
const ALLOWED_FILE_TYPES = [
'application/pdf',
@@ -37,7 +36,6 @@ export async function POST(request: NextRequest) {
try {
const formData = await request.formData()
// Extract form fields
const data = {
name: formData.get('name') as string,
email: formData.get('email') as string,
@@ -50,7 +48,6 @@ export async function POST(request: NextRequest) {
message: formData.get('message') as string,
}
// Extract and validate resume file
const resumeFile = formData.get('resume') as File | null
if (!resumeFile) {
return NextResponse.json(
@@ -63,7 +60,6 @@ export async function POST(request: NextRequest) {
)
}
// Validate file size
if (resumeFile.size > MAX_FILE_SIZE) {
return NextResponse.json(
{
@@ -75,7 +71,6 @@ export async function POST(request: NextRequest) {
)
}
// Validate file type
if (!ALLOWED_FILE_TYPES.includes(resumeFile.type)) {
return NextResponse.json(
{
@@ -87,7 +82,6 @@ export async function POST(request: NextRequest) {
)
}
// Convert file to base64 for email attachment
const resumeBuffer = await resumeFile.arrayBuffer()
const resumeBase64 = Buffer.from(resumeBuffer).toString('base64')
@@ -126,7 +120,6 @@ export async function POST(request: NextRequest) {
})
)
// Send email with resume attachment
const careersEmailResult = await sendEmail({
to: 'careers@sim.ai',
subject: `New Career Application: ${validatedData.name} - ${validatedData.position}`,

View File

@@ -98,7 +98,6 @@ function extractWorkspaceIdFromKey(key: string): string | null {
* Verify file access based on file path patterns and metadata
* @param cloudKey The file key/path (e.g., "workspace_id/workflow_id/execution_id/filename" or "kb/filename")
* @param userId The authenticated user ID
* @param bucketType Optional bucket type (e.g., 'copilot', 'execution-files')
* @param customConfig Optional custom storage configuration
* @param context Optional explicit storage context
* @param isLocal Optional flag indicating if this is local storage
@@ -107,7 +106,6 @@ function extractWorkspaceIdFromKey(key: string): string | null {
export async function verifyFileAccess(
cloudKey: string,
userId: string,
bucketType?: string | null,
customConfig?: StorageConfig,
context?: StorageContext,
isLocal?: boolean
@@ -128,12 +126,12 @@ export async function verifyFileAccess(
}
// 2. Execution files: workspace_id/workflow_id/execution_id/filename
if (inferredContext === 'execution' || (!context && isExecutionFile(cloudKey, bucketType))) {
if (inferredContext === 'execution') {
return await verifyExecutionFileAccess(cloudKey, userId, customConfig)
}
// 3. Copilot files: Check database first, then metadata, then path pattern (legacy)
if (inferredContext === 'copilot' || bucketType === 'copilot') {
if (inferredContext === 'copilot') {
return await verifyCopilotFileAccess(cloudKey, userId, customConfig)
}
@@ -223,18 +221,6 @@ async function verifyWorkspaceFileAccess(
}
}
/**
* Check if file is an execution file based on path pattern
* Execution files have format: workspace_id/workflow_id/execution_id/filename
*/
function isExecutionFile(cloudKey: string, bucketType?: string | null): boolean {
if (bucketType === 'execution-files' || bucketType === 'execution') {
return true
}
return inferContextFromKey(cloudKey) === 'execution'
}
/**
* Verify access to execution files
* Modern format: execution/workspace_id/workflow_id/execution_id/filename
@@ -590,7 +576,7 @@ export async function authorizeFileAccess(
storageConfig?: StorageConfig,
isLocal?: boolean
): Promise<AuthorizationResult> {
const granted = await verifyFileAccess(key, userId, null, storageConfig, context, isLocal)
const granted = await verifyFileAccess(key, userId, storageConfig, context, isLocal)
if (granted) {
let workspaceId: string | undefined

View File

@@ -59,7 +59,7 @@ describe('File Delete API Route', () => {
})
const req = createMockRequest('POST', {
filePath: '/api/files/serve/s3/workspace/test-workspace-id/1234567890-test-file.txt',
filePath: '/api/files/serve/workspace/test-workspace-id/1234567890-test-file.txt',
})
const { POST } = await import('@/app/api/files/delete/route')
@@ -85,7 +85,7 @@ describe('File Delete API Route', () => {
})
const req = createMockRequest('POST', {
filePath: '/api/files/serve/blob/workspace/test-workspace-id/1234567890-test-document.pdf',
filePath: '/api/files/serve/workspace/test-workspace-id/1234567890-test-document.pdf',
})
const { POST } = await import('@/app/api/files/delete/route')

View File

@@ -13,9 +13,6 @@ import {
extractFilename,
FileNotFoundError,
InvalidRequestError,
isBlobPath,
isCloudPath,
isS3Path,
} from '@/app/api/files/utils'
export const dynamic = 'force-dynamic'
@@ -54,9 +51,8 @@ export async function POST(request: NextRequest) {
const hasAccess = await verifyFileAccess(
key,
userId,
null,
undefined,
storageContext,
undefined, // customConfig
storageContext, // context
!hasCloudStorage() // isLocal
)
@@ -99,15 +95,11 @@ export async function POST(request: NextRequest) {
* Extract storage key from file path
*/
function extractStorageKeyFromPath(filePath: string): string {
if (isS3Path(filePath) || isBlobPath(filePath) || filePath.startsWith('/api/files/serve/')) {
if (filePath.startsWith('/api/files/serve/')) {
return extractStorageKey(filePath)
}
if (!isCloudPath(filePath)) {
return extractFilename(filePath)
}
return filePath
return extractFilename(filePath)
}
/**

View File

@@ -51,10 +51,9 @@ export async function POST(request: NextRequest) {
const hasAccess = await verifyFileAccess(
key,
userId,
isExecutionFile ? 'execution' : null,
undefined,
storageContext,
!hasCloudStorage()
undefined, // customConfig
storageContext, // context
!hasCloudStorage() // isLocal
)
if (!hasAccess) {

View File

@@ -427,9 +427,8 @@ async function handleCloudFile(
const hasAccess = await verifyFileAccess(
cloudKey,
userId,
null,
undefined,
context,
undefined, // customConfig
context, // context
false // isLocal
)
@@ -534,9 +533,8 @@ async function handleLocalFile(
const hasAccess = await verifyFileAccess(
filename,
userId,
null,
undefined,
context,
undefined, // customConfig
context, // context
true // isLocal
)
@@ -812,11 +810,7 @@ function prettySize(bytes: number): string {
* Create a formatted message for PDF content
*/
function createPdfFallbackMessage(pageCount: number, size: number, path?: string): string {
const formattedPath = path
? path.includes('/api/files/serve/s3/')
? `S3 path: ${decodeURIComponent(path.split('/api/files/serve/s3/')[1])}`
: `Local path: ${path}`
: 'Unknown path'
const formattedPath = path || 'Unknown path'
return `PDF document - ${pageCount} page(s), ${prettySize(size)}
Path: ${formattedPath}
@@ -834,12 +828,8 @@ function createPdfFailureMessage(
path: string,
error: string
): string {
const formattedPath = path.includes('/api/files/serve/s3/')
? `S3 path: ${decodeURIComponent(path.split('/api/files/serve/s3/')[1])}`
: `Local path: ${path}`
return `PDF document - Processing failed, ${prettySize(size)}
Path: ${formattedPath}
Path: ${path}
Error: ${error}
This file appears to be a PDF document that could not be processed.

View File

@@ -54,8 +54,6 @@ describe('File Serve API Route', () => {
})
}),
getContentType: vi.fn().mockReturnValue('text/plain'),
isS3Path: vi.fn().mockReturnValue(false),
isBlobPath: vi.fn().mockReturnValue(false),
extractStorageKey: vi.fn().mockImplementation((path) => path.split('/').pop()),
extractFilename: vi.fn().mockImplementation((path) => path.split('/').pop()),
findLocalFile: vi.fn().mockReturnValue('/test/uploads/test-file.txt'),
@@ -112,8 +110,6 @@ describe('File Serve API Route', () => {
})
}),
getContentType: vi.fn().mockReturnValue('text/plain'),
isS3Path: vi.fn().mockReturnValue(false),
isBlobPath: vi.fn().mockReturnValue(false),
extractStorageKey: vi.fn().mockImplementation((path) => path.split('/').pop()),
extractFilename: vi.fn().mockImplementation((path) => path.split('/').pop()),
findLocalFile: vi.fn().mockReturnValue('/test/uploads/nested/path/file.txt'),
@@ -203,17 +199,15 @@ describe('File Serve API Route', () => {
})
}),
getContentType: vi.fn().mockReturnValue('image/png'),
isS3Path: vi.fn().mockReturnValue(false),
isBlobPath: vi.fn().mockReturnValue(false),
extractStorageKey: vi.fn().mockImplementation((path) => path.split('/').pop()),
extractFilename: vi.fn().mockImplementation((path) => path.split('/').pop()),
findLocalFile: vi.fn().mockReturnValue('/test/uploads/test-file.txt'),
}))
const req = new NextRequest(
'http://localhost:3000/api/files/serve/s3/workspace/test-workspace-id/1234567890-image.png'
'http://localhost:3000/api/files/serve/workspace/test-workspace-id/1234567890-image.png'
)
const params = { path: ['s3', 'workspace', 'test-workspace-id', '1234567890-image.png'] }
const params = { path: ['workspace', 'test-workspace-id', '1234567890-image.png'] }
const { GET } = await import('@/app/api/files/serve/[...path]/route')
const response = await GET(req, { params: Promise.resolve(params) })
@@ -262,8 +256,6 @@ describe('File Serve API Route', () => {
})
}),
getContentType: vi.fn().mockReturnValue('text/plain'),
isS3Path: vi.fn().mockReturnValue(false),
isBlobPath: vi.fn().mockReturnValue(false),
extractStorageKey: vi.fn(),
extractFilename: vi.fn(),
findLocalFile: vi.fn().mockReturnValue(null),

View File

@@ -62,11 +62,11 @@ export async function GET(
const userId = authResult.userId
if (isUsingCloudStorage() || isCloudPath) {
return await handleCloudProxy(cloudKey, userId, contextParam, legacyBucketType)
if (isUsingCloudStorage()) {
return await handleCloudProxy(cloudKey, userId, contextParam)
}
return await handleLocalFile(fullPath, userId)
return await handleLocalFile(cloudKey, userId)
} catch (error) {
logger.error('Error serving file:', error)
@@ -87,10 +87,9 @@ async function handleLocalFile(filename: string, userId: string): Promise<NextRe
const hasAccess = await verifyFileAccess(
filename,
userId,
null,
undefined,
contextParam,
true // isLocal = true
undefined, // customConfig
contextParam, // context
true // isLocal
)
if (!hasAccess) {
@@ -123,8 +122,7 @@ async function handleLocalFile(filename: string, userId: string): Promise<NextRe
async function handleCloudProxy(
cloudKey: string,
userId: string,
contextParam?: string | null,
legacyBucketType?: string | null
contextParam?: string | null
): Promise<NextResponse> {
try {
let context: StorageContext
@@ -132,9 +130,6 @@ async function handleCloudProxy(
if (contextParam) {
context = contextParam as StorageContext
logger.info(`Using explicit context: ${context} for key: ${cloudKey}`)
} else if (legacyBucketType === 'copilot') {
context = 'copilot'
logger.info(`Using legacy bucket parameter for copilot context: ${cloudKey}`)
} else {
context = inferContextFromKey(cloudKey)
logger.info(`Inferred context: ${context} from key pattern: ${cloudKey}`)
@@ -143,10 +138,9 @@ async function handleCloudProxy(
const hasAccess = await verifyFileAccess(
cloudKey,
userId,
legacyBucketType || null,
undefined,
context,
false // isLocal = false
undefined, // customConfig
context, // context
false // isLocal
)
if (!hasAccess) {

View File

@@ -137,6 +137,10 @@ export async function POST(request: NextRequest) {
logger.info(`Uploading knowledge-base file: ${originalName}`)
const timestamp = Date.now()
const safeFileName = originalName.replace(/\s+/g, '-')
const storageKey = `kb/${timestamp}-${safeFileName}`
const metadata: Record<string, string> = {
originalName: originalName,
uploadedAt: new Date().toISOString(),
@@ -150,9 +154,11 @@ export async function POST(request: NextRequest) {
const fileInfo = await storageService.uploadFile({
file: buffer,
fileName: originalName,
fileName: storageKey,
contentType: file.type,
context: 'knowledge-base',
preserveKey: true,
customKey: storageKey,
metadata,
})

View File

@@ -3,6 +3,7 @@ import { join, resolve, sep } from 'path'
import { NextResponse } from 'next/server'
import { createLogger } from '@/lib/logs/console/logger'
import { UPLOAD_DIR } from '@/lib/uploads/config'
import { sanitizeFileKey } from '@/lib/uploads/utils/file-utils'
const logger = createLogger('FilesUtils')
@@ -37,7 +38,6 @@ export class InvalidRequestError extends Error {
}
export const contentTypeMap: Record<string, string> = {
// Text formats
txt: 'text/plain',
csv: 'text/csv',
json: 'application/json',
@@ -47,26 +47,20 @@ export const contentTypeMap: Record<string, string> = {
css: 'text/css',
js: 'application/javascript',
ts: 'application/typescript',
// Document formats
pdf: 'application/pdf',
googleDoc: 'application/vnd.google-apps.document',
doc: 'application/msword',
docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
// Spreadsheet formats
xls: 'application/vnd.ms-excel',
xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
googleSheet: 'application/vnd.google-apps.spreadsheet',
// Presentation formats
ppt: 'application/vnd.ms-powerpoint',
pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
// Image formats
png: 'image/png',
jpg: 'image/jpeg',
jpeg: 'image/jpeg',
gif: 'image/gif',
// Archive formats
zip: 'application/zip',
// Folder format
googleFolder: 'application/vnd.google-apps.folder',
}
@@ -90,18 +84,6 @@ export function getContentType(filename: string): string {
return contentTypeMap[extension] || 'application/octet-stream'
}
export function isS3Path(path: string): boolean {
return path.includes('/api/files/serve/s3/')
}
export function isBlobPath(path: string): boolean {
return path.includes('/api/files/serve/blob/')
}
export function isCloudPath(path: string): boolean {
return isS3Path(path) || isBlobPath(path)
}
export function extractFilename(path: string): string {
let filename: string
@@ -142,29 +124,48 @@ function sanitizeFilename(filename: string): string {
throw new Error('Invalid filename provided')
}
const sanitized = filename.replace(/\.\./g, '').replace(/[/\\]/g, '').replace(/^\./g, '').trim()
if (!sanitized || sanitized.length === 0) {
throw new Error('Invalid or empty filename after sanitization')
if (!filename.includes('/')) {
throw new Error('File key must include a context prefix (e.g., kb/, workspace/, execution/)')
}
if (
sanitized.includes(':') ||
sanitized.includes('|') ||
sanitized.includes('?') ||
sanitized.includes('*') ||
sanitized.includes('\x00') ||
/[\x00-\x1F\x7F]/.test(sanitized)
) {
throw new Error('Filename contains invalid characters')
}
const segments = filename.split('/')
return sanitized
const sanitizedSegments = segments.map((segment) => {
if (segment === '..' || segment === '.') {
throw new Error('Path traversal detected')
}
const sanitized = segment.replace(/\.\./g, '').replace(/[\\]/g, '').replace(/^\./g, '').trim()
if (!sanitized) {
throw new Error('Invalid or empty path segment after sanitization')
}
if (
sanitized.includes(':') ||
sanitized.includes('|') ||
sanitized.includes('?') ||
sanitized.includes('*') ||
sanitized.includes('\x00') ||
/[\x00-\x1F\x7F]/.test(sanitized)
) {
throw new Error('Path segment contains invalid characters')
}
return sanitized
})
return sanitizedSegments.join(sep)
}
export function findLocalFile(filename: string): string | null {
try {
const sanitizedFilename = sanitizeFilename(filename)
const sanitizedFilename = sanitizeFileKey(filename)
// Reject if sanitized filename is empty or only contains path separators/dots
if (!sanitizedFilename || !sanitizedFilename.trim() || /^[/\\.\s]+$/.test(sanitizedFilename)) {
return null
}
const possiblePaths = [
join(UPLOAD_DIR, sanitizedFilename),
@@ -175,8 +176,9 @@ export function findLocalFile(filename: string): string | null {
const resolvedPath = resolve(path)
const allowedDirs = [resolve(UPLOAD_DIR), resolve(process.cwd(), 'uploads')]
// Must be within allowed directory but NOT the directory itself
const isWithinAllowedDir = allowedDirs.some(
(allowedDir) => resolvedPath.startsWith(allowedDir + sep) || resolvedPath === allowedDir
(allowedDir) => resolvedPath.startsWith(allowedDir + sep) && resolvedPath !== allowedDir
)
if (!isWithinAllowedDir) {
@@ -233,7 +235,6 @@ function getSecureFileHeaders(filename: string, originalContentType: string) {
}
function encodeFilenameForHeader(storageKey: string): string {
// Extract just the filename from the storage key (last segment after /)
const filename = storageKey.split('/').pop() || storageKey
const hasNonAscii = /[^\x00-\x7F]/.test(filename)

View File

@@ -123,6 +123,8 @@ export async function GET(request: NextRequest) {
fileName: enhancedLogKey,
contentType: 'application/json',
context: 'logs',
preserveKey: true,
customKey: enhancedLogKey,
metadata: {
logId: String(log.id),
workflowId: String(log.workflowId),

View File

@@ -62,9 +62,8 @@ export async function POST(request: NextRequest) {
const hasAccess = await verifyFileAccess(
storageKey,
userId,
null,
undefined,
context,
undefined, // customConfig
context, // context
false // isLocal
)

View File

@@ -13,6 +13,7 @@ import {
TableHeader,
TableRow,
} from '@/components/ui/table'
import { getEnv, isTruthy } from '@/lib/env'
import { createLogger } from '@/lib/logs/console/logger'
import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace'
import { getFileExtension } from '@/lib/uploads/utils/file-utils'
@@ -22,6 +23,7 @@ import { useUserPermissions } from '@/hooks/use-user-permissions'
import { useWorkspacePermissions } from '@/hooks/use-workspace-permissions'
const logger = createLogger('FileUploadsSettings')
const isBillingEnabled = isTruthy(getEnv('NEXT_PUBLIC_BILLING_ENABLED'))
const SUPPORTED_EXTENSIONS = [
'pdf',
@@ -36,8 +38,12 @@ const SUPPORTED_EXTENSIONS = [
'htm',
'pptx',
'ppt',
'json',
'yaml',
'yml',
] as const
const ACCEPT_ATTR = '.pdf,.csv,.doc,.docx,.txt,.md,.xlsx,.xls,.html,.htm,.pptx,.ppt'
const ACCEPT_ATTR =
'.pdf,.csv,.doc,.docx,.txt,.md,.xlsx,.xls,.html,.htm,.pptx,.ppt,.json,.yaml,.yml'
interface StorageInfo {
usedBytes: number
@@ -45,11 +51,6 @@ interface StorageInfo {
percentUsed: number
}
interface UsageData {
plan: string
storage: StorageInfo
}
export function FileUploads() {
const params = useParams()
const workspaceId = params?.workspaceId as string
@@ -87,6 +88,11 @@ export function FileUploads() {
}
const loadStorageInfo = async () => {
if (!isBillingEnabled) {
setStorageLoading(false)
return
}
try {
setStorageLoading(true)
const response = await fetch('/api/users/me/usage-limits')
@@ -158,7 +164,9 @@ export function FileUploads() {
}
await loadFiles()
await loadStorageInfo()
if (isBillingEnabled) {
await loadStorageInfo()
}
if (unsupported.length) {
lastError = `Unsupported file type: ${unsupported.join(', ')}`
}
@@ -193,7 +201,7 @@ export function FileUploads() {
setFiles((prev) => prev.filter((f) => f.id !== file.id))
if (storageInfo) {
if (isBillingEnabled && storageInfo) {
const newUsedBytes = Math.max(0, storageInfo.usedBytes - file.size)
const newPercentUsed = (newUsedBytes / storageInfo.limitBytes) * 100
setStorageInfo({
@@ -217,7 +225,9 @@ export function FileUploads() {
} catch (error) {
logger.error('Error deleting file:', error)
await loadFiles()
await loadStorageInfo()
if (isBillingEnabled) {
await loadStorageInfo()
}
} finally {
setDeletingFileId(null)
}
@@ -283,31 +293,35 @@ export function FileUploads() {
/>
</div>
<div className='flex items-center gap-3'>
{storageLoading ? (
<Skeleton className='h-4 w-32' />
) : storageInfo ? (
<div className='flex flex-col items-end gap-1'>
<div className='flex items-center gap-2 text-sm'>
<span
className={cn(
'font-medium',
planName === 'free' ? 'text-foreground' : GRADIENT_TEXT_STYLES
)}
>
{displayPlanName}
</span>
<span className='text-muted-foreground tabular-nums'>
{formatStorageSize(storageInfo.usedBytes)} /{' '}
{formatStorageSize(storageInfo.limitBytes)}
</span>
</div>
<Progress
value={Math.min(storageInfo.percentUsed, 100)}
className='h-1 w-full'
indicatorClassName='bg-black dark:bg-white'
/>
</div>
) : null}
{isBillingEnabled && (
<>
{storageLoading ? (
<Skeleton className='h-4 w-32' />
) : storageInfo ? (
<div className='flex flex-col items-end gap-1'>
<div className='flex items-center gap-2 text-sm'>
<span
className={cn(
'font-medium',
planName === 'free' ? 'text-foreground' : GRADIENT_TEXT_STYLES
)}
>
{displayPlanName}
</span>
<span className='text-muted-foreground tabular-nums'>
{formatStorageSize(storageInfo.usedBytes)} /{' '}
{formatStorageSize(storageInfo.limitBytes)}
</span>
</div>
<Progress
value={Math.min(storageInfo.percentUsed, 100)}
className='h-1 w-full'
indicatorClassName='bg-black dark:bg-white'
/>
</div>
) : null}
</>
)}
{userPermissions.canEdit && (
<div className='flex items-center'>
<input

View File

@@ -13,6 +13,7 @@ import {
import { organization, subscription, userStats } from '@sim/db/schema'
import { eq } from 'drizzle-orm'
import { getEnv } from '@/lib/env'
import { isBillingEnabled } from '@/lib/environment'
import { createLogger } from '@/lib/logs/console/logger'
const logger = createLogger('StorageLimits')
@@ -156,11 +157,20 @@ export async function getUserStorageUsage(userId: string): Promise<number> {
/**
* Check if user has storage quota available
* Always allows uploads when billing is disabled
*/
export async function checkStorageQuota(
userId: string,
additionalBytes: number
): Promise<{ allowed: boolean; currentUsage: number; limit: number; error?: string }> {
if (!isBillingEnabled) {
return {
allowed: true,
currentUsage: 0,
limit: Number.MAX_SAFE_INTEGER,
}
}
try {
const [currentUsage, limit] = await Promise.all([
getUserStorageUsage(userId),

View File

@@ -1,19 +1,27 @@
/**
* Storage usage tracking
* Updates storage_used_bytes for users and organizations
* Only tracks when billing is enabled
*/
import { db } from '@sim/db'
import { organization, userStats } from '@sim/db/schema'
import { eq, sql } from 'drizzle-orm'
import { isBillingEnabled } from '@/lib/environment'
import { createLogger } from '@/lib/logs/console/logger'
const logger = createLogger('StorageTracking')
/**
* Increment storage usage after successful file upload
* Only tracks if billing is enabled
*/
export async function incrementStorageUsage(userId: string, bytes: number): Promise<void> {
if (!isBillingEnabled) {
logger.debug('Billing disabled, skipping storage increment')
return
}
try {
// Check if user is in a team/enterprise org
const { getHighestPrioritySubscription } = await import('@/lib/billing/core/subscription')
@@ -48,8 +56,14 @@ export async function incrementStorageUsage(userId: string, bytes: number): Prom
/**
* Decrement storage usage after file deletion
* Only tracks if billing is enabled
*/
export async function decrementStorageUsage(userId: string, bytes: number): Promise<void> {
if (!isBillingEnabled) {
logger.debug('Billing disabled, skipping storage decrement')
return
}
try {
// Check if user is in a team/enterprise org
const { getHighestPrioritySubscription } = await import('@/lib/billing/core/subscription')

View File

@@ -1,5 +1,4 @@
import { readFile } from 'fs/promises'
import { PDFParse } from 'pdf-parse'
import type { FileParseResult, FileParser } from '@/lib/file-parsers/types'
import { createLogger } from '@/lib/logs/console/logger'
@@ -29,6 +28,8 @@ export class PdfParser implements FileParser {
try {
logger.info('Starting to parse buffer, size:', dataBuffer.length)
const { PDFParse } = await import('pdf-parse')
const parser = new PDFParse({ data: dataBuffer })
const textResult = await parser.getText()
const infoResult = await parser.getInfo()
@@ -41,7 +42,6 @@ export class PdfParser implements FileParser {
textResult.text.length
)
// Remove null bytes from content (PostgreSQL JSONB doesn't allow them)
const cleanContent = textResult.text.replace(/\u0000/g, '')
return {

View File

@@ -189,11 +189,17 @@ async function handleFileForOCR(
...(workspaceId && { workspaceId }),
}
const timestamp = Date.now()
const uniqueId = Math.random().toString(36).substring(2, 9)
const safeFileName = filename.replace(/[^a-zA-Z0-9.-]/g, '_')
const customKey = `kb/${timestamp}-${uniqueId}-${safeFileName}`
const cloudResult = await StorageService.uploadFile({
file: buffer,
fileName: filename,
contentType: mimeType,
context: 'knowledge-base',
customKey,
metadata,
})

View File

@@ -3,11 +3,6 @@ import { db } from '@sim/db'
import { document, embedding, knowledgeBase, knowledgeBaseTagDefinitions } from '@sim/db/schema'
import { tasks } from '@trigger.dev/sdk'
import { and, asc, desc, eq, inArray, isNull, sql } from 'drizzle-orm'
import {
checkStorageQuota,
decrementStorageUsage,
incrementStorageUsage,
} from '@/lib/billing/storage'
import { generateEmbeddings } from '@/lib/embeddings/utils'
import { env } from '@/lib/env'
import { getSlotsForFieldType, type TAG_SLOT_CONFIG } from '@/lib/knowledge/consts'
@@ -696,13 +691,6 @@ export async function createDocumentRecords(
if (kb.length === 0) {
throw new Error('Knowledge base not found')
}
// Always meter the knowledge base owner
const quotaCheck = await checkStorageQuota(kb[0].userId, totalSize)
if (!quotaCheck.allowed) {
throw new Error(quotaCheck.error || 'Storage limit exceeded')
}
}
return await db.transaction(async (tx) => {
@@ -787,21 +775,6 @@ export async function createDocumentRecords(
.from(knowledgeBase)
.where(eq(knowledgeBase.id, knowledgeBaseId))
.limit(1)
if (kb.length > 0) {
// Always meter the knowledge base owner
try {
await incrementStorageUsage(kb[0].userId, totalSize)
logger.info(
`[${requestId}] Updated knowledge base owner storage usage for ${totalSize} bytes`
)
} catch (error) {
logger.error(
`[${requestId}] Failed to update knowledge base owner storage usage:`,
error
)
}
}
}
}
@@ -1037,13 +1010,6 @@ export async function createSingleDocument(
if (kb.length === 0) {
throw new Error('Knowledge base not found')
}
// Always meter the knowledge base owner
const quotaCheck = await checkStorageQuota(kb[0].userId, documentData.fileSize)
if (!quotaCheck.allowed) {
throw new Error(quotaCheck.error || 'Storage limit exceeded')
}
}
const documentId = randomUUID()
@@ -1103,18 +1069,6 @@ export async function createSingleDocument(
.from(knowledgeBase)
.where(eq(knowledgeBase.id, knowledgeBaseId))
.limit(1)
if (kb.length > 0) {
// Always meter the knowledge base owner
try {
await incrementStorageUsage(kb[0].userId, documentData.fileSize)
logger.info(
`[${requestId}] Updated knowledge base owner storage usage for ${documentData.fileSize} bytes`
)
} catch (error) {
logger.error(`[${requestId}] Failed to update knowledge base owner storage usage:`, error)
}
}
}
return newDocument as {
@@ -1226,28 +1180,6 @@ export async function bulkDocumentOperation(
)
)
.returning({ id: document.id, deletedAt: document.deletedAt })
// Decrement storage usage tracking
if (userId && totalSize > 0) {
// Get knowledge base owner
const kb = await db
.select({ userId: knowledgeBase.userId })
.from(knowledgeBase)
.where(eq(knowledgeBase.id, knowledgeBaseId))
.limit(1)
if (kb.length > 0) {
// Always meter the knowledge base owner
try {
await decrementStorageUsage(kb[0].userId, totalSize)
logger.info(
`[${requestId}] Updated knowledge base owner storage usage for -${totalSize} bytes`
)
} catch (error) {
logger.error(`[${requestId}] Failed to update knowledge base owner storage usage:`, error)
}
}
}
} else {
// Handle bulk enable/disable
const enabled = operation === 'enable'

View File

@@ -7,7 +7,6 @@ import { getStorageProvider, USE_BLOB_STORAGE, USE_S3_STORAGE } from '@/lib/uplo
const logger = createLogger('UploadsSetup')
// Server-only upload directory path
const PROJECT_ROOT = path.resolve(process.cwd())
export const UPLOAD_DIR_SERVER = join(PROJECT_ROOT, 'uploads')

View File

@@ -1,244 +1,7 @@
import { createLogger } from '@/lib/logs/console/logger'
import { USE_BLOB_STORAGE, USE_S3_STORAGE } from '@/lib/uploads/config'
import type { BlobConfig } from '@/lib/uploads/providers/blob/types'
import type { S3Config } from '@/lib/uploads/providers/s3/types'
import type { FileInfo, StorageConfig } from '@/lib/uploads/shared/types'
import { sanitizeFileKey } from '@/lib/uploads/utils/file-utils'
import type { StorageConfig } from '@/lib/uploads/shared/types'
const logger = createLogger('StorageClient')
export type { FileInfo, StorageConfig } from '@/lib/uploads/shared/types'
/**
* Validate and resolve local file path ensuring it's within the allowed directory
* @param key File key/name
* @param uploadDir Upload directory path
* @returns Resolved file path
* @throws Error if path is invalid or outside allowed directory
*/
async function validateLocalFilePath(key: string, uploadDir: string): Promise<string> {
const { join, resolve, sep } = await import('path')
const safeKey = sanitizeFileKey(key)
const filePath = join(uploadDir, safeKey)
const resolvedPath = resolve(filePath)
const allowedDir = resolve(uploadDir)
if (!resolvedPath.startsWith(allowedDir + sep) && resolvedPath !== allowedDir) {
throw new Error('Invalid file path')
}
return filePath
}
/**
* Upload a file to the configured storage provider
* @param file Buffer containing file data
* @param fileName Original file name
* @param contentType MIME type of the file
* @param size File size in bytes (optional, will use buffer length if not provided)
* @returns Object with file information
*/
export async function uploadFile(
file: Buffer,
fileName: string,
contentType: string,
size?: number
): Promise<FileInfo>
/**
* Upload a file to the configured storage provider with custom configuration
* @param file Buffer containing file data
* @param fileName Original file name
* @param contentType MIME type of the file
* @param customConfig Custom storage configuration
* @param size File size in bytes (optional, will use buffer length if not provided)
* @returns Object with file information
*/
export async function uploadFile(
file: Buffer,
fileName: string,
contentType: string,
customConfig: StorageConfig,
size?: number
): Promise<FileInfo>
export async function uploadFile(
file: Buffer,
fileName: string,
contentType: string,
configOrSize?: StorageConfig | number,
size?: number
): Promise<FileInfo> {
if (USE_BLOB_STORAGE) {
const { uploadToBlob } = await import('@/lib/uploads/providers/blob/client')
if (typeof configOrSize === 'object') {
if (!configOrSize.containerName || !configOrSize.accountName) {
throw new Error(
'Blob configuration missing required properties: containerName and accountName'
)
}
if (!configOrSize.connectionString && !configOrSize.accountKey) {
throw new Error(
'Blob configuration missing authentication: either connectionString or accountKey must be provided'
)
}
const blobConfig: BlobConfig = {
containerName: configOrSize.containerName,
accountName: configOrSize.accountName,
accountKey: configOrSize.accountKey,
connectionString: configOrSize.connectionString,
}
return uploadToBlob(file, fileName, contentType, blobConfig, size)
}
return uploadToBlob(file, fileName, contentType, configOrSize)
}
if (USE_S3_STORAGE) {
const { uploadToS3 } = await import('@/lib/uploads/providers/s3/client')
if (typeof configOrSize === 'object') {
if (!configOrSize.bucket || !configOrSize.region) {
throw new Error('S3 configuration missing required properties: bucket and region')
}
const s3Config: S3Config = {
bucket: configOrSize.bucket,
region: configOrSize.region,
}
return uploadToS3(file, fileName, contentType, s3Config, size)
}
return uploadToS3(file, fileName, contentType, configOrSize)
}
const { writeFile } = await import('fs/promises')
const { join } = await import('path')
const { v4: uuidv4 } = await import('uuid')
const { UPLOAD_DIR_SERVER } = await import('@/lib/uploads/core/setup.server')
const safeFileName = sanitizeFileKey(fileName)
const uniqueKey = `${uuidv4()}-${safeFileName}`
const filePath = join(UPLOAD_DIR_SERVER, uniqueKey)
try {
await writeFile(filePath, file)
} catch (error) {
logger.error(`Failed to write file to local storage: ${fileName}`, error)
throw error
}
const fileSize = typeof configOrSize === 'number' ? configOrSize : size || file.length
return {
path: `/api/files/serve/${uniqueKey}`,
key: uniqueKey,
name: fileName,
size: fileSize,
type: contentType,
}
}
/**
* Download a file from the configured storage provider
* @param key File key/name
* @returns File buffer
*/
export async function downloadFile(key: string): Promise<Buffer>
/**
* Download a file from the configured storage provider with custom configuration
* @param key File key/name
* @param customConfig Custom storage configuration
* @returns File buffer
*/
export async function downloadFile(key: string, customConfig: StorageConfig): Promise<Buffer>
export async function downloadFile(key: string, customConfig?: StorageConfig): Promise<Buffer> {
if (USE_BLOB_STORAGE) {
const { downloadFromBlob } = await import('@/lib/uploads/providers/blob/client')
if (customConfig) {
if (!customConfig.containerName || !customConfig.accountName) {
throw new Error(
'Blob configuration missing required properties: containerName and accountName'
)
}
if (!customConfig.connectionString && !customConfig.accountKey) {
throw new Error(
'Blob configuration missing authentication: either connectionString or accountKey must be provided'
)
}
const blobConfig: BlobConfig = {
containerName: customConfig.containerName,
accountName: customConfig.accountName,
accountKey: customConfig.accountKey,
connectionString: customConfig.connectionString,
}
return downloadFromBlob(key, blobConfig)
}
return downloadFromBlob(key)
}
if (USE_S3_STORAGE) {
const { downloadFromS3 } = await import('@/lib/uploads/providers/s3/client')
if (customConfig) {
if (!customConfig.bucket || !customConfig.region) {
throw new Error('S3 configuration missing required properties: bucket and region')
}
const s3Config: S3Config = {
bucket: customConfig.bucket,
region: customConfig.region,
}
return downloadFromS3(key, s3Config)
}
return downloadFromS3(key)
}
const { readFile } = await import('fs/promises')
const { UPLOAD_DIR_SERVER } = await import('@/lib/uploads/core/setup.server')
const filePath = await validateLocalFilePath(key, UPLOAD_DIR_SERVER)
try {
return await readFile(filePath)
} catch (error) {
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
throw new Error(`File not found: ${key}`)
}
throw error
}
}
/**
* Delete a file from the configured storage provider
* @param key File key/name
*/
export async function deleteFile(key: string): Promise<void> {
if (USE_BLOB_STORAGE) {
const { deleteFromBlob } = await import('@/lib/uploads/providers/blob/client')
return deleteFromBlob(key)
}
if (USE_S3_STORAGE) {
const { deleteFromS3 } = await import('@/lib/uploads/providers/s3/client')
return deleteFromS3(key)
}
const { unlink } = await import('fs/promises')
const { UPLOAD_DIR_SERVER } = await import('@/lib/uploads/core/setup.server')
const filePath = await validateLocalFilePath(key, UPLOAD_DIR_SERVER)
try {
await unlink(filePath)
} catch (error) {
// File deletion is idempotent - if file doesn't exist, that's fine
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
throw error
}
}
const { deleteFileMetadata } = await import('../server/metadata')
await deleteFileMetadata(key)
}
export type { StorageConfig } from '@/lib/uploads/shared/types'
/**
* Get the current storage provider name
@@ -250,11 +13,9 @@ export function getStorageProvider(): 'blob' | 's3' | 'local' {
}
/**
* Get the appropriate serve path prefix based on storage provider
* Get the serve path prefix (unified across all storage providers)
*/
export function getServePathPrefix(): string {
if (USE_BLOB_STORAGE) return '/api/files/serve/blob/'
if (USE_S3_STORAGE) return '/api/files/serve/s3/'
return '/api/files/serve/'
}

View File

@@ -101,6 +101,7 @@ export async function uploadFile(options: UploadFileOptions): Promise<FileInfo>
contentType,
createBlobConfig(config),
file.length,
preserveKey,
metadata
)
@@ -144,24 +145,32 @@ export async function uploadFile(options: UploadFileOptions): Promise<FileInfo>
return uploadResult
}
const { writeFile } = await import('fs/promises')
const { join } = await import('path')
const { v4: uuidv4 } = await import('uuid')
const { writeFile, mkdir } = await import('fs/promises')
const { join, dirname } = await import('path')
const { UPLOAD_DIR_SERVER } = await import('./setup.server')
const safeKey = sanitizeFileKey(keyToUse)
const uniqueKey = `${uuidv4()}-${safeKey}`
const filePath = join(UPLOAD_DIR_SERVER, uniqueKey)
const storageKey = keyToUse
const safeKey = sanitizeFileKey(keyToUse) // Validates and preserves path structure
const filesystemPath = join(UPLOAD_DIR_SERVER, safeKey)
await writeFile(filePath, file)
await mkdir(dirname(filesystemPath), { recursive: true })
await writeFile(filesystemPath, file)
if (metadata) {
await insertFileMetadataHelper(uniqueKey, metadata, context, fileName, contentType, file.length)
await insertFileMetadataHelper(
storageKey,
metadata,
context,
fileName,
contentType,
file.length
)
}
return {
path: `/api/files/serve/${uniqueKey}`,
key: uniqueKey,
path: `/api/files/serve/${storageKey}`,
key: storageKey,
name: fileName,
size: file.length,
type: contentType,
@@ -188,8 +197,14 @@ export async function downloadFile(options: DownloadFileOptions): Promise<Buffer
}
}
const { downloadFile: defaultDownload } = await import('./storage-client')
return defaultDownload(key)
const { readFile } = await import('fs/promises')
const { join } = await import('path')
const { UPLOAD_DIR_SERVER } = await import('./setup.server')
const safeKey = sanitizeFileKey(key)
const filePath = join(UPLOAD_DIR_SERVER, safeKey)
return readFile(filePath)
}
/**
@@ -212,8 +227,14 @@ export async function deleteFile(options: DeleteFileOptions): Promise<void> {
}
}
const { deleteFile: defaultDelete } = await import('@/lib/uploads/core/storage-client')
return defaultDelete(key)
const { unlink } = await import('fs/promises')
const { join } = await import('path')
const { UPLOAD_DIR_SERVER } = await import('./setup.server')
const safeKey = sanitizeFileKey(key)
const filePath = join(UPLOAD_DIR_SERVER, safeKey)
await unlink(filePath)
}
/**
@@ -423,7 +444,9 @@ export async function generatePresignedDownloadUrl(
return getPresignedUrlWithConfig(key, createBlobConfig(config), expirationSeconds)
}
return `/api/files/serve/${encodeURIComponent(key)}`
const { getBaseUrl } = await import('@/lib/urls/utils')
const baseUrl = getBaseUrl()
return `${baseUrl}/api/files/serve/${encodeURIComponent(key)}`
}
/**
@@ -432,12 +455,3 @@ export async function generatePresignedDownloadUrl(
export function hasCloudStorage(): boolean {
return USE_BLOB_STORAGE || USE_S3_STORAGE
}
/**
* Get the current storage provider name
*/
export function getStorageProviderName(): 'Azure Blob' | 'S3' | 'Local' {
if (USE_BLOB_STORAGE) return 'Azure Blob'
if (USE_S3_STORAGE) return 'S3'
return 'Local'
}

View File

@@ -12,7 +12,6 @@ export * as CopilotFiles from '@/lib/uploads/contexts/copilot'
export * as ExecutionFiles from '@/lib/uploads/contexts/execution'
export * as WorkspaceFiles from '@/lib/uploads/contexts/workspace'
export {
type FileInfo,
getFileMetadata,
getServePathPrefix,
getStorageProvider,

View File

@@ -111,7 +111,7 @@ describe('Azure Blob Storage Client', () => {
})
expect(result).toEqual({
path: expect.stringContaining('/api/files/serve/blob/'),
path: expect.stringContaining('/api/files/serve/'),
key: expect.stringContaining(fileName.replace(/\s+/g, '-')),
name: fileName,
size: testBuffer.length,

View File

@@ -47,6 +47,7 @@ export async function getBlobServiceClient(): Promise<BlobServiceClientInstance>
* @param contentType MIME type of the file
* @param configOrSize Custom Blob configuration OR file size in bytes (optional)
* @param size File size in bytes (required if configOrSize is BlobConfig, optional otherwise)
* @param preserveKey Preserve the fileName as the storage key without adding timestamp prefix (default: false)
* @param metadata Optional metadata to store with the file
* @returns Object with file information
*/
@@ -56,23 +57,17 @@ export async function uploadToBlob(
contentType: string,
configOrSize?: BlobConfig | number,
size?: number,
metadata?: Record<string, string>
): Promise<FileInfo>
export async function uploadToBlob(
file: Buffer,
fileName: string,
contentType: string,
configOrSize?: BlobConfig | number,
size?: number,
preserveKey?: boolean,
metadata?: Record<string, string>
): Promise<FileInfo> {
let config: BlobConfig
let fileSize: number
let shouldPreserveKey: boolean
if (typeof configOrSize === 'object') {
config = configOrSize
fileSize = size ?? file.length
shouldPreserveKey = preserveKey ?? false
} else {
config = {
containerName: BLOB_CONFIG.containerName,
@@ -81,10 +76,11 @@ export async function uploadToBlob(
connectionString: BLOB_CONFIG.connectionString,
}
fileSize = configOrSize ?? file.length
shouldPreserveKey = preserveKey ?? false
}
const safeFileName = fileName.replace(/\s+/g, '-') // Replace spaces with hyphens
const uniqueKey = `${Date.now()}-${safeFileName}`
const uniqueKey = shouldPreserveKey ? fileName : `${Date.now()}-${safeFileName}`
const blobServiceClient = await getBlobServiceClient()
const containerClient = blobServiceClient.getContainerClient(config.containerName)
@@ -106,7 +102,7 @@ export async function uploadToBlob(
metadata: blobMetadata,
})
const servePath = `/api/files/serve/blob/${encodeURIComponent(uniqueKey)}`
const servePath = `/api/files/serve/${encodeURIComponent(uniqueKey)}`
return {
path: servePath,
@@ -518,7 +514,7 @@ export async function completeMultipartUpload(
})
const location = blockBlobClient.url
const path = `/api/files/serve/blob/${encodeURIComponent(key)}`
const path = `/api/files/serve/${encodeURIComponent(key)}`
return {
location,

View File

@@ -90,7 +90,7 @@ describe('S3 Client', () => {
expect(mockSend).toHaveBeenCalledWith(expect.any(Object))
expect(result).toEqual({
path: expect.stringContaining('/api/files/serve/s3/'),
path: expect.stringContaining('/api/files/serve/'),
key: expect.stringContaining('test-file.txt'),
name: 'test-file.txt',
size: file.length,

View File

@@ -61,16 +61,6 @@ export function getS3Client(): S3Client {
* @param metadata Optional metadata to store with the file
* @returns Object with file information
*/
export async function uploadToS3(
file: Buffer,
fileName: string,
contentType: string,
configOrSize?: S3Config | number,
size?: number,
skipTimestampPrefix?: boolean,
metadata?: Record<string, string>
): Promise<FileInfo>
export async function uploadToS3(
file: Buffer,
fileName: string,
@@ -95,7 +85,7 @@ export async function uploadToS3(
}
const safeFileName = fileName.replace(/\s+/g, '-') // Replace spaces with hyphens
const uniqueKey = shouldSkipTimestamp ? safeFileName : `${Date.now()}-${safeFileName}`
const uniqueKey = shouldSkipTimestamp ? fileName : `${Date.now()}-${safeFileName}`
const s3Client = getS3Client()
@@ -118,7 +108,7 @@ export async function uploadToS3(
})
)
const servePath = `/api/files/serve/s3/${encodeURIComponent(uniqueKey)}`
const servePath = `/api/files/serve/${encodeURIComponent(uniqueKey)}`
return {
path: servePath,
@@ -313,7 +303,7 @@ export async function completeS3MultipartUpload(
const response = await s3Client.send(command)
const location =
response.Location || `https://${config.bucket}.s3.${config.region}.amazonaws.com/${key}`
const path = `/api/files/serve/s3/${encodeURIComponent(key)}`
const path = `/api/files/serve/${encodeURIComponent(key)}`
return {
location,

View File

@@ -214,12 +214,6 @@ export function extractStorageKey(filePath: string): string {
// If URL parsing fails, use the original path
}
if (pathWithoutQuery.includes('/api/files/serve/s3/')) {
return decodeURIComponent(pathWithoutQuery.split('/api/files/serve/s3/')[1])
}
if (pathWithoutQuery.includes('/api/files/serve/blob/')) {
return decodeURIComponent(pathWithoutQuery.split('/api/files/serve/blob/')[1])
}
if (pathWithoutQuery.startsWith('/api/files/serve/')) {
return decodeURIComponent(pathWithoutQuery.substring('/api/files/serve/'.length))
}
@@ -421,11 +415,30 @@ export function sanitizeStorageMetadata(
/**
* Sanitize a file key/path for local storage
* Removes dangerous characters and prevents path traversal
* Preserves forward slashes for structured paths (e.g., kb/file.json, workspace/id/file.json)
* All keys must have a context prefix structure
* @param key Original file key/path
* @returns Sanitized key safe for filesystem use
*/
export function sanitizeFileKey(key: string): string {
return key.replace(/[^a-zA-Z0-9.-]/g, '_').replace(/\.\./g, '')
if (!key.includes('/')) {
throw new Error('File key must include a context prefix (e.g., kb/, workspace/, execution/)')
}
const segments = key.split('/')
const sanitizedSegments = segments.map((segment, index) => {
if (segment === '..' || segment === '.') {
throw new Error('Path traversal detected in file key')
}
if (index === segments.length - 1) {
return segment.replace(/[^a-zA-Z0-9.-]/g, '_')
}
return segment.replace(/[^a-zA-Z0-9-]/g, '_')
})
return sanitizedSegments.join('/')
}
/**

View File

@@ -80,11 +80,13 @@ export function validateFileType(fileName: string, mimeType: string): FileValida
}
}
const baseMimeType = mimeType.split(';')[0].trim()
const allowedMimeTypes = SUPPORTED_MIME_TYPES[extension]
if (!allowedMimeTypes.includes(mimeType)) {
if (!allowedMimeTypes.includes(baseMimeType)) {
return {
code: 'MIME_TYPE_MISMATCH',
message: `MIME type ${mimeType} does not match file extension ${extension}. Expected: ${allowedMimeTypes.join(', ')}`,
message: `MIME type ${baseMimeType} does not match file extension ${extension}. Expected: ${allowedMimeTypes.join(', ')}`,
supportedTypes: allowedMimeTypes,
}
}