github/sim
1
0
Fork 0
mirror of https://github.com/simstudioai/sim.git synced 2026-01-08 22:48:14 -05:00
Code Issues Packages Projects Releases Wiki Activity

improvement(kb): add fallbacks for kb configs (#1199)

Browse Source
This commit is contained in:
Waleed
2025-08-29 21:09:09 -07:00
committed by GitHub
parent a3838302e0
commit f4f8fc051e
2 changed files with 18 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
apps/sim/background/knowledge-processing.ts
View File

@@ -26,15 +26,15 @@ export type DocumentProcessingPayload = {
export const processDocument = task({
id: 'knowledge-process-document',
maxDuration: env.KB_CONFIG_MAX_DURATION,
maxDuration: env.KB_CONFIG_MAX_DURATION || 300,
retry: {
maxAttempts: env.KB_CONFIG_MAX_ATTEMPTS,
factor: env.KB_CONFIG_RETRY_FACTOR,
minTimeoutInMs: env.KB_CONFIG_MIN_TIMEOUT,
maxTimeoutInMs: env.KB_CONFIG_MAX_TIMEOUT,
maxAttempts: env.KB_CONFIG_MAX_ATTEMPTS || 3,
factor: env.KB_CONFIG_RETRY_FACTOR || 2,
minTimeoutInMs: env.KB_CONFIG_MIN_TIMEOUT || 1000,
maxTimeoutInMs: env.KB_CONFIG_MAX_TIMEOUT || 10000,
},
queue: {
concurrencyLimit: env.KB_CONFIG_CONCURRENCY_LIMIT,
concurrencyLimit: env.KB_CONFIG_CONCURRENCY_LIMIT || 20,
name: 'document-processing-queue',
},
run: async (payload: DocumentProcessingPayload) => {

24
apps/sim/lib/knowledge/documents/service.ts
View File

@@ -17,8 +17,8 @@ import type { DocumentSortField, SortOrder } from './types'
const logger = createLogger('DocumentService')
const TIMEOUTS = {
OVERALL_PROCESSING: env.KB_CONFIG_MAX_DURATION * 1000,
EMBEDDINGS_API: env.KB_CONFIG_MAX_TIMEOUT * 18,
OVERALL_PROCESSING: (env.KB_CONFIG_MAX_DURATION || 300) * 1000,
EMBEDDINGS_API: (env.KB_CONFIG_MAX_TIMEOUT || 10000) * 18,
} as const
/**
@@ -38,17 +38,17 @@ function withTimeout<T>(
}
const PROCESSING_CONFIG = {
maxConcurrentDocuments: Math.max(1, Math.floor(env.KB_CONFIG_CONCURRENCY_LIMIT / 5)) || 4,
batchSize: Math.max(1, Math.floor(env.KB_CONFIG_BATCH_SIZE / 2)) || 10,
delayBetweenBatches: env.KB_CONFIG_DELAY_BETWEEN_BATCHES * 2,
delayBetweenDocuments: env.KB_CONFIG_DELAY_BETWEEN_DOCUMENTS * 2,
maxConcurrentDocuments: Math.max(1, Math.floor((env.KB_CONFIG_CONCURRENCY_LIMIT || 20) / 5)) || 4,
batchSize: Math.max(1, Math.floor((env.KB_CONFIG_BATCH_SIZE || 20) / 2)) || 10,
delayBetweenBatches: (env.KB_CONFIG_DELAY_BETWEEN_BATCHES || 100) * 2,
delayBetweenDocuments: (env.KB_CONFIG_DELAY_BETWEEN_DOCUMENTS || 50) * 2,
}
const REDIS_PROCESSING_CONFIG = {
maxConcurrentDocuments: env.KB_CONFIG_CONCURRENCY_LIMIT,
batchSize: env.KB_CONFIG_BATCH_SIZE,
delayBetweenBatches: env.KB_CONFIG_DELAY_BETWEEN_BATCHES,
delayBetweenDocuments: env.KB_CONFIG_DELAY_BETWEEN_DOCUMENTS,
maxConcurrentDocuments: env.KB_CONFIG_CONCURRENCY_LIMIT || 20,
batchSize: env.KB_CONFIG_BATCH_SIZE || 20,
delayBetweenBatches: env.KB_CONFIG_DELAY_BETWEEN_BATCHES || 100,
delayBetweenDocuments: env.KB_CONFIG_DELAY_BETWEEN_DOCUMENTS || 50,
}
let documentQueue: DocumentProcessingQueue | null = null
@@ -59,8 +59,8 @@ export function getDocumentQueue(): DocumentProcessingQueue {
const config = redisClient ? REDIS_PROCESSING_CONFIG : PROCESSING_CONFIG
documentQueue = new DocumentProcessingQueue({
maxConcurrent: config.maxConcurrentDocuments,
retryDelay: env.KB_CONFIG_MIN_TIMEOUT,
maxRetries: env.KB_CONFIG_MAX_ATTEMPTS,
retryDelay: env.KB_CONFIG_MIN_TIMEOUT || 1000,
maxRetries: env.KB_CONFIG_MAX_ATTEMPTS || 3,
})
}
return documentQueue