From 77bbdc1089639a11a019a5467ead74d6d089ae4c Mon Sep 17 00:00:00 2001 From: Emir Karabeg Date: Tue, 3 Jun 2025 19:56:15 -0700 Subject: [PATCH] fix(knowledge): ui and infinite load for knowledge --- apps/sim/app/api/knowledge/route.ts | 13 -- apps/sim/app/api/workflows/sync/route.ts | 3 - apps/sim/app/w/knowledge/[id]/base.tsx | 134 ++++-------- .../components/create-form/create-form.tsx | 16 +- .../components/skeletons/table-skeleton.tsx | 28 +-- apps/sim/hooks/use-knowledge.ts | 202 +++++++++++++----- apps/sim/stores/knowledge/knowledge.ts | 33 ++- 7 files changed, 233 insertions(+), 196 deletions(-) diff --git a/apps/sim/app/api/knowledge/route.ts b/apps/sim/app/api/knowledge/route.ts index 3c6188963f..09b6a39570 100644 --- a/apps/sim/app/api/knowledge/route.ts +++ b/apps/sim/app/api/knowledge/route.ts @@ -64,19 +64,6 @@ export async function GET(req: NextRequest) { .groupBy(knowledgeBase.id) .orderBy(knowledgeBase.createdAt) - // Debug logging - logger.info(`[${requestId}] Knowledge bases with counts:`, { - data: knowledgeBasesWithCounts.map((kb) => ({ - id: kb.id, - name: kb.name, - docCount: kb.docCount, - })), - }) - - logger.info( - `[${requestId}] Retrieved ${knowledgeBasesWithCounts.length} knowledge bases for user ${session.user.id}` - ) - return NextResponse.json({ success: true, data: knowledgeBasesWithCounts, diff --git a/apps/sim/app/api/workflows/sync/route.ts b/apps/sim/app/api/workflows/sync/route.ts index 115e29791c..f17dbb967c 100644 --- a/apps/sim/app/api/workflows/sync/route.ts +++ b/apps/sim/app/api/workflows/sync/route.ts @@ -202,9 +202,6 @@ export async function GET(request: Request) { } const elapsed = Date.now() - startTime - logger.info( - `[${requestId}] Workflow fetch completed in ${elapsed}ms for ${workflows.length} workflows` - ) // Return the workflows return NextResponse.json({ data: workflows }, { status: 200 }) diff --git a/apps/sim/app/w/knowledge/[id]/base.tsx b/apps/sim/app/w/knowledge/[id]/base.tsx index 7f4a3bbd60..f3dabcf7a4 100644 --- a/apps/sim/app/w/knowledge/[id]/base.tsx +++ b/apps/sim/app/w/knowledge/[id]/base.tsx @@ -62,64 +62,50 @@ function formatFileSize(bytes: number): string { } const getStatusDisplay = (doc: DocumentData) => { - const processingStatus = (() => { - switch (doc.processingStatus) { - case 'pending': - return { - text: 'Pending', - className: - 'inline-flex items-center rounded-md bg-gray-100 px-2 py-1 text-xs font-medium text-gray-700 dark:bg-gray-800 dark:text-gray-300', - } - case 'processing': - return { - text: ( - <> - - Processing - - ), - className: - 'inline-flex items-center rounded-md bg-blue-100 px-2 py-1 text-xs font-medium text-blue-700 dark:bg-blue-900/30 dark:text-blue-300', - } - case 'completed': - return { - text: 'Completed', - className: - 'inline-flex items-center rounded-md bg-green-100 px-2 py-1 text-xs font-medium text-green-700 dark:bg-green-900/30 dark:text-green-400', - } - case 'failed': - return { - text: 'Failed', - className: - 'inline-flex items-center rounded-md bg-red-100 px-2 py-1 text-xs font-medium text-red-700 dark:bg-red-900/30 dark:text-red-300', - } - default: - return { - text: 'Unknown', - className: - 'inline-flex items-center rounded-md bg-gray-100 px-2 py-1 text-xs font-medium text-gray-700 dark:bg-gray-800 dark:text-gray-300', - } - } - })() - - const activeStatus = (() => { - if (doc.processingStatus === 'completed') { + // Consolidated status: show processing status when not completed, otherwise show enabled/disabled + switch (doc.processingStatus) { + case 'pending': + return { + text: 'Pending', + className: + 'inline-flex items-center rounded-md bg-gray-100 px-2 py-1 text-xs font-medium text-gray-700 dark:bg-gray-800 dark:text-gray-300', + } + case 'processing': + return { + text: ( + <> + + Processing + + ), + className: + 'inline-flex items-center rounded-md bg-[#701FFC]/10 px-2 py-1 text-xs font-medium text-[#701FFC] dark:bg-[#701FFC]/20 dark:text-[#8B5FFF]', + } + case 'failed': + return { + text: 'Failed', + className: + 'inline-flex items-center rounded-md bg-red-100 px-2 py-1 text-xs font-medium text-red-700 dark:bg-red-900/30 dark:text-red-300', + } + case 'completed': return doc.enabled ? { text: 'Enabled', className: - 'inline-flex items-center rounded-md bg-emerald-100 px-2 py-1 text-xs font-medium text-emerald-700 dark:bg-emerald-900/30 dark:text-emerald-400', + 'inline-flex items-center rounded-md bg-green-100 px-2 py-1 text-xs font-medium text-green-700 dark:bg-green-900/30 dark:text-green-400', } : { text: 'Disabled', className: 'inline-flex items-center rounded-md bg-orange-100 px-2 py-1 text-xs font-medium text-orange-700 dark:bg-orange-900/30 dark:text-orange-400', } - } - return null - })() - - return { processingStatus, activeStatus } + default: + return { + text: 'Unknown', + className: + 'inline-flex items-center rounded-md bg-gray-100 px-2 py-1 text-xs font-medium text-gray-700 dark:bg-gray-800 dark:text-gray-300', + } + } } const getProcessingTime = (doc: DocumentData) => { @@ -583,9 +569,9 @@ export function KnowledgeBase({ onClick={handleAddDocuments} disabled={isUploading} size='sm' - className='mt-1 mr-1 bg-[#701FFC] font-[480] text-primary-foreground shadow-[0_0_0_0_#701FFC] transition-all duration-200 hover:bg-[#6518E6] hover:shadow-[0_0_0_3px_rgba(127,47,255,0.12)]' + className='flex items-center gap-1 bg-[#701FFC] font-[480] text-primary-foreground shadow-[0_0_0_0_#701FFC] transition-all duration-200 hover:bg-[#6518E6] hover:shadow-[0_0_0_3px_rgba(127,47,255,0.12)]' > - + {isUploading ? 'Uploading...' : 'Add Documents'} @@ -602,17 +588,16 @@ export function KnowledgeBase({
{/* Table header - fixed */}
- +
- + - - - + + @@ -641,11 +626,6 @@ export function KnowledgeBase({ Uploaded - @@ -661,17 +641,16 @@ export function KnowledgeBase({ {/* Table body - scrollable */}
-
- - Processing - - Status
+
- + - - - + + {filteredDocuments.length === 0 && !isLoadingDocuments ? ( @@ -713,11 +692,6 @@ export function KnowledgeBase({
—
- {/* Processing column */} - - {/* Status column */} - @@ -764,7 +735,7 @@ export function KnowledgeBase({ filteredDocuments.map((doc, index) => { const isSelected = selectedDocuments.has(doc.id) const statusDisplay = getStatusDisplay(doc) - const processingTime = getProcessingTime(doc) + // const processingTime = getProcessingTime(doc) return ( - {/* Processing column */} - - {/* Status column */} {/* Actions column */} diff --git a/apps/sim/app/w/knowledge/components/create-modal/components/create-form/create-form.tsx b/apps/sim/app/w/knowledge/components/create-modal/components/create-form/create-form.tsx index cc24044c1a..001c022216 100644 --- a/apps/sim/app/w/knowledge/components/create-modal/components/create-form/create-form.tsx +++ b/apps/sim/app/w/knowledge/components/create-modal/components/create-form/create-form.tsx @@ -442,7 +442,7 @@ export function CreateForm({ onClose, onKnowledgeBaseCreated }: CreateFormProps) onDragLeave={handleDragLeave} onDrop={handleDrop} onClick={() => fileInputRef.current?.click()} - className={`relative cursor-pointer rounded-lg border-2 border-dashed p-16 text-center transition-all duration-200 ${ + className={`relative cursor-pointer rounded-lg border-[1.5px] border-dashed p-16 text-center transition-all duration-200 ${ isDragging ? 'border-purple-300 bg-purple-50 shadow-sm' : 'border-muted-foreground/25 hover:border-muted-foreground/40 hover:bg-muted/10' @@ -457,13 +457,6 @@ export function CreateForm({ onClose, onKnowledgeBaseCreated }: CreateFormProps) multiple />
-
- 📁 -

-
- 📁 -

- {/* Processing Status column */} -

- - {/* Active Status column */} + {/* Status column */} @@ -113,17 +108,16 @@ export function DocumentTableSkeleton({
{/* Table header - fixed */}
-
-
—
-
—
@@ -752,9 +726,6 @@ export function KnowledgeBase({
-
-
-
- {statusDisplay.processingStatus.text} -
- 		- {statusDisplay.activeStatus ? ( -
- {statusDisplay.activeStatus.text} -
- ) : ( -
—
- )} +
{statusDisplay.text}
 -
-
+
- + - - - + + @@ -145,9 +139,6 @@ export function DocumentTableSkeleton({ - @@ -161,17 +152,16 @@ export function DocumentTableSkeleton({ {/* Table body - scrollable */}
-
Uploaded - Processing - Status
+
- + - - - + + {Array.from({ length: rowCount }).map((_, i) => ( diff --git a/apps/sim/hooks/use-knowledge.ts b/apps/sim/hooks/use-knowledge.ts index 055d4aea53..dfd2d5f0a4 100644 --- a/apps/sim/hooks/use-knowledge.ts +++ b/apps/sim/hooks/use-knowledge.ts @@ -10,19 +10,27 @@ export function useKnowledgeBase(id: string) { const isLoading = loadingKnowledgeBases.has(id) useEffect(() => { + if (!id || knowledgeBase || isLoading) return + + let isMounted = true + const loadData = async () => { try { setError(null) await getKnowledgeBase(id) } catch (err) { - setError(err instanceof Error ? err.message : 'Failed to load knowledge base') + if (isMounted) { + setError(err instanceof Error ? err.message : 'Failed to load knowledge base') + } } } - if (id && !knowledgeBase && !isLoading) { - loadData() + loadData() + + return () => { + isMounted = false } - }, [id, knowledgeBase, isLoading, getKnowledgeBase]) + }, [id, knowledgeBase, isLoading]) // Removed getKnowledgeBase from dependencies return { knowledgeBase, @@ -41,19 +49,27 @@ export function useKnowledgeBaseDocuments(knowledgeBaseId: string) { const isLoading = loadingDocuments.has(knowledgeBaseId) useEffect(() => { + if (!knowledgeBaseId || documents.length > 0 || isLoading) return + + let isMounted = true + const loadData = async () => { try { setError(null) await getDocuments(knowledgeBaseId) } catch (err) { - setError(err instanceof Error ? err.message : 'Failed to load documents') + if (isMounted) { + setError(err instanceof Error ? err.message : 'Failed to load documents') + } } } - if (knowledgeBaseId && documents.length === 0 && !isLoading) { - loadData() + loadData() + + return () => { + isMounted = false } - }, [knowledgeBaseId, documents.length, isLoading, getDocuments]) + }, [knowledgeBaseId, documents.length, isLoading]) // Removed getDocuments from dependencies const refreshDocumentsData = async () => { try { @@ -88,29 +104,95 @@ export function useKnowledgeBasesList() { } = useKnowledgeStore() const [error, setError] = useState(null) + const [retryCount, setRetryCount] = useState(0) + const maxRetries = 3 useEffect(() => { - const loadData = async () => { + if (knowledgeBasesList.length > 0 || loadingKnowledgeBasesList) return + + let isMounted = true + let retryTimeoutId: NodeJS.Timeout | null = null + + const loadData = async (attempt = 0) => { + // Don't proceed if component is unmounted + if (!isMounted) return + try { setError(null) await getKnowledgeBasesList() + + // Reset retry count on success + if (isMounted) { + setRetryCount(0) + } } catch (err) { - setError(err instanceof Error ? err.message : 'Failed to load knowledge bases') + if (!isMounted) return + + const errorMessage = err instanceof Error ? err.message : 'Failed to load knowledge bases' + + // Only set error and retry if we haven't exceeded max retries + if (attempt < maxRetries) { + console.warn(`Knowledge bases load attempt ${attempt + 1} failed, retrying...`, err) + setRetryCount(attempt + 1) + + // Exponential backoff: 1s, 2s, 4s + const delay = 2 ** attempt * 1000 + retryTimeoutId = setTimeout(() => { + if (isMounted) { + loadData(attempt + 1) + } + }, delay) + } else { + console.error('All retry attempts failed for knowledge bases list:', err) + setError(errorMessage) + setRetryCount(maxRetries) + } } } - if (knowledgeBasesList.length === 0 && !loadingKnowledgeBasesList) { - loadData() + // Always start from attempt 0 + loadData(0) + + // Cleanup function + return () => { + isMounted = false + if (retryTimeoutId) { + clearTimeout(retryTimeoutId) + } } - }, [knowledgeBasesList.length, loadingKnowledgeBasesList, getKnowledgeBasesList]) + }, [knowledgeBasesList.length, loadingKnowledgeBasesList]) // Removed getKnowledgeBasesList from dependencies const refreshList = async () => { try { setError(null) + setRetryCount(0) clearKnowledgeBasesList() await getKnowledgeBasesList() } catch (err) { - setError(err instanceof Error ? err.message : 'Failed to refresh knowledge bases') + const errorMessage = err instanceof Error ? err.message : 'Failed to refresh knowledge bases' + setError(errorMessage) + console.error('Error refreshing knowledge bases list:', err) + } + } + + // Force refresh function that bypasses cache and resets everything + const forceRefresh = async () => { + setError(null) + setRetryCount(0) + clearKnowledgeBasesList() + + // Force reload by clearing cache and loading state + useKnowledgeStore.setState({ + knowledgeBasesList: [], + loadingKnowledgeBasesList: false, + }) + + try { + await getKnowledgeBasesList() + } catch (err) { + const errorMessage = err instanceof Error ? err.message : 'Failed to refresh knowledge bases' + setError(errorMessage) + console.error('Error force refreshing knowledge bases list:', err) } } @@ -119,8 +201,11 @@ export function useKnowledgeBasesList() { isLoading: loadingKnowledgeBasesList, error, refreshList, + forceRefresh, addKnowledgeBase, removeKnowledgeBase, + retryCount, + maxRetries, } } @@ -140,79 +225,86 @@ export function useDocumentChunks(knowledgeBaseId: string, documentId: string) { offset: 0, hasMore: false, }) + const [initialLoadDone, setInitialLoadDone] = useState(false) const isStoreLoading = isChunksLoading(documentId) const combinedIsLoading = isLoading || isStoreLoading + // Single effect to handle all data loading and syncing useEffect(() => { if (!knowledgeBaseId || !documentId) return - const cached = getCachedChunks(documentId) - if (cached) { - setChunks(cached.chunks) - setPagination(cached.pagination) - setIsLoading(false) - } - }, [knowledgeBaseId, documentId, getCachedChunks]) + let isMounted = true - // Initial load - useEffect(() => { - if (!knowledgeBaseId || !documentId) return - - const loadChunks = async () => { + const loadAndSyncData = async () => { try { - setIsLoading(true) - setError(null) - - // Try to get cached chunks first + // Check cache first const cached = getCachedChunks(documentId) if (cached) { - setChunks(cached.chunks) - setPagination(cached.pagination) - setIsLoading(false) + if (isMounted) { + setChunks(cached.chunks) + setPagination(cached.pagination) + setIsLoading(false) + setInitialLoadDone(true) + } return } - // If not cached, fetch from API - const fetchedChunks = await getChunks(knowledgeBaseId, documentId, { - limit: pagination.limit, - offset: pagination.offset, - }) + // If not cached and we haven't done initial load, fetch from API + if (!initialLoadDone && !isStoreLoading) { + setIsLoading(true) + setError(null) - setChunks(fetchedChunks) + const fetchedChunks = await getChunks(knowledgeBaseId, documentId, { + limit: 50, // Use fixed initial values to avoid dependency issues + offset: 0, + }) - // Update pagination from cache after fetch - const updatedCache = getCachedChunks(documentId) - if (updatedCache) { - setPagination(updatedCache.pagination) + if (isMounted) { + setChunks(fetchedChunks) + + // Update pagination from cache after fetch + const updatedCache = getCachedChunks(documentId) + if (updatedCache) { + setPagination(updatedCache.pagination) + } + + setInitialLoadDone(true) + } } } catch (err) { - setError(err instanceof Error ? err.message : 'Failed to load chunks') + if (isMounted) { + setError(err instanceof Error ? err.message : 'Failed to load chunks') + } } finally { - setIsLoading(false) + if (isMounted) { + setIsLoading(false) + } } } - loadChunks() - }, [knowledgeBaseId, documentId, getChunks, getCachedChunks]) + loadAndSyncData() - // Sync with store state changes + return () => { + isMounted = false + } + }, [knowledgeBaseId, documentId, isStoreLoading, initialLoadDone]) // Removed getCachedChunks and getChunks from dependencies + + // Separate effect to sync with store state changes (no API calls) useEffect(() => { + if (!documentId || !initialLoadDone) return + const cached = getCachedChunks(documentId) if (cached) { setChunks(cached.chunks) setPagination(cached.pagination) } - }, [documentId, getCachedChunks]) - useEffect(() => { + // Update loading state based on store if (!isStoreLoading && isLoading) { - const cached = getCachedChunks(documentId) - if (cached) { - setIsLoading(false) - } + setIsLoading(false) } - }, [isStoreLoading, isLoading, documentId, getCachedChunks]) + }, [documentId, isStoreLoading, isLoading, initialLoadDone]) // Removed getCachedChunks from dependencies const refreshChunksData = async (options?: { search?: string diff --git a/apps/sim/stores/knowledge/knowledge.ts b/apps/sim/stores/knowledge/knowledge.ts index 272e8ec07b..29644c2ed6 100644 --- a/apps/sim/stores/knowledge/knowledge.ts +++ b/apps/sim/stores/knowledge/knowledge.ts @@ -373,13 +373,29 @@ export const useKnowledgeStore = create((set, get) => ({ return state.knowledgeBasesList } + // Create an AbortController for request cancellation + const abortController = new AbortController() + const timeoutId = setTimeout(() => { + abortController.abort() + }, 10000) // 10 second timeout + try { set({ loadingKnowledgeBasesList: true }) - const response = await fetch('/api/knowledge') + const response = await fetch('/api/knowledge', { + signal: abortController.signal, + headers: { + 'Content-Type': 'application/json', + }, + }) + + // Clear the timeout since request completed + clearTimeout(timeoutId) if (!response.ok) { - throw new Error(`Failed to fetch knowledge bases: ${response.statusText}`) + throw new Error( + `Failed to fetch knowledge bases: ${response.status} ${response.statusText}` + ) } const result = await response.json() @@ -388,7 +404,7 @@ export const useKnowledgeStore = create((set, get) => ({ throw new Error(result.error || 'Failed to fetch knowledge bases') } - const knowledgeBasesList = result.data + const knowledgeBasesList = result.data || [] set({ knowledgeBasesList, @@ -398,9 +414,20 @@ export const useKnowledgeStore = create((set, get) => ({ logger.info(`Knowledge bases list loaded: ${knowledgeBasesList.length} items`) return knowledgeBasesList } catch (error) { + // Clear the timeout in case of error + clearTimeout(timeoutId) + logger.error('Error fetching knowledge bases list:', error) + // Always set loading to false, even on error set({ loadingKnowledgeBasesList: false }) + + // Don't throw on AbortError (timeout or cancellation) + if (error instanceof Error && error.name === 'AbortError') { + logger.warn('Knowledge bases list request was aborted (timeout or cancellation)') + return state.knowledgeBasesList // Return whatever we have cached + } + throw error } },