diff --git a/apps/sim/app/api/knowledge/[id]/documents/route.test.ts b/apps/sim/app/api/knowledge/[id]/documents/route.test.ts index 9ba218cbe7..a8117b38f7 100644 --- a/apps/sim/app/api/knowledge/[id]/documents/route.test.ts +++ b/apps/sim/app/api/knowledge/[id]/documents/route.test.ts @@ -30,6 +30,8 @@ describe('Knowledge Base Documents API Route', () => { from: vi.fn().mockReturnThis(), where: vi.fn().mockReturnThis(), orderBy: vi.fn().mockReturnThis(), + limit: vi.fn().mockReturnThis(), + offset: vi.fn().mockReturnThis(), insert: vi.fn().mockReturnThis(), values: vi.fn().mockReturnThis(), update: vi.fn().mockReturnThis(), @@ -99,7 +101,12 @@ describe('Knowledge Base Documents API Route', () => { it('should retrieve documents successfully for authenticated user', async () => { mockAuth$.mockAuthenticatedUser() mockCheckKnowledgeBaseAccess.mockResolvedValue({ hasAccess: true }) - mockDbChain.orderBy.mockResolvedValue([mockDocument]) + + // Mock the count query (first query) + mockDbChain.where.mockResolvedValueOnce([{ count: 1 }]) + + // Mock the documents query (second query) + mockDbChain.offset.mockResolvedValue([mockDocument]) const req = createMockRequest('GET') const { GET } = await import('./route') @@ -108,8 +115,8 @@ describe('Knowledge Base Documents API Route', () => { expect(response.status).toBe(200) expect(data.success).toBe(true) - expect(data.data).toHaveLength(1) - expect(data.data[0].id).toBe('doc-123') + expect(data.data.documents).toHaveLength(1) + expect(data.data.documents[0].id).toBe('doc-123') expect(mockDbChain.select).toHaveBeenCalled() expect(mockCheckKnowledgeBaseAccess).toHaveBeenCalledWith('kb-123', 'user-123') }) @@ -117,7 +124,12 @@ describe('Knowledge Base Documents API Route', () => { it('should filter disabled documents by default', async () => { mockAuth$.mockAuthenticatedUser() mockCheckKnowledgeBaseAccess.mockResolvedValue({ hasAccess: true }) - mockDbChain.orderBy.mockResolvedValue([mockDocument]) + + // Mock the count query (first query) + mockDbChain.where.mockResolvedValueOnce([{ count: 1 }]) + + // Mock the documents query (second query) + mockDbChain.offset.mockResolvedValue([mockDocument]) const req = createMockRequest('GET') const { GET } = await import('./route') @@ -130,7 +142,12 @@ describe('Knowledge Base Documents API Route', () => { it('should include disabled documents when requested', async () => { mockAuth$.mockAuthenticatedUser() mockCheckKnowledgeBaseAccess.mockResolvedValue({ hasAccess: true }) - mockDbChain.orderBy.mockResolvedValue([mockDocument]) + + // Mock the count query (first query) + mockDbChain.where.mockResolvedValueOnce([{ count: 1 }]) + + // Mock the documents query (second query) + mockDbChain.offset.mockResolvedValue([mockDocument]) const url = 'http://localhost:3000/api/knowledge/kb-123/documents?includeDisabled=true' const req = new Request(url, { method: 'GET' }) as any diff --git a/apps/sim/app/api/knowledge/[id]/documents/route.ts b/apps/sim/app/api/knowledge/[id]/documents/route.ts index 1e683209b5..8091cfafaa 100644 --- a/apps/sim/app/api/knowledge/[id]/documents/route.ts +++ b/apps/sim/app/api/knowledge/[id]/documents/route.ts @@ -1,5 +1,5 @@ import crypto from 'node:crypto' -import { and, desc, eq, inArray, isNull } from 'drizzle-orm' +import { and, desc, eq, inArray, isNull, sql } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { getSession } from '@/lib/auth' @@ -210,6 +210,9 @@ export async function GET(req: NextRequest, { params }: { params: Promise<{ id: const url = new URL(req.url) const includeDisabled = url.searchParams.get('includeDisabled') === 'true' + const search = url.searchParams.get('search') + const limit = Number.parseInt(url.searchParams.get('limit') || '50') + const offset = Number.parseInt(url.searchParams.get('offset') || '0') // Build where conditions const whereConditions = [ @@ -222,6 +225,23 @@ export async function GET(req: NextRequest, { params }: { params: Promise<{ id: whereConditions.push(eq(document.enabled, true)) } + // Add search condition if provided + if (search) { + whereConditions.push( + // Search in filename + sql`LOWER(${document.filename}) LIKE LOWER(${`%${search}%`})` + ) + } + + // Get total count for pagination + const totalResult = await db + .select({ count: sql`COUNT(*)` }) + .from(document) + .where(and(...whereConditions)) + + const total = totalResult[0]?.count || 0 + const hasMore = offset + limit < total + const documents = await db .select({ id: document.id, @@ -250,14 +270,24 @@ export async function GET(req: NextRequest, { params }: { params: Promise<{ id: .from(document) .where(and(...whereConditions)) .orderBy(desc(document.uploadedAt)) + .limit(limit) + .offset(offset) logger.info( - `[${requestId}] Retrieved ${documents.length} documents for knowledge base ${knowledgeBaseId}` + `[${requestId}] Retrieved ${documents.length} documents (${offset}-${offset + documents.length} of ${total}) for knowledge base ${knowledgeBaseId}` ) return NextResponse.json({ success: true, - data: documents, + data: { + documents, + pagination: { + total, + limit, + offset, + hasMore, + }, + }, }) } catch (error) { logger.error(`[${requestId}] Error fetching documents`, error) diff --git a/apps/sim/app/api/knowledge/utils.test.ts b/apps/sim/app/api/knowledge/utils.test.ts index 5e1188b2de..c5f0421df6 100644 --- a/apps/sim/app/api/knowledge/utils.test.ts +++ b/apps/sim/app/api/knowledge/utils.test.ts @@ -176,7 +176,7 @@ describe('Knowledge Utils', () => { {} ) - expect(dbOps.order).toEqual(['insert', 'updateDoc', 'updateKb']) + expect(dbOps.order).toEqual(['insert', 'updateDoc']) expect(dbOps.updatePayloads[0]).toMatchObject({ processingStatus: 'completed', diff --git a/apps/sim/app/api/knowledge/utils.ts b/apps/sim/app/api/knowledge/utils.ts index 1c36361063..55900d94b8 100644 --- a/apps/sim/app/api/knowledge/utils.ts +++ b/apps/sim/app/api/knowledge/utils.ts @@ -1,5 +1,5 @@ import crypto from 'crypto' -import { and, eq, isNull, sql } from 'drizzle-orm' +import { and, eq, isNull } from 'drizzle-orm' import { processDocument } from '@/lib/documents/document-processor' import { retryWithExponentialBackoff } from '@/lib/documents/utils' import { env } from '@/lib/env' @@ -522,14 +522,6 @@ export async function processDocumentAsync( processingError: null, }) .where(eq(document.id, documentId)) - - await tx - .update(knowledgeBase) - .set({ - tokenCount: sql`${knowledgeBase.tokenCount} + ${processed.metadata.tokenCount}`, - updatedAt: now, - }) - .where(eq(knowledgeBase.id, knowledgeBaseId)) }) })(), TIMEOUTS.OVERALL_PROCESSING, diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx index 8f73295dac..b71155eea5 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx @@ -117,7 +117,7 @@ export function Document({ setError(null) const cachedDocuments = getCachedDocuments(knowledgeBaseId) - const cachedDoc = cachedDocuments?.find((d) => d.id === documentId) + const cachedDoc = cachedDocuments?.documents?.find((d) => d.id === documentId) if (cachedDoc) { setDocument(cachedDoc) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx index 422f638dd0..d87d77d6f5 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx @@ -1,9 +1,11 @@ 'use client' -import { useEffect, useState } from 'react' +import { useCallback, useEffect, useState } from 'react' import { format } from 'date-fns' import { AlertCircle, + ChevronLeft, + ChevronRight, Circle, CircleOff, FileText, @@ -25,6 +27,7 @@ import { } from '@/components/ui/alert-dialog' import { Button } from '@/components/ui/button' import { Checkbox } from '@/components/ui/checkbox' +import { SearchHighlight } from '@/components/ui/search-highlight' import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip' import { createLogger } from '@/lib/logs/console-logger' import { ActionBar } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/action-bar/action-bar' @@ -40,6 +43,9 @@ import { UploadModal } from './components/upload-modal/upload-modal' const logger = createLogger('KnowledgeBase') +// Constants +const DOCUMENTS_PER_PAGE = 50 + interface KnowledgeBaseProps { id: string knowledgeBaseName?: string @@ -118,6 +124,22 @@ export function KnowledgeBase({ const { removeKnowledgeBase } = useKnowledgeStore() const params = useParams() const workspaceId = params.workspaceId as string + + const [searchQuery, setSearchQuery] = useState('') + + // Memoize the search query setter to prevent unnecessary re-renders + const handleSearchChange = useCallback((newQuery: string) => { + setSearchQuery(newQuery) + setCurrentPage(1) // Reset to page 1 when searching + }, []) + + const [selectedDocuments, setSelectedDocuments] = useState>(new Set()) + const [showDeleteDialog, setShowDeleteDialog] = useState(false) + const [showUploadModal, setShowUploadModal] = useState(false) + const [isDeleting, setIsDeleting] = useState(false) + const [isBulkOperating, setIsBulkOperating] = useState(false) + const [currentPage, setCurrentPage] = useState(1) + const { knowledgeBase, isLoading: isLoadingKnowledgeBase, @@ -125,27 +147,52 @@ export function KnowledgeBase({ } = useKnowledgeBase(id) const { documents, + pagination, isLoading: isLoadingDocuments, error: documentsError, updateDocument, refreshDocuments, - } = useKnowledgeBaseDocuments(id) + } = useKnowledgeBaseDocuments(id, { + search: searchQuery || undefined, + limit: DOCUMENTS_PER_PAGE, + offset: (currentPage - 1) * DOCUMENTS_PER_PAGE, + }) const isSidebarCollapsed = mode === 'expanded' ? !isExpanded : mode === 'collapsed' || mode === 'hover' - const [searchQuery, setSearchQuery] = useState('') - const [selectedDocuments, setSelectedDocuments] = useState>(new Set()) - const [showDeleteDialog, setShowDeleteDialog] = useState(false) - const [showUploadModal, setShowUploadModal] = useState(false) - const [isDeleting, setIsDeleting] = useState(false) - const [isBulkOperating, setIsBulkOperating] = useState(false) - const router = useRouter() const knowledgeBaseName = knowledgeBase?.name || passedKnowledgeBaseName || 'Knowledge Base' const error = knowledgeBaseError || documentsError + // Pagination calculations + const totalPages = Math.ceil(pagination.total / pagination.limit) + const hasNextPage = currentPage < totalPages + const hasPrevPage = currentPage > 1 + + // Navigation functions + const goToPage = useCallback( + (page: number) => { + if (page >= 1 && page <= totalPages) { + setCurrentPage(page) + } + }, + [totalPages] + ) + + const nextPage = useCallback(() => { + if (hasNextPage) { + setCurrentPage((prev) => prev + 1) + } + }, [hasNextPage]) + + const prevPage = useCallback(() => { + if (hasPrevPage) { + setCurrentPage((prev) => prev - 1) + } + }, [hasPrevPage]) + // Auto-refresh documents when there are processing documents useEffect(() => { const hasProcessingDocuments = documents.some( @@ -220,10 +267,8 @@ export function KnowledgeBase({ await Promise.allSettled(markFailedPromises) } - // Filter documents based on search query - const filteredDocuments = documents.filter((doc) => - doc.filename.toLowerCase().includes(searchQuery.toLowerCase()) - ) + // Calculate pagination info for display + const totalItems = pagination?.total || 0 const handleToggleEnabled = async (docId: string) => { const document = documents.find((doc) => doc.id === docId) @@ -366,14 +411,13 @@ export function KnowledgeBase({ const handleSelectAll = (checked: boolean) => { if (checked) { - setSelectedDocuments(new Set(filteredDocuments.map((doc) => doc.id))) + setSelectedDocuments(new Set(documents.map((doc) => doc.id))) } else { setSelectedDocuments(new Set()) } } - const isAllSelected = - filteredDocuments.length > 0 && selectedDocuments.size === filteredDocuments.length + const isAllSelected = documents.length > 0 && selectedDocuments.size === documents.length const handleDocumentClick = (docId: string) => { // Find the document to get its filename @@ -621,20 +665,35 @@ export function KnowledgeBase({ {/* Main Content */}
- {/* Search and Create Section */} -
- + {/* Search and Filters Section */} +
+
+ -
- {/* Add Documents Button */} - - - Add Documents - +
+ {/* Clear Search Button */} + {searchQuery && ( + + )} + + {/* Add Documents Button */} + + + Add Documents + +
@@ -714,7 +773,7 @@ export function KnowledgeBase({ - {filteredDocuments.length === 0 && !isLoadingDocuments ? ( + {documents.length === 0 && !isLoadingDocuments ? ( {/* Select column */} @@ -726,7 +785,7 @@ export function KnowledgeBase({
- {documents.length === 0 + {totalItems === 0 ? 'No documents yet' : 'No documents match your search'} @@ -793,7 +852,7 @@ export function KnowledgeBase({ )) ) : ( - filteredDocuments.map((doc) => { + documents.map((doc) => { const isSelected = selectedDocuments.has(doc.id) const statusDisplay = getStatusDisplay(doc) // const processingTime = getProcessingTime(doc) @@ -834,7 +893,10 @@ export function KnowledgeBase({ - {doc.filename} + {doc.filename} @@ -998,6 +1060,64 @@ export function KnowledgeBase({
+ + {/* Pagination Controls */} + {totalPages > 1 && ( +
+
+ + + {/* Page numbers - show a few around current page */} +
+ {Array.from({ length: Math.min(totalPages, 5) }, (_, i) => { + let page: number + if (totalPages <= 5) { + page = i + 1 + } else if (currentPage <= 3) { + page = i + 1 + } else if (currentPage >= totalPages - 2) { + page = totalPages - 4 + i + } else { + page = currentPage - 2 + i + } + + if (page < 1 || page > totalPages) return null + + return ( + + ) + })} +
+ + +
+
+ )}
@@ -1011,8 +1131,8 @@ export function KnowledgeBase({ Delete Knowledge Base Are you sure you want to delete "{knowledgeBaseName}"? This will permanently delete - the knowledge base and all {documents.length} document - {documents.length === 1 ? '' : 's'} within it. This action cannot be undone. + the knowledge base and all {totalItems} document + {totalItems === 1 ? '' : 's'} within it. This action cannot be undone. diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/components/document-selector/document-selector.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/components/document-selector/document-selector.tsx index 463610e384..6ef02537c0 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/components/document-selector/document-selector.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/components/document-selector/document-selector.tsx @@ -87,7 +87,7 @@ export function DocumentSelector({ throw new Error(result.error || 'Failed to fetch documents') } - const fetchedDocuments = result.data || [] + const fetchedDocuments = result.data.documents || result.data || [] setDocuments(fetchedDocuments) } catch (err) { if ((err as Error).name === 'AbortError') return diff --git a/apps/sim/components/ui/search-highlight.tsx b/apps/sim/components/ui/search-highlight.tsx index 0c94729d3b..49fa427521 100644 --- a/apps/sim/components/ui/search-highlight.tsx +++ b/apps/sim/components/ui/search-highlight.tsx @@ -1,7 +1,5 @@ 'use client' -import { Fragment } from 'react' - interface SearchHighlightProps { text: string searchQuery: string @@ -13,22 +11,18 @@ export function SearchHighlight({ text, searchQuery, className = '' }: SearchHig return {text} } - // Create a regex to find matches (case-insensitive) - // Handle multiple search terms separated by spaces + // Create regex pattern for all search terms const searchTerms = searchQuery .trim() .split(/\s+/) .filter((term) => term.length > 0) + .map((term) => term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')) if (searchTerms.length === 0) { return {text} } - // Create regex pattern for all search terms - const escapedTerms = searchTerms.map((term) => term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')) - const regexPattern = `(${escapedTerms.join('|')})` - const regex = new RegExp(regexPattern, 'gi') - + const regex = new RegExp(`(${searchTerms.join('|')})`, 'gi') const parts = text.split(regex) return ( @@ -36,18 +30,17 @@ export function SearchHighlight({ text, searchQuery, className = '' }: SearchHig {parts.map((part, index) => { if (!part) return null - const isMatch = regex.test(part) + const isMatch = searchTerms.some((term) => new RegExp(term, 'gi').test(part)) - return ( - - {isMatch ? ( - - {part} - - ) : ( - part - )} - + return isMatch ? ( + + {part} + + ) : ( + {part} ) })} diff --git a/apps/sim/hooks/use-knowledge.ts b/apps/sim/hooks/use-knowledge.ts index fc4b14dd19..653f96070c 100644 --- a/apps/sim/hooks/use-knowledge.ts +++ b/apps/sim/hooks/use-knowledge.ts @@ -40,24 +40,33 @@ export function useKnowledgeBase(id: string) { } } -export function useKnowledgeBaseDocuments(knowledgeBaseId: string) { +// Constants +const MAX_DOCUMENTS_LIMIT = 10000 +const DEFAULT_PAGE_SIZE = 50 + +export function useKnowledgeBaseDocuments( + knowledgeBaseId: string, + options?: { search?: string; limit?: number; offset?: number } +) { const { getDocuments, getCachedDocuments, loadingDocuments, updateDocument, refreshDocuments } = useKnowledgeStore() const [error, setError] = useState(null) - const documents = getCachedDocuments(knowledgeBaseId) || [] + const documentsCache = getCachedDocuments(knowledgeBaseId) + const allDocuments = documentsCache?.documents || [] const isLoading = loadingDocuments.has(knowledgeBaseId) + // Load all documents on initial mount useEffect(() => { - if (!knowledgeBaseId || documents.length > 0 || isLoading) return + if (!knowledgeBaseId || allDocuments.length > 0 || isLoading) return let isMounted = true - const loadData = async () => { + const loadAllDocuments = async () => { try { setError(null) - await getDocuments(knowledgeBaseId) + await getDocuments(knowledgeBaseId, { limit: MAX_DOCUMENTS_LIMIT }) } catch (err) { if (isMounted) { setError(err instanceof Error ? err.message : 'Failed to load documents') @@ -65,28 +74,59 @@ export function useKnowledgeBaseDocuments(knowledgeBaseId: string) { } } - loadData() + loadAllDocuments() return () => { isMounted = false } - }, [knowledgeBaseId, documents.length, isLoading]) // Removed getDocuments from dependencies + }, [knowledgeBaseId, allDocuments.length, isLoading, getDocuments]) - const refreshDocumentsData = async () => { + // Client-side filtering and pagination + const { documents, pagination } = useMemo(() => { + let filteredDocs = allDocuments + + // Apply search filter + if (options?.search) { + const searchLower = options.search.toLowerCase() + filteredDocs = filteredDocs.filter((doc) => doc.filename.toLowerCase().includes(searchLower)) + } + + // Apply pagination + const offset = options?.offset || 0 + const limit = options?.limit || DEFAULT_PAGE_SIZE + const total = filteredDocs.length + const paginatedDocs = filteredDocs.slice(offset, offset + limit) + + return { + documents: paginatedDocs, + pagination: { + total, + limit, + offset, + hasMore: offset + limit < total, + }, + } + }, [allDocuments, options?.search, options?.limit, options?.offset]) + + const refreshDocumentsData = useCallback(async () => { try { setError(null) - await refreshDocuments(knowledgeBaseId) + await refreshDocuments(knowledgeBaseId, { limit: MAX_DOCUMENTS_LIMIT }) } catch (err) { setError(err instanceof Error ? err.message : 'Failed to refresh documents') } - } + }, [knowledgeBaseId, refreshDocuments]) - const updateDocumentLocal = (documentId: string, updates: Partial) => { - updateDocument(knowledgeBaseId, documentId, updates) - } + const updateDocumentLocal = useCallback( + (documentId: string, updates: Partial) => { + updateDocument(knowledgeBaseId, documentId, updates) + }, + [knowledgeBaseId, updateDocument] + ) return { documents, + pagination, isLoading, error, refreshDocuments: refreshDocumentsData, diff --git a/apps/sim/stores/knowledge/store.ts b/apps/sim/stores/knowledge/store.ts index 2e8ca30608..0be8e4b8ea 100644 --- a/apps/sim/stores/knowledge/store.ts +++ b/apps/sim/stores/knowledge/store.ts @@ -88,10 +88,24 @@ export interface ChunksCache { lastFetchTime: number } +export interface DocumentsPagination { + total: number + limit: number + offset: number + hasMore: boolean +} + +export interface DocumentsCache { + documents: DocumentData[] + pagination: DocumentsPagination + searchQuery?: string + lastFetchTime: number +} + interface KnowledgeStore { // State knowledgeBases: Record - documents: Record // knowledgeBaseId -> documents + documents: Record // knowledgeBaseId -> documents cache chunks: Record // documentId -> chunks cache knowledgeBasesList: KnowledgeBaseData[] @@ -104,14 +118,20 @@ interface KnowledgeStore { // Actions getKnowledgeBase: (id: string) => Promise - getDocuments: (knowledgeBaseId: string) => Promise + getDocuments: ( + knowledgeBaseId: string, + options?: { search?: string; limit?: number; offset?: number } + ) => Promise getChunks: ( knowledgeBaseId: string, documentId: string, options?: { search?: string; limit?: number; offset?: number } ) => Promise getKnowledgeBasesList: () => Promise - refreshDocuments: (knowledgeBaseId: string) => Promise + refreshDocuments: ( + knowledgeBaseId: string, + options?: { search?: string; limit?: number; offset?: number } + ) => Promise refreshChunks: ( knowledgeBaseId: string, documentId: string, @@ -133,7 +153,7 @@ interface KnowledgeStore { // Getters getCachedKnowledgeBase: (id: string) => KnowledgeBaseData | null - getCachedDocuments: (knowledgeBaseId: string) => DocumentData[] | null + getCachedDocuments: (knowledgeBaseId: string) => DocumentsCache | null getCachedChunks: (documentId: string, options?: { search?: string }) => ChunksCache | null // Loading state getters @@ -235,18 +255,21 @@ export const useKnowledgeStore = create((set, get) => ({ } }, - getDocuments: async (knowledgeBaseId: string) => { + getDocuments: async ( + knowledgeBaseId: string, + options?: { search?: string; limit?: number; offset?: number } + ) => { const state = get() - // Return cached documents if they exist + // Return cached documents if they exist (no search-based caching since we do client-side filtering) const cached = state.documents[knowledgeBaseId] - if (cached) { - return cached + if (cached && cached.documents.length > 0) { + return cached.documents } // Return empty array if already loading to prevent duplicate requests if (state.loadingDocuments.has(knowledgeBaseId)) { - return [] + return cached?.documents || [] } try { @@ -254,7 +277,14 @@ export const useKnowledgeStore = create((set, get) => ({ loadingDocuments: new Set([...state.loadingDocuments, knowledgeBaseId]), })) - const response = await fetch(`/api/knowledge/${knowledgeBaseId}/documents`) + // Build query parameters + const params = new URLSearchParams() + if (options?.search) params.set('search', options.search) + if (options?.limit) params.set('limit', options.limit.toString()) + if (options?.offset) params.set('offset', options.offset.toString()) + + const url = `/api/knowledge/${knowledgeBaseId}/documents${params.toString() ? `?${params.toString()}` : ''}` + const response = await fetch(url) if (!response.ok) { throw new Error(`Failed to fetch documents: ${response.statusText}`) @@ -266,12 +296,25 @@ export const useKnowledgeStore = create((set, get) => ({ throw new Error(result.error || 'Failed to fetch documents') } - const documents = result.data + const documents = result.data.documents || result.data // Handle both paginated and non-paginated responses + const pagination = result.data.pagination || { + total: documents.length, + limit: options?.limit || 50, + offset: options?.offset || 0, + hasMore: false, + } + + const documentsCache: DocumentsCache = { + documents, + pagination, + searchQuery: options?.search, + lastFetchTime: Date.now(), + } set((state) => ({ documents: { ...state.documents, - [knowledgeBaseId]: documents, + [knowledgeBaseId]: documentsCache, }, loadingDocuments: new Set( [...state.loadingDocuments].filter((loadingId) => loadingId !== knowledgeBaseId) @@ -455,12 +498,15 @@ export const useKnowledgeStore = create((set, get) => ({ } }, - refreshDocuments: async (knowledgeBaseId: string) => { + refreshDocuments: async ( + knowledgeBaseId: string, + options?: { search?: string; limit?: number; offset?: number } + ) => { const state = get() // Return empty array if already loading to prevent duplicate requests if (state.loadingDocuments.has(knowledgeBaseId)) { - return state.documents[knowledgeBaseId] || [] + return state.documents[knowledgeBaseId]?.documents || [] } try { @@ -468,7 +514,14 @@ export const useKnowledgeStore = create((set, get) => ({ loadingDocuments: new Set([...state.loadingDocuments, knowledgeBaseId]), })) - const response = await fetch(`/api/knowledge/${knowledgeBaseId}/documents`) + // Build query parameters - for refresh, always start from offset 0 + const params = new URLSearchParams() + if (options?.search) params.set('search', options.search) + if (options?.limit) params.set('limit', options.limit.toString()) + params.set('offset', '0') // Always start fresh on refresh + + const url = `/api/knowledge/${knowledgeBaseId}/documents${params.toString() ? `?${params.toString()}` : ''}` + const response = await fetch(url) if (!response.ok) { throw new Error(`Failed to fetch documents: ${response.statusText}`) @@ -480,10 +533,16 @@ export const useKnowledgeStore = create((set, get) => ({ throw new Error(result.error || 'Failed to fetch documents') } - const serverDocuments = result.data + const serverDocuments = result.data.documents || result.data + const pagination = result.data.pagination || { + total: serverDocuments.length, + limit: options?.limit || 50, + offset: 0, + hasMore: false, + } set((state) => { - const currentDocuments = state.documents[knowledgeBaseId] || [] + const currentDocuments = state.documents[knowledgeBaseId]?.documents || [] // Create a map of server documents by filename for quick lookup const serverDocumentsByFilename = new Map() @@ -535,10 +594,17 @@ export const useKnowledgeStore = create((set, get) => ({ // Add any remaining temporary documents that don't have server equivalents const finalDocuments = [...mergedDocuments, ...filteredCurrentDocs] + const documentsCache: DocumentsCache = { + documents: finalDocuments, + pagination, + searchQuery: options?.search, + lastFetchTime: Date.now(), + } + return { documents: { ...state.documents, - [knowledgeBaseId]: finalDocuments, + [knowledgeBaseId]: documentsCache, }, loadingDocuments: new Set( [...state.loadingDocuments].filter((loadingId) => loadingId !== knowledgeBaseId) @@ -638,17 +704,20 @@ export const useKnowledgeStore = create((set, get) => ({ updateDocument: (knowledgeBaseId: string, documentId: string, updates: Partial) => { set((state) => { - const documents = state.documents[knowledgeBaseId] - if (!documents) return state + const documentsCache = state.documents[knowledgeBaseId] + if (!documentsCache) return state - const updatedDocuments = documents.map((doc) => + const updatedDocuments = documentsCache.documents.map((doc) => doc.id === documentId ? { ...doc, ...updates } : doc ) return { documents: { ...state.documents, - [knowledgeBaseId]: updatedDocuments, + [knowledgeBaseId]: { + ...documentsCache, + documents: updatedDocuments, + }, }, } }) @@ -677,7 +746,8 @@ export const useKnowledgeStore = create((set, get) => ({ addPendingDocuments: (knowledgeBaseId: string, newDocuments: DocumentData[]) => { set((state) => { - const existingDocuments = state.documents[knowledgeBaseId] || [] + const existingDocumentsCache = state.documents[knowledgeBaseId] + const existingDocuments = existingDocumentsCache?.documents || [] const existingIds = new Set(existingDocuments.map((doc) => doc.id)) const uniqueNewDocuments = newDocuments.filter((doc) => !existingIds.has(doc.id)) @@ -689,15 +759,29 @@ export const useKnowledgeStore = create((set, get) => ({ const updatedDocuments = [...existingDocuments, ...uniqueNewDocuments] + const documentsCache: DocumentsCache = { + documents: updatedDocuments, + pagination: { + ...(existingDocumentsCache?.pagination || { + limit: 50, + offset: 0, + hasMore: false, + }), + total: updatedDocuments.length, + }, + searchQuery: existingDocumentsCache?.searchQuery, + lastFetchTime: Date.now(), + } + return { documents: { ...state.documents, - [knowledgeBaseId]: updatedDocuments, + [knowledgeBaseId]: documentsCache, }, } }) logger.info( - `Added ${newDocuments.filter((doc) => !get().documents[knowledgeBaseId]?.some((existing) => existing.id === doc.id)).length} pending documents for knowledge base: ${knowledgeBaseId}` + `Added ${newDocuments.filter((doc) => !get().documents[knowledgeBaseId]?.documents?.some((existing) => existing.id === doc.id)).length} pending documents for knowledge base: ${knowledgeBaseId}` ) }, @@ -731,10 +815,10 @@ export const useKnowledgeStore = create((set, get) => ({ removeDocument: (knowledgeBaseId: string, documentId: string) => { set((state) => { - const documents = state.documents[knowledgeBaseId] - if (!documents) return state + const documentsCache = state.documents[knowledgeBaseId] + if (!documentsCache) return state - const updatedDocuments = documents.filter((doc) => doc.id !== documentId) + const updatedDocuments = documentsCache.documents.filter((doc) => doc.id !== documentId) // Also clear chunks for the removed document const newChunks = { ...state.chunks } @@ -743,7 +827,10 @@ export const useKnowledgeStore = create((set, get) => ({ return { documents: { ...state.documents, - [knowledgeBaseId]: updatedDocuments, + [knowledgeBaseId]: { + ...documentsCache, + documents: updatedDocuments, + }, }, chunks: newChunks, }