improvement(kb): pagination for docs + remove kb token count problematic query (#689)

* added pagination to docs in kb * ack PR comments * fix failing tests * fix(kb): remove problematic query updating kb token count --------- Co-authored-by: Waleed Latif <walif6@gmail.com>
2026-01-09 23:17:59 -05:00 · 2025-07-15 00:28:07 -07:00
parent a7a2056b5f
commit bb759368d9
10 changed files with 396 additions and 117 deletions
--- a/apps/sim/app/api/knowledge/[id]/documents/route.test.ts
+++ b/apps/sim/app/api/knowledge/[id]/documents/route.test.ts
@@ -30,6 +30,8 @@ describe('Knowledge Base Documents API Route', () => {
    from: vi.fn().mockReturnThis(),
    where: vi.fn().mockReturnThis(),
    orderBy: vi.fn().mockReturnThis(),
+    limit: vi.fn().mockReturnThis(),
+    offset: vi.fn().mockReturnThis(),
    insert: vi.fn().mockReturnThis(),
    values: vi.fn().mockReturnThis(),
    update: vi.fn().mockReturnThis(),
@@ -99,7 +101,12 @@ describe('Knowledge Base Documents API Route', () => {
    it('should retrieve documents successfully for authenticated user', async () => {
      mockAuth$.mockAuthenticatedUser()
      mockCheckKnowledgeBaseAccess.mockResolvedValue({ hasAccess: true })
-      mockDbChain.orderBy.mockResolvedValue([mockDocument])
+
+      // Mock the count query (first query)
+      mockDbChain.where.mockResolvedValueOnce([{ count: 1 }])
+
+      // Mock the documents query (second query)
+      mockDbChain.offset.mockResolvedValue([mockDocument])

      const req = createMockRequest('GET')
      const { GET } = await import('./route')
@@ -108,8 +115,8 @@ describe('Knowledge Base Documents API Route', () => {

      expect(response.status).toBe(200)
      expect(data.success).toBe(true)
-      expect(data.data).toHaveLength(1)
-      expect(data.data[0].id).toBe('doc-123')
+      expect(data.data.documents).toHaveLength(1)
+      expect(data.data.documents[0].id).toBe('doc-123')
      expect(mockDbChain.select).toHaveBeenCalled()
      expect(mockCheckKnowledgeBaseAccess).toHaveBeenCalledWith('kb-123', 'user-123')
    })
@@ -117,7 +124,12 @@ describe('Knowledge Base Documents API Route', () => {
    it('should filter disabled documents by default', async () => {
      mockAuth$.mockAuthenticatedUser()
      mockCheckKnowledgeBaseAccess.mockResolvedValue({ hasAccess: true })
-      mockDbChain.orderBy.mockResolvedValue([mockDocument])
+
+      // Mock the count query (first query)
+      mockDbChain.where.mockResolvedValueOnce([{ count: 1 }])
+
+      // Mock the documents query (second query)
+      mockDbChain.offset.mockResolvedValue([mockDocument])

      const req = createMockRequest('GET')
      const { GET } = await import('./route')
@@ -130,7 +142,12 @@ describe('Knowledge Base Documents API Route', () => {
    it('should include disabled documents when requested', async () => {
      mockAuth$.mockAuthenticatedUser()
      mockCheckKnowledgeBaseAccess.mockResolvedValue({ hasAccess: true })
-      mockDbChain.orderBy.mockResolvedValue([mockDocument])
+
+      // Mock the count query (first query)
+      mockDbChain.where.mockResolvedValueOnce([{ count: 1 }])
+
+      // Mock the documents query (second query)
+      mockDbChain.offset.mockResolvedValue([mockDocument])

      const url = 'http://localhost:3000/api/knowledge/kb-123/documents?includeDisabled=true'
      const req = new Request(url, { method: 'GET' }) as any
--- a/apps/sim/app/api/knowledge/[id]/documents/route.ts
+++ b/apps/sim/app/api/knowledge/[id]/documents/route.ts
@@ -1,5 +1,5 @@
 import crypto from 'node:crypto'
-import { and, desc, eq, inArray, isNull } from 'drizzle-orm'
+import { and, desc, eq, inArray, isNull, sql } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { z } from 'zod'
 import { getSession } from '@/lib/auth'
@@ -210,6 +210,9 @@ export async function GET(req: NextRequest, { params }: { params: Promise<{ id:

    const url = new URL(req.url)
    const includeDisabled = url.searchParams.get('includeDisabled') === 'true'
+    const search = url.searchParams.get('search')
+    const limit = Number.parseInt(url.searchParams.get('limit') || '50')
+    const offset = Number.parseInt(url.searchParams.get('offset') || '0')

    // Build where conditions
    const whereConditions = [
@@ -222,6 +225,23 @@ export async function GET(req: NextRequest, { params }: { params: Promise<{ id:
      whereConditions.push(eq(document.enabled, true))
    }

+    // Add search condition if provided
+    if (search) {
+      whereConditions.push(
+        // Search in filename
+        sql`LOWER(${document.filename}) LIKE LOWER(${`%${search}%`})`
+      )
+    }
+
+    // Get total count for pagination
+    const totalResult = await db
+      .select({ count: sql<number>`COUNT(*)` })
+      .from(document)
+      .where(and(...whereConditions))
+
+    const total = totalResult[0]?.count || 0
+    const hasMore = offset + limit < total
+
    const documents = await db
      .select({
        id: document.id,
@@ -250,14 +270,24 @@ export async function GET(req: NextRequest, { params }: { params: Promise<{ id:
      .from(document)
      .where(and(...whereConditions))
      .orderBy(desc(document.uploadedAt))
+      .limit(limit)
+      .offset(offset)

    logger.info(
-      `[${requestId}] Retrieved ${documents.length} documents for knowledge base ${knowledgeBaseId}`
+      `[${requestId}] Retrieved ${documents.length} documents (${offset}-${offset + documents.length} of ${total}) for knowledge base ${knowledgeBaseId}`
    )

    return NextResponse.json({
      success: true,
-      data: documents,
+      data: {
+        documents,
+        pagination: {
+          total,
+          limit,
+          offset,
+          hasMore,
+        },
+      },
    })
  } catch (error) {
    logger.error(`[${requestId}] Error fetching documents`, error)
--- a/apps/sim/app/api/knowledge/utils.test.ts
+++ b/apps/sim/app/api/knowledge/utils.test.ts
@@ -176,7 +176,7 @@ describe('Knowledge Utils', () => {
        {}
      )

-      expect(dbOps.order).toEqual(['insert', 'updateDoc', 'updateKb'])
+      expect(dbOps.order).toEqual(['insert', 'updateDoc'])

      expect(dbOps.updatePayloads[0]).toMatchObject({
        processingStatus: 'completed',
--- a/apps/sim/app/api/knowledge/utils.ts
+++ b/apps/sim/app/api/knowledge/utils.ts
@@ -1,5 +1,5 @@
 import crypto from 'crypto'
-import { and, eq, isNull, sql } from 'drizzle-orm'
+import { and, eq, isNull } from 'drizzle-orm'
 import { processDocument } from '@/lib/documents/document-processor'
 import { retryWithExponentialBackoff } from '@/lib/documents/utils'
 import { env } from '@/lib/env'
@@ -522,14 +522,6 @@ export async function processDocumentAsync(
              processingError: null,
            })
            .where(eq(document.id, documentId))
-
-          await tx
-            .update(knowledgeBase)
-            .set({
-              tokenCount: sql`${knowledgeBase.tokenCount} + ${processed.metadata.tokenCount}`,
-              updatedAt: now,
-            })
-            .where(eq(knowledgeBase.id, knowledgeBaseId))
        })
      })(),
      TIMEOUTS.OVERALL_PROCESSING,
--- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx
+++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx
@@ -117,7 +117,7 @@ export function Document({
        setError(null)

        const cachedDocuments = getCachedDocuments(knowledgeBaseId)
-        const cachedDoc = cachedDocuments?.find((d) => d.id === documentId)
+        const cachedDoc = cachedDocuments?.documents?.find((d) => d.id === documentId)

        if (cachedDoc) {
          setDocument(cachedDoc)
--- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx
+++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx
@@ -1,9 +1,11 @@
 'use client'

-import { useEffect, useState } from 'react'
+import { useCallback, useEffect, useState } from 'react'
 import { format } from 'date-fns'
 import {
  AlertCircle,
+  ChevronLeft,
+  ChevronRight,
  Circle,
  CircleOff,
  FileText,
@@ -25,6 +27,7 @@ import {
 } from '@/components/ui/alert-dialog'
 import { Button } from '@/components/ui/button'
 import { Checkbox } from '@/components/ui/checkbox'
+import { SearchHighlight } from '@/components/ui/search-highlight'
 import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip'
 import { createLogger } from '@/lib/logs/console-logger'
 import { ActionBar } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/action-bar/action-bar'
@@ -40,6 +43,9 @@ import { UploadModal } from './components/upload-modal/upload-modal'

 const logger = createLogger('KnowledgeBase')

+// Constants
+const DOCUMENTS_PER_PAGE = 50
+
 interface KnowledgeBaseProps {
  id: string
  knowledgeBaseName?: string
@@ -118,6 +124,22 @@ export function KnowledgeBase({
  const { removeKnowledgeBase } = useKnowledgeStore()
  const params = useParams()
  const workspaceId = params.workspaceId as string
+
+  const [searchQuery, setSearchQuery] = useState('')
+
+  // Memoize the search query setter to prevent unnecessary re-renders
+  const handleSearchChange = useCallback((newQuery: string) => {
+    setSearchQuery(newQuery)
+    setCurrentPage(1) // Reset to page 1 when searching
+  }, [])
+
+  const [selectedDocuments, setSelectedDocuments] = useState<Set<string>>(new Set())
+  const [showDeleteDialog, setShowDeleteDialog] = useState(false)
+  const [showUploadModal, setShowUploadModal] = useState(false)
+  const [isDeleting, setIsDeleting] = useState(false)
+  const [isBulkOperating, setIsBulkOperating] = useState(false)
+  const [currentPage, setCurrentPage] = useState(1)
+
  const {
    knowledgeBase,
    isLoading: isLoadingKnowledgeBase,
@@ -125,27 +147,52 @@ export function KnowledgeBase({
  } = useKnowledgeBase(id)
  const {
    documents,
+    pagination,
    isLoading: isLoadingDocuments,
    error: documentsError,
    updateDocument,
    refreshDocuments,
-  } = useKnowledgeBaseDocuments(id)
+  } = useKnowledgeBaseDocuments(id, {
+    search: searchQuery || undefined,
+    limit: DOCUMENTS_PER_PAGE,
+    offset: (currentPage - 1) * DOCUMENTS_PER_PAGE,
+  })

  const isSidebarCollapsed =
    mode === 'expanded' ? !isExpanded : mode === 'collapsed' || mode === 'hover'

-  const [searchQuery, setSearchQuery] = useState('')
-  const [selectedDocuments, setSelectedDocuments] = useState<Set<string>>(new Set())
-  const [showDeleteDialog, setShowDeleteDialog] = useState(false)
-  const [showUploadModal, setShowUploadModal] = useState(false)
-  const [isDeleting, setIsDeleting] = useState(false)
-  const [isBulkOperating, setIsBulkOperating] = useState(false)
-
  const router = useRouter()

  const knowledgeBaseName = knowledgeBase?.name || passedKnowledgeBaseName || 'Knowledge Base'
  const error = knowledgeBaseError || documentsError

+  // Pagination calculations
+  const totalPages = Math.ceil(pagination.total / pagination.limit)
+  const hasNextPage = currentPage < totalPages
+  const hasPrevPage = currentPage > 1
+
+  // Navigation functions
+  const goToPage = useCallback(
+    (page: number) => {
+      if (page >= 1 && page <= totalPages) {
+        setCurrentPage(page)
+      }
+    },
+    [totalPages]
+  )
+
+  const nextPage = useCallback(() => {
+    if (hasNextPage) {
+      setCurrentPage((prev) => prev + 1)
+    }
+  }, [hasNextPage])
+
+  const prevPage = useCallback(() => {
+    if (hasPrevPage) {
+      setCurrentPage((prev) => prev - 1)
+    }
+  }, [hasPrevPage])
+
  // Auto-refresh documents when there are processing documents
  useEffect(() => {
    const hasProcessingDocuments = documents.some(
@@ -220,10 +267,8 @@ export function KnowledgeBase({
    await Promise.allSettled(markFailedPromises)
  }

-  // Filter documents based on search query
-  const filteredDocuments = documents.filter((doc) =>
-    doc.filename.toLowerCase().includes(searchQuery.toLowerCase())
-  )
+  // Calculate pagination info for display
+  const totalItems = pagination?.total || 0

  const handleToggleEnabled = async (docId: string) => {
    const document = documents.find((doc) => doc.id === docId)
@@ -366,14 +411,13 @@ export function KnowledgeBase({

  const handleSelectAll = (checked: boolean) => {
    if (checked) {
-      setSelectedDocuments(new Set(filteredDocuments.map((doc) => doc.id)))
+      setSelectedDocuments(new Set(documents.map((doc) => doc.id)))
    } else {
      setSelectedDocuments(new Set())
    }
  }

-  const isAllSelected =
-    filteredDocuments.length > 0 && selectedDocuments.size === filteredDocuments.length
+  const isAllSelected = documents.length > 0 && selectedDocuments.size === documents.length

  const handleDocumentClick = (docId: string) => {
    // Find the document to get its filename
@@ -621,20 +665,35 @@ export function KnowledgeBase({
          {/* Main Content */}
          <div className='flex-1 overflow-auto'>
            <div className='px-6 pb-6'>
-              {/* Search and Create Section */}
-              <div className='mb-4 flex items-center justify-between pt-1'>
-                <SearchInput
-                  value={searchQuery}
-                  onChange={setSearchQuery}
-                  placeholder='Search documents...'
-                />
+              {/* Search and Filters Section */}
+              <div className='mb-4 space-y-3 pt-1'>
+                <div className='flex items-center justify-between'>
+                  <SearchInput
+                    value={searchQuery}
+                    onChange={handleSearchChange}
+                    placeholder='Search documents...'
+                  />

-                <div className='flex items-center gap-3'>
-                  {/* Add Documents Button */}
-                  <PrimaryButton onClick={handleAddDocuments}>
-                    <Plus className='h-3.5 w-3.5' />
-                    Add Documents
-                  </PrimaryButton>
+                  <div className='flex items-center gap-3'>
+                    {/* Clear Search Button */}
+                    {searchQuery && (
+                      <button
+                        onClick={() => {
+                          setSearchQuery('')
+                          setCurrentPage(1)
+                        }}
+                        className='text-muted-foreground text-sm hover:text-foreground'
+                      >
+                        Clear search
+                      </button>
+                    )}
+
+                    {/* Add Documents Button */}
+                    <PrimaryButton onClick={handleAddDocuments}>
+                      <Plus className='h-3.5 w-3.5' />
+                      Add Documents
+                    </PrimaryButton>
+                  </div>
                </div>
              </div>

@@ -714,7 +773,7 @@ export function KnowledgeBase({
                      <col className='w-[14%]' />
                    </colgroup>
                    <tbody>
-                      {filteredDocuments.length === 0 && !isLoadingDocuments ? (
+                      {documents.length === 0 && !isLoadingDocuments ? (
                        <tr className='border-b transition-colors hover:bg-accent/30'>
                          {/* Select column */}
                          <td className='px-4 py-3'>
@@ -726,7 +785,7 @@ export function KnowledgeBase({
                            <div className='flex items-center gap-2'>
                              <FileText className='h-6 w-5 text-muted-foreground' />
                              <span className='text-muted-foreground text-sm italic'>
-                                {documents.length === 0
+                                {totalItems === 0
                                  ? 'No documents yet'
                                  : 'No documents match your search'}
                              </span>
@@ -793,7 +852,7 @@ export function KnowledgeBase({
                          </tr>
                        ))
                      ) : (
-                        filteredDocuments.map((doc) => {
+                        documents.map((doc) => {
                          const isSelected = selectedDocuments.has(doc.id)
                          const statusDisplay = getStatusDisplay(doc)
                          // const processingTime = getProcessingTime(doc)
@@ -834,7 +893,10 @@ export function KnowledgeBase({
                                  <Tooltip>
                                    <TooltipTrigger asChild>
                                      <span className='block truncate text-sm' title={doc.filename}>
-                                        {doc.filename}
+                                        <SearchHighlight
+                                          text={doc.filename}
+                                          searchQuery={searchQuery}
+                                        />
                                      </span>
                                    </TooltipTrigger>
                                    <TooltipContent side='top'>{doc.filename}</TooltipContent>
@@ -998,6 +1060,64 @@ export function KnowledgeBase({
                    </tbody>
                  </table>
                </div>
+
+                {/* Pagination Controls */}
+                {totalPages > 1 && (
+                  <div className='flex items-center justify-center border-t bg-background px-6 py-4'>
+                    <div className='flex items-center gap-1'>
+                      <Button
+                        variant='ghost'
+                        size='sm'
+                        onClick={prevPage}
+                        disabled={!hasPrevPage || isLoadingDocuments}
+                        className='h-8 w-8 p-0'
+                      >
+                        <ChevronLeft className='h-4 w-4' />
+                      </Button>
+
+                      {/* Page numbers - show a few around current page */}
+                      <div className='mx-4 flex items-center gap-6'>
+                        {Array.from({ length: Math.min(totalPages, 5) }, (_, i) => {
+                          let page: number
+                          if (totalPages <= 5) {
+                            page = i + 1
+                          } else if (currentPage <= 3) {
+                            page = i + 1
+                          } else if (currentPage >= totalPages - 2) {
+                            page = totalPages - 4 + i
+                          } else {
+                            page = currentPage - 2 + i
+                          }
+
+                          if (page < 1 || page > totalPages) return null
+
+                          return (
+                            <button
+                              key={page}
+                              onClick={() => goToPage(page)}
+                              disabled={isLoadingDocuments}
+                              className={`font-medium text-sm transition-colors hover:text-foreground disabled:cursor-not-allowed disabled:opacity-50 ${
+                                page === currentPage ? 'text-foreground' : 'text-muted-foreground'
+                              }`}
+                            >
+                              {page}
+                            </button>
+                          )
+                        })}
+                      </div>
+
+                      <Button
+                        variant='ghost'
+                        size='sm'
+                        onClick={nextPage}
+                        disabled={!hasNextPage || isLoadingDocuments}
+                        className='h-8 w-8 p-0'
+                      >
+                        <ChevronRight className='h-4 w-4' />
+                      </Button>
+                    </div>
+                  </div>
+                )}
              </div>
            </div>
          </div>
@@ -1011,8 +1131,8 @@ export function KnowledgeBase({
            <AlertDialogTitle>Delete Knowledge Base</AlertDialogTitle>
            <AlertDialogDescription>
              Are you sure you want to delete "{knowledgeBaseName}"? This will permanently delete
-              the knowledge base and all {documents.length} document
-              {documents.length === 1 ? '' : 's'} within it. This action cannot be undone.
+              the knowledge base and all {totalItems} document
+              {totalItems === 1 ? '' : 's'} within it. This action cannot be undone.
            </AlertDialogDescription>
          </AlertDialogHeader>
          <AlertDialogFooter>
--- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/components/document-selector/document-selector.tsx
+++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/components/document-selector/document-selector.tsx
@@ -87,7 +87,7 @@ export function DocumentSelector({
        throw new Error(result.error || 'Failed to fetch documents')
      }

-      const fetchedDocuments = result.data || []
+      const fetchedDocuments = result.data.documents || result.data || []
      setDocuments(fetchedDocuments)
    } catch (err) {
      if ((err as Error).name === 'AbortError') return
--- a/apps/sim/components/ui/search-highlight.tsx
+++ b/apps/sim/components/ui/search-highlight.tsx
@@ -1,7 +1,5 @@
 'use client'

-import { Fragment } from 'react'
-
 interface SearchHighlightProps {
  text: string
  searchQuery: string
@@ -13,22 +11,18 @@ export function SearchHighlight({ text, searchQuery, className = '' }: SearchHig
    return <span className={className}>{text}</span>
  }

-  // Create a regex to find matches (case-insensitive)
-  // Handle multiple search terms separated by spaces
+  // Create regex pattern for all search terms
  const searchTerms = searchQuery
    .trim()
    .split(/\s+/)
    .filter((term) => term.length > 0)
+    .map((term) => term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))

  if (searchTerms.length === 0) {
    return <span className={className}>{text}</span>
  }

-  // Create regex pattern for all search terms
-  const escapedTerms = searchTerms.map((term) => term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
-  const regexPattern = `(${escapedTerms.join('|')})`
-  const regex = new RegExp(regexPattern, 'gi')
-
+  const regex = new RegExp(`(${searchTerms.join('|')})`, 'gi')
  const parts = text.split(regex)

  return (
@@ -36,18 +30,17 @@ export function SearchHighlight({ text, searchQuery, className = '' }: SearchHig
      {parts.map((part, index) => {
        if (!part) return null

-        const isMatch = regex.test(part)
+        const isMatch = searchTerms.some((term) => new RegExp(term, 'gi').test(part))

-        return (
-          <Fragment key={index}>
-            {isMatch ? (
-              <span className='rounded-sm bg-yellow-200 px-0.5 py-0.5 font-medium text-yellow-900 dark:bg-yellow-900/50 dark:text-yellow-200'>
-                {part}
-              </span>
-            ) : (
-              part
-            )}
-          </Fragment>
+        return isMatch ? (
+          <span
+            key={index}
+            className='bg-yellow-200 text-yellow-900 dark:bg-yellow-900/50 dark:text-yellow-200'
+          >
+            {part}
+          </span>
+        ) : (
+          <span key={index}>{part}</span>
        )
      })}
    </span>
--- a/apps/sim/hooks/use-knowledge.ts
+++ b/apps/sim/hooks/use-knowledge.ts
@@ -40,24 +40,33 @@ export function useKnowledgeBase(id: string) {
  }
 }

-export function useKnowledgeBaseDocuments(knowledgeBaseId: string) {
+// Constants
+const MAX_DOCUMENTS_LIMIT = 10000
+const DEFAULT_PAGE_SIZE = 50
+
+export function useKnowledgeBaseDocuments(
+  knowledgeBaseId: string,
+  options?: { search?: string; limit?: number; offset?: number }
+) {
  const { getDocuments, getCachedDocuments, loadingDocuments, updateDocument, refreshDocuments } =
    useKnowledgeStore()

  const [error, setError] = useState<string | null>(null)

-  const documents = getCachedDocuments(knowledgeBaseId) || []
+  const documentsCache = getCachedDocuments(knowledgeBaseId)
+  const allDocuments = documentsCache?.documents || []
  const isLoading = loadingDocuments.has(knowledgeBaseId)

+  // Load all documents on initial mount
  useEffect(() => {
-    if (!knowledgeBaseId || documents.length > 0 || isLoading) return
+    if (!knowledgeBaseId || allDocuments.length > 0 || isLoading) return

    let isMounted = true

-    const loadData = async () => {
+    const loadAllDocuments = async () => {
      try {
        setError(null)
-        await getDocuments(knowledgeBaseId)
+        await getDocuments(knowledgeBaseId, { limit: MAX_DOCUMENTS_LIMIT })
      } catch (err) {
        if (isMounted) {
          setError(err instanceof Error ? err.message : 'Failed to load documents')
@@ -65,28 +74,59 @@ export function useKnowledgeBaseDocuments(knowledgeBaseId: string) {
      }
    }

-    loadData()
+    loadAllDocuments()

    return () => {
      isMounted = false
    }
-  }, [knowledgeBaseId, documents.length, isLoading]) // Removed getDocuments from dependencies
+  }, [knowledgeBaseId, allDocuments.length, isLoading, getDocuments])

-  const refreshDocumentsData = async () => {
+  // Client-side filtering and pagination
+  const { documents, pagination } = useMemo(() => {
+    let filteredDocs = allDocuments
+
+    // Apply search filter
+    if (options?.search) {
+      const searchLower = options.search.toLowerCase()
+      filteredDocs = filteredDocs.filter((doc) => doc.filename.toLowerCase().includes(searchLower))
+    }
+
+    // Apply pagination
+    const offset = options?.offset || 0
+    const limit = options?.limit || DEFAULT_PAGE_SIZE
+    const total = filteredDocs.length
+    const paginatedDocs = filteredDocs.slice(offset, offset + limit)
+
+    return {
+      documents: paginatedDocs,
+      pagination: {
+        total,
+        limit,
+        offset,
+        hasMore: offset + limit < total,
+      },
+    }
+  }, [allDocuments, options?.search, options?.limit, options?.offset])
+
+  const refreshDocumentsData = useCallback(async () => {
    try {
      setError(null)
-      await refreshDocuments(knowledgeBaseId)
+      await refreshDocuments(knowledgeBaseId, { limit: MAX_DOCUMENTS_LIMIT })
    } catch (err) {
      setError(err instanceof Error ? err.message : 'Failed to refresh documents')
    }
-  }
+  }, [knowledgeBaseId, refreshDocuments])

-  const updateDocumentLocal = (documentId: string, updates: Partial<DocumentData>) => {
-    updateDocument(knowledgeBaseId, documentId, updates)
-  }
+  const updateDocumentLocal = useCallback(
+    (documentId: string, updates: Partial<DocumentData>) => {
+      updateDocument(knowledgeBaseId, documentId, updates)
+    },
+    [knowledgeBaseId, updateDocument]
+  )

  return {
    documents,
+    pagination,
    isLoading,
    error,
    refreshDocuments: refreshDocumentsData,
--- a/apps/sim/stores/knowledge/store.ts
+++ b/apps/sim/stores/knowledge/store.ts
@@ -88,10 +88,24 @@ export interface ChunksCache {
  lastFetchTime: number
 }

+export interface DocumentsPagination {
+  total: number
+  limit: number
+  offset: number
+  hasMore: boolean
+}
+
+export interface DocumentsCache {
+  documents: DocumentData[]
+  pagination: DocumentsPagination
+  searchQuery?: string
+  lastFetchTime: number
+}
+
 interface KnowledgeStore {
  // State
  knowledgeBases: Record<string, KnowledgeBaseData>
-  documents: Record<string, DocumentData[]> // knowledgeBaseId -> documents
+  documents: Record<string, DocumentsCache> // knowledgeBaseId -> documents cache
  chunks: Record<string, ChunksCache> // documentId -> chunks cache
  knowledgeBasesList: KnowledgeBaseData[]

@@ -104,14 +118,20 @@ interface KnowledgeStore {

  // Actions
  getKnowledgeBase: (id: string) => Promise<KnowledgeBaseData | null>
-  getDocuments: (knowledgeBaseId: string) => Promise<DocumentData[]>
+  getDocuments: (
+    knowledgeBaseId: string,
+    options?: { search?: string; limit?: number; offset?: number }
+  ) => Promise<DocumentData[]>
  getChunks: (
    knowledgeBaseId: string,
    documentId: string,
    options?: { search?: string; limit?: number; offset?: number }
  ) => Promise<ChunkData[]>
  getKnowledgeBasesList: () => Promise<KnowledgeBaseData[]>
-  refreshDocuments: (knowledgeBaseId: string) => Promise<DocumentData[]>
+  refreshDocuments: (
+    knowledgeBaseId: string,
+    options?: { search?: string; limit?: number; offset?: number }
+  ) => Promise<DocumentData[]>
  refreshChunks: (
    knowledgeBaseId: string,
    documentId: string,
@@ -133,7 +153,7 @@ interface KnowledgeStore {

  // Getters
  getCachedKnowledgeBase: (id: string) => KnowledgeBaseData | null
-  getCachedDocuments: (knowledgeBaseId: string) => DocumentData[] | null
+  getCachedDocuments: (knowledgeBaseId: string) => DocumentsCache | null
  getCachedChunks: (documentId: string, options?: { search?: string }) => ChunksCache | null

  // Loading state getters
@@ -235,18 +255,21 @@ export const useKnowledgeStore = create<KnowledgeStore>((set, get) => ({
    }
  },

-  getDocuments: async (knowledgeBaseId: string) => {
+  getDocuments: async (
+    knowledgeBaseId: string,
+    options?: { search?: string; limit?: number; offset?: number }
+  ) => {
    const state = get()

-    // Return cached documents if they exist
+    // Return cached documents if they exist (no search-based caching since we do client-side filtering)
    const cached = state.documents[knowledgeBaseId]
-    if (cached) {
-      return cached
+    if (cached && cached.documents.length > 0) {
+      return cached.documents
    }

    // Return empty array if already loading to prevent duplicate requests
    if (state.loadingDocuments.has(knowledgeBaseId)) {
-      return []
+      return cached?.documents || []
    }

    try {
@@ -254,7 +277,14 @@ export const useKnowledgeStore = create<KnowledgeStore>((set, get) => ({
        loadingDocuments: new Set([...state.loadingDocuments, knowledgeBaseId]),
      }))

-      const response = await fetch(`/api/knowledge/${knowledgeBaseId}/documents`)
+      // Build query parameters
+      const params = new URLSearchParams()
+      if (options?.search) params.set('search', options.search)
+      if (options?.limit) params.set('limit', options.limit.toString())
+      if (options?.offset) params.set('offset', options.offset.toString())
+
+      const url = `/api/knowledge/${knowledgeBaseId}/documents${params.toString() ? `?${params.toString()}` : ''}`
+      const response = await fetch(url)

      if (!response.ok) {
        throw new Error(`Failed to fetch documents: ${response.statusText}`)
@@ -266,12 +296,25 @@ export const useKnowledgeStore = create<KnowledgeStore>((set, get) => ({
        throw new Error(result.error || 'Failed to fetch documents')
      }

-      const documents = result.data
+      const documents = result.data.documents || result.data // Handle both paginated and non-paginated responses
+      const pagination = result.data.pagination || {
+        total: documents.length,
+        limit: options?.limit || 50,
+        offset: options?.offset || 0,
+        hasMore: false,
+      }
+
+      const documentsCache: DocumentsCache = {
+        documents,
+        pagination,
+        searchQuery: options?.search,
+        lastFetchTime: Date.now(),
+      }

      set((state) => ({
        documents: {
          ...state.documents,
-          [knowledgeBaseId]: documents,
+          [knowledgeBaseId]: documentsCache,
        },
        loadingDocuments: new Set(
          [...state.loadingDocuments].filter((loadingId) => loadingId !== knowledgeBaseId)
@@ -455,12 +498,15 @@ export const useKnowledgeStore = create<KnowledgeStore>((set, get) => ({
    }
  },

-  refreshDocuments: async (knowledgeBaseId: string) => {
+  refreshDocuments: async (
+    knowledgeBaseId: string,
+    options?: { search?: string; limit?: number; offset?: number }
+  ) => {
    const state = get()

    // Return empty array if already loading to prevent duplicate requests
    if (state.loadingDocuments.has(knowledgeBaseId)) {
-      return state.documents[knowledgeBaseId] || []
+      return state.documents[knowledgeBaseId]?.documents || []
    }

    try {
@@ -468,7 +514,14 @@ export const useKnowledgeStore = create<KnowledgeStore>((set, get) => ({
        loadingDocuments: new Set([...state.loadingDocuments, knowledgeBaseId]),
      }))

-      const response = await fetch(`/api/knowledge/${knowledgeBaseId}/documents`)
+      // Build query parameters - for refresh, always start from offset 0
+      const params = new URLSearchParams()
+      if (options?.search) params.set('search', options.search)
+      if (options?.limit) params.set('limit', options.limit.toString())
+      params.set('offset', '0') // Always start fresh on refresh
+
+      const url = `/api/knowledge/${knowledgeBaseId}/documents${params.toString() ? `?${params.toString()}` : ''}`
+      const response = await fetch(url)

      if (!response.ok) {
        throw new Error(`Failed to fetch documents: ${response.statusText}`)
@@ -480,10 +533,16 @@ export const useKnowledgeStore = create<KnowledgeStore>((set, get) => ({
        throw new Error(result.error || 'Failed to fetch documents')
      }

-      const serverDocuments = result.data
+      const serverDocuments = result.data.documents || result.data
+      const pagination = result.data.pagination || {
+        total: serverDocuments.length,
+        limit: options?.limit || 50,
+        offset: 0,
+        hasMore: false,
+      }

      set((state) => {
-        const currentDocuments = state.documents[knowledgeBaseId] || []
+        const currentDocuments = state.documents[knowledgeBaseId]?.documents || []

        // Create a map of server documents by filename for quick lookup
        const serverDocumentsByFilename = new Map()
@@ -535,10 +594,17 @@ export const useKnowledgeStore = create<KnowledgeStore>((set, get) => ({
        // Add any remaining temporary documents that don't have server equivalents
        const finalDocuments = [...mergedDocuments, ...filteredCurrentDocs]

+        const documentsCache: DocumentsCache = {
+          documents: finalDocuments,
+          pagination,
+          searchQuery: options?.search,
+          lastFetchTime: Date.now(),
+        }
+
        return {
          documents: {
            ...state.documents,
-            [knowledgeBaseId]: finalDocuments,
+            [knowledgeBaseId]: documentsCache,
          },
          loadingDocuments: new Set(
            [...state.loadingDocuments].filter((loadingId) => loadingId !== knowledgeBaseId)
@@ -638,17 +704,20 @@ export const useKnowledgeStore = create<KnowledgeStore>((set, get) => ({

  updateDocument: (knowledgeBaseId: string, documentId: string, updates: Partial<DocumentData>) => {
    set((state) => {
-      const documents = state.documents[knowledgeBaseId]
-      if (!documents) return state
+      const documentsCache = state.documents[knowledgeBaseId]
+      if (!documentsCache) return state

-      const updatedDocuments = documents.map((doc) =>
+      const updatedDocuments = documentsCache.documents.map((doc) =>
        doc.id === documentId ? { ...doc, ...updates } : doc
      )

      return {
        documents: {
          ...state.documents,
-          [knowledgeBaseId]: updatedDocuments,
+          [knowledgeBaseId]: {
+            ...documentsCache,
+            documents: updatedDocuments,
+          },
        },
      }
    })
@@ -677,7 +746,8 @@ export const useKnowledgeStore = create<KnowledgeStore>((set, get) => ({

  addPendingDocuments: (knowledgeBaseId: string, newDocuments: DocumentData[]) => {
    set((state) => {
-      const existingDocuments = state.documents[knowledgeBaseId] || []
+      const existingDocumentsCache = state.documents[knowledgeBaseId]
+      const existingDocuments = existingDocumentsCache?.documents || []

      const existingIds = new Set(existingDocuments.map((doc) => doc.id))
      const uniqueNewDocuments = newDocuments.filter((doc) => !existingIds.has(doc.id))
@@ -689,15 +759,29 @@ export const useKnowledgeStore = create<KnowledgeStore>((set, get) => ({

      const updatedDocuments = [...existingDocuments, ...uniqueNewDocuments]

+      const documentsCache: DocumentsCache = {
+        documents: updatedDocuments,
+        pagination: {
+          ...(existingDocumentsCache?.pagination || {
+            limit: 50,
+            offset: 0,
+            hasMore: false,
+          }),
+          total: updatedDocuments.length,
+        },
+        searchQuery: existingDocumentsCache?.searchQuery,
+        lastFetchTime: Date.now(),
+      }
+
      return {
        documents: {
          ...state.documents,
-          [knowledgeBaseId]: updatedDocuments,
+          [knowledgeBaseId]: documentsCache,
        },
      }
    })
    logger.info(
-      `Added ${newDocuments.filter((doc) => !get().documents[knowledgeBaseId]?.some((existing) => existing.id === doc.id)).length} pending documents for knowledge base: ${knowledgeBaseId}`
+      `Added ${newDocuments.filter((doc) => !get().documents[knowledgeBaseId]?.documents?.some((existing) => existing.id === doc.id)).length} pending documents for knowledge base: ${knowledgeBaseId}`
    )
  },

@@ -731,10 +815,10 @@ export const useKnowledgeStore = create<KnowledgeStore>((set, get) => ({

  removeDocument: (knowledgeBaseId: string, documentId: string) => {
    set((state) => {
-      const documents = state.documents[knowledgeBaseId]
-      if (!documents) return state
+      const documentsCache = state.documents[knowledgeBaseId]
+      if (!documentsCache) return state

-      const updatedDocuments = documents.filter((doc) => doc.id !== documentId)
+      const updatedDocuments = documentsCache.documents.filter((doc) => doc.id !== documentId)

      // Also clear chunks for the removed document
      const newChunks = { ...state.chunks }
@@ -743,7 +827,10 @@ export const useKnowledgeStore = create<KnowledgeStore>((set, get) => ({
      return {
        documents: {
          ...state.documents,
-          [knowledgeBaseId]: updatedDocuments,
+          [knowledgeBaseId]: {
+            ...documentsCache,
+            documents: updatedDocuments,
+          },
        },
        chunks: newChunks,
      }