feat(kb-tags-filtering): filter kb docs using pre-set tags (#648)

* feat(knowledge-base): tag filtering

* fix lint

* remove migrations

* fix migrations

* fix lint

* Update apps/sim/app/api/knowledge/search/route.ts

Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>

* fix lint

* fix lint

* UI

---------

Co-authored-by: Vikhyath Mondreti <vikhyathmondreti@Vikhyaths-Air.attlocal.net>
Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>
Co-authored-by: Waleed Latif <walif6@gmail.com>
This commit is contained in:
Vikhyath Mondreti
2025-07-09 22:54:40 -07:00
committed by GitHub
parent e5080febd5
commit 31d9e2a4a8
19 changed files with 6221 additions and 142 deletions

View File

@@ -619,6 +619,13 @@ export function mockKnowledgeSchemas() {
processingCompletedAt: 'processing_completed_at',
processingError: 'processing_error',
enabled: 'enabled',
tag1: 'tag1',
tag2: 'tag2',
tag3: 'tag3',
tag4: 'tag4',
tag5: 'tag5',
tag6: 'tag6',
tag7: 'tag7',
uploadedAt: 'uploaded_at',
deletedAt: 'deleted_at',
},
@@ -631,6 +638,13 @@ export function mockKnowledgeSchemas() {
embedding: 'embedding',
tokenCount: 'token_count',
characterCount: 'character_count',
tag1: 'tag1',
tag2: 'tag2',
tag3: 'tag3',
tag4: 'tag4',
tag5: 'tag5',
tag6: 'tag6',
tag7: 'tag7',
createdAt: 'created_at',
},
}))

View File

@@ -118,7 +118,13 @@ export async function GET(
enabled: embedding.enabled,
startOffset: embedding.startOffset,
endOffset: embedding.endOffset,
metadata: embedding.metadata,
tag1: embedding.tag1,
tag2: embedding.tag2,
tag3: embedding.tag3,
tag4: embedding.tag4,
tag5: embedding.tag5,
tag6: embedding.tag6,
tag7: embedding.tag7,
createdAt: embedding.createdAt,
updatedAt: embedding.updatedAt,
})
@@ -239,7 +245,14 @@ export async function POST(
embeddingModel: 'text-embedding-3-small',
startOffset: 0, // Manual chunks don't have document offsets
endOffset: validatedData.content.length,
metadata: { manual: true }, // Mark as manually created
// Inherit tags from parent document
tag1: doc.tag1,
tag2: doc.tag2,
tag3: doc.tag3,
tag4: doc.tag4,
tag5: doc.tag5,
tag6: doc.tag6,
tag7: doc.tag7,
enabled: validatedData.enabled,
createdAt: now,
updatedAt: now,

View File

@@ -153,6 +153,14 @@ const CreateDocumentSchema = z.object({
fileUrl: z.string().url('File URL must be valid'),
fileSize: z.number().min(1, 'File size must be greater than 0'),
mimeType: z.string().min(1, 'MIME type is required'),
// Document tags for filtering
tag1: z.string().optional(),
tag2: z.string().optional(),
tag3: z.string().optional(),
tag4: z.string().optional(),
tag5: z.string().optional(),
tag6: z.string().optional(),
tag7: z.string().optional(),
})
const BulkCreateDocumentsSchema = z.object({
@@ -229,6 +237,14 @@ export async function GET(req: NextRequest, { params }: { params: Promise<{ id:
processingError: document.processingError,
enabled: document.enabled,
uploadedAt: document.uploadedAt,
// Include tags in response
tag1: document.tag1,
tag2: document.tag2,
tag3: document.tag3,
tag4: document.tag4,
tag5: document.tag5,
tag6: document.tag6,
tag7: document.tag7,
})
.from(document)
.where(and(...whereConditions))
@@ -298,6 +314,14 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
processingStatus: 'pending' as const,
enabled: true,
uploadedAt: now,
// Include tags from upload
tag1: docData.tag1 || null,
tag2: docData.tag2 || null,
tag3: docData.tag3 || null,
tag4: docData.tag4 || null,
tag5: docData.tag5 || null,
tag6: docData.tag6 || null,
tag7: docData.tag7 || null,
}
await tx.insert(document).values(newDocument)
@@ -372,6 +396,14 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
characterCount: 0,
enabled: true,
uploadedAt: now,
// Include tags from upload
tag1: validatedData.tag1 || null,
tag2: validatedData.tag2 || null,
tag3: validatedData.tag3 || null,
tag4: validatedData.tag4 || null,
tag5: validatedData.tag5 || null,
tag6: validatedData.tag6 || null,
tag7: validatedData.tag7 || null,
}
await db.insert(document).values(newDocument)

View File

@@ -10,6 +10,30 @@ import { embedding, knowledgeBase } from '@/db/schema'
const logger = createLogger('VectorSearchAPI')
// Helper function to create tag filters
function getTagFilters(filters: Record<string, string>, embedding: any) {
return Object.entries(filters).map(([key, value]) => {
switch (key) {
case 'tag1':
return sql`LOWER(${embedding.tag1}) = LOWER(${value})`
case 'tag2':
return sql`LOWER(${embedding.tag2}) = LOWER(${value})`
case 'tag3':
return sql`LOWER(${embedding.tag3}) = LOWER(${value})`
case 'tag4':
return sql`LOWER(${embedding.tag4}) = LOWER(${value})`
case 'tag5':
return sql`LOWER(${embedding.tag5}) = LOWER(${value})`
case 'tag6':
return sql`LOWER(${embedding.tag6}) = LOWER(${value})`
case 'tag7':
return sql`LOWER(${embedding.tag7}) = LOWER(${value})`
default:
return sql`1=1` // No-op for unknown keys
}
})
}
class APIError extends Error {
public status: number
@@ -27,6 +51,18 @@ const VectorSearchSchema = z.object({
]),
query: z.string().min(1, 'Search query is required'),
topK: z.number().min(1).max(100).default(10),
// Tag filters for pre-filtering
filters: z
.object({
tag1: z.string().optional(),
tag2: z.string().optional(),
tag3: z.string().optional(),
tag4: z.string().optional(),
tag5: z.string().optional(),
tag6: z.string().optional(),
tag7: z.string().optional(),
})
.optional(),
})
async function generateSearchEmbedding(query: string): Promise<number[]> {
@@ -102,7 +138,8 @@ async function executeParallelQueries(
knowledgeBaseIds: string[],
queryVector: string,
topK: number,
distanceThreshold: number
distanceThreshold: number,
filters?: Record<string, string>
) {
const parallelLimit = Math.ceil(topK / knowledgeBaseIds.length) + 5
@@ -113,7 +150,13 @@ async function executeParallelQueries(
content: embedding.content,
documentId: embedding.documentId,
chunkIndex: embedding.chunkIndex,
metadata: embedding.metadata,
tag1: embedding.tag1,
tag2: embedding.tag2,
tag3: embedding.tag3,
tag4: embedding.tag4,
tag5: embedding.tag5,
tag6: embedding.tag6,
tag7: embedding.tag7,
distance: sql<number>`${embedding.embedding} <=> ${queryVector}::vector`.as('distance'),
knowledgeBaseId: embedding.knowledgeBaseId,
})
@@ -122,7 +165,9 @@ async function executeParallelQueries(
and(
eq(embedding.knowledgeBaseId, kbId),
eq(embedding.enabled, true),
sql`${embedding.embedding} <=> ${queryVector}::vector < ${distanceThreshold}`
sql`${embedding.embedding} <=> ${queryVector}::vector < ${distanceThreshold}`,
// Apply tag filters if provided (case-insensitive)
...(filters ? getTagFilters(filters, embedding) : [])
)
)
.orderBy(sql`${embedding.embedding} <=> ${queryVector}::vector`)
@@ -139,7 +184,8 @@ async function executeSingleQuery(
knowledgeBaseIds: string[],
queryVector: string,
topK: number,
distanceThreshold: number
distanceThreshold: number,
filters?: Record<string, string>
) {
return await db
.select({
@@ -147,7 +193,13 @@ async function executeSingleQuery(
content: embedding.content,
documentId: embedding.documentId,
chunkIndex: embedding.chunkIndex,
metadata: embedding.metadata,
tag1: embedding.tag1,
tag2: embedding.tag2,
tag3: embedding.tag3,
tag4: embedding.tag4,
tag5: embedding.tag5,
tag6: embedding.tag6,
tag7: embedding.tag7,
distance: sql<number>`${embedding.embedding} <=> ${queryVector}::vector`.as('distance'),
})
.from(embedding)
@@ -155,7 +207,30 @@ async function executeSingleQuery(
and(
inArray(embedding.knowledgeBaseId, knowledgeBaseIds),
eq(embedding.enabled, true),
sql`${embedding.embedding} <=> ${queryVector}::vector < ${distanceThreshold}`
sql`${embedding.embedding} <=> ${queryVector}::vector < ${distanceThreshold}`,
// Apply tag filters if provided (case-insensitive)
...(filters
? Object.entries(filters).map(([key, value]) => {
switch (key) {
case 'tag1':
return sql`LOWER(${embedding.tag1}) = LOWER(${value})`
case 'tag2':
return sql`LOWER(${embedding.tag2}) = LOWER(${value})`
case 'tag3':
return sql`LOWER(${embedding.tag3}) = LOWER(${value})`
case 'tag4':
return sql`LOWER(${embedding.tag4}) = LOWER(${value})`
case 'tag5':
return sql`LOWER(${embedding.tag5}) = LOWER(${value})`
case 'tag6':
return sql`LOWER(${embedding.tag6}) = LOWER(${value})`
case 'tag7':
return sql`LOWER(${embedding.tag7}) = LOWER(${value})`
default:
return sql`1=1` // No-op for unknown keys
}
})
: [])
)
)
.orderBy(sql`${embedding.embedding} <=> ${queryVector}::vector`)
@@ -231,7 +306,8 @@ export async function POST(request: NextRequest) {
foundKbIds,
queryVector,
validatedData.topK,
strategy.distanceThreshold
strategy.distanceThreshold,
validatedData.filters
)
results = mergeAndRankResults(parallelResults, validatedData.topK)
} else {
@@ -240,7 +316,8 @@ export async function POST(request: NextRequest) {
foundKbIds,
queryVector,
validatedData.topK,
strategy.distanceThreshold
strategy.distanceThreshold,
validatedData.filters
)
}
@@ -252,7 +329,13 @@ export async function POST(request: NextRequest) {
content: result.content,
documentId: result.documentId,
chunkIndex: result.chunkIndex,
metadata: result.metadata,
tag1: result.tag1,
tag2: result.tag2,
tag3: result.tag3,
tag4: result.tag4,
tag5: result.tag5,
tag6: result.tag6,
tag7: result.tag7,
similarity: 1 - result.distance,
})),
query: validatedData.query,

View File

@@ -73,6 +73,14 @@ export interface DocumentData {
enabled: boolean
deletedAt?: Date | null
uploadedAt: Date
// Document tags
tag1?: string | null
tag2?: string | null
tag3?: string | null
tag4?: string | null
tag5?: string | null
tag6?: string | null
tag7?: string | null
}
export interface EmbeddingData {
@@ -88,7 +96,14 @@ export interface EmbeddingData {
embeddingModel: string
startOffset: number
endOffset: number
metadata: unknown
// Tag fields for filtering
tag1?: string | null
tag2?: string | null
tag3?: string | null
tag4?: string | null
tag5?: string | null
tag6?: string | null
tag7?: string | null
enabled: boolean
createdAt: Date
updatedAt: Date
@@ -445,7 +460,26 @@ export async function processDocumentAsync(
const chunkTexts = processed.chunks.map((chunk) => chunk.text)
const embeddings = chunkTexts.length > 0 ? await generateEmbeddings(chunkTexts) : []
logger.info(`[${documentId}] Embeddings generated, updating document record`)
logger.info(`[${documentId}] Embeddings generated, fetching document tags`)
// Fetch document to get tags
const documentRecord = await db
.select({
tag1: document.tag1,
tag2: document.tag2,
tag3: document.tag3,
tag4: document.tag4,
tag5: document.tag5,
tag6: document.tag6,
tag7: document.tag7,
})
.from(document)
.where(eq(document.id, documentId))
.limit(1)
const documentTags = documentRecord[0] || {}
logger.info(`[${documentId}] Creating embedding records with tags`)
const embeddingRecords = processed.chunks.map((chunk, chunkIndex) => ({
id: crypto.randomUUID(),
@@ -460,7 +494,14 @@ export async function processDocumentAsync(
embeddingModel: 'text-embedding-3-small',
startOffset: chunk.metadata.startIndex,
endOffset: chunk.metadata.endIndex,
metadata: {},
// Copy tags from document
tag1: documentTags.tag1,
tag2: documentTags.tag2,
tag3: documentTags.tag3,
tag4: documentTags.tag4,
tag5: documentTags.tag5,
tag6: documentTags.tag6,
tag7: documentTags.tag7,
createdAt: now,
updatedAt: now,
}))

View File

@@ -417,6 +417,37 @@ export function Document({
</Button>
</div>
{/* Document Tags Display */}
{document &&
(() => {
const tags = [
{ label: 'Tag 1', value: document.tag1 },
{ label: 'Tag 2', value: document.tag2 },
{ label: 'Tag 3', value: document.tag3 },
{ label: 'Tag 4', value: document.tag4 },
{ label: 'Tag 5', value: document.tag5 },
{ label: 'Tag 6', value: document.tag6 },
{ label: 'Tag 7', value: document.tag7 },
].filter((tag) => tag.value?.trim())
return tags.length > 0 ? (
<div className='mb-4 rounded-md bg-muted/50 p-3'>
<p className='mb-2 text-muted-foreground text-xs'>Document Tags:</p>
<div className='flex flex-wrap gap-2'>
{tags.map((tag, index) => (
<span
key={index}
className='inline-flex items-center gap-1 rounded-md bg-primary/10 px-2 py-1 text-primary text-xs'
>
<span className='font-medium'>{tag.label}:</span>
<span>{tag.value}</span>
</span>
))}
</div>
</div>
) : null
})()}
{/* Error State for chunks */}
{combinedError && !isLoadingAllChunks && (
<div className='mb-4 rounded-md border border-red-200 bg-red-50 p-4'>

View File

@@ -1,7 +1,7 @@
'use client'
import { useEffect, useRef, useState } from 'react'
import { format, formatDistanceToNow } from 'date-fns'
import { useEffect, useState } from 'react'
import { format } from 'date-fns'
import {
AlertCircle,
Circle,
@@ -11,7 +11,6 @@ import {
Plus,
RotateCcw,
Trash2,
X,
} from 'lucide-react'
import { useParams, useRouter } from 'next/navigation'
import {
@@ -36,8 +35,8 @@ import { useKnowledgeBase, useKnowledgeBaseDocuments } from '@/hooks/use-knowled
import { type DocumentData, useKnowledgeStore } from '@/stores/knowledge/store'
import { useSidebarStore } from '@/stores/sidebar/store'
import { KnowledgeHeader } from '../components/knowledge-header/knowledge-header'
import { useKnowledgeUpload } from '../hooks/use-knowledge-upload'
import { KnowledgeBaseLoading } from './components/knowledge-base-loading/knowledge-base-loading'
import { UploadModal } from './components/upload-modal/upload-modal'
const logger = createLogger('KnowledgeBase')
@@ -138,36 +137,11 @@ export function KnowledgeBase({
const [searchQuery, setSearchQuery] = useState('')
const [selectedDocuments, setSelectedDocuments] = useState<Set<string>>(new Set())
const [showDeleteDialog, setShowDeleteDialog] = useState(false)
const [showUploadModal, setShowUploadModal] = useState(false)
const [isDeleting, setIsDeleting] = useState(false)
const [isBulkOperating, setIsBulkOperating] = useState(false)
const { isUploading, uploadProgress, uploadError, uploadFiles, clearError } = useKnowledgeUpload({
onUploadComplete: async (uploadedFiles) => {
const pendingDocuments: DocumentData[] = uploadedFiles.map((file, index) => ({
id: `temp-${Date.now()}-${index}`,
knowledgeBaseId: id,
filename: file.filename,
fileUrl: file.fileUrl,
fileSize: file.fileSize,
mimeType: file.mimeType,
chunkCount: 0,
tokenCount: 0,
characterCount: 0,
processingStatus: 'pending' as const,
processingStartedAt: null,
processingCompletedAt: null,
processingError: null,
enabled: true,
uploadedAt: new Date().toISOString(),
}))
useKnowledgeStore.getState().addPendingDocuments(id, pendingDocuments)
await refreshDocuments()
},
})
const router = useRouter()
const fileInputRef = useRef<HTMLInputElement>(null)
const knowledgeBaseName = knowledgeBase?.name || passedKnowledgeBaseName || 'Knowledge Base'
const error = knowledgeBaseError || documentsError
@@ -183,7 +157,7 @@ export function KnowledgeBase({
const refreshInterval = setInterval(async () => {
try {
// Only refresh if we're not in the middle of other operations
if (!isUploading && !isDeleting) {
if (!isDeleting) {
// Check for dead processes before refreshing
await checkForDeadProcesses()
await refreshDocuments()
@@ -194,7 +168,7 @@ export function KnowledgeBase({
}, 3000) // Refresh every 3 seconds
return () => clearInterval(refreshInterval)
}, [documents, refreshDocuments, isUploading, isDeleting])
}, [documents, refreshDocuments, isDeleting])
// Check for documents stuck in processing due to dead processes
const checkForDeadProcesses = async () => {
@@ -246,16 +220,6 @@ export function KnowledgeBase({
await Promise.allSettled(markFailedPromises)
}
// Auto-dismiss upload error after 8 seconds
useEffect(() => {
if (uploadError) {
const timer = setTimeout(() => {
clearError()
}, 8000)
return () => clearTimeout(timer)
}
}, [uploadError, clearError])
// Filter documents based on search query
const filteredDocuments = documents.filter((doc) =>
doc.filename.toLowerCase().includes(searchQuery.toLowerCase())
@@ -451,30 +415,7 @@ export function KnowledgeBase({
}
const handleAddDocuments = () => {
fileInputRef.current?.click()
}
const handleFileUpload = async (e: React.ChangeEvent<HTMLInputElement>) => {
const files = e.target.files
if (!files || files.length === 0) return
try {
const chunkingConfig = knowledgeBase?.chunkingConfig
await uploadFiles(Array.from(files), id, {
chunkSize: chunkingConfig?.maxSize || 1024,
minCharactersPerChunk: chunkingConfig?.minSize || 100,
chunkOverlap: chunkingConfig?.overlap || 200,
recipe: 'default',
})
} catch (error) {
logger.error('Error uploading files:', error)
// Error handling is managed by the upload hook
} finally {
// Reset the file input
if (fileInputRef.current) {
fileInputRef.current.value = ''
}
}
setShowUploadModal(true)
}
const handleBulkEnable = async () => {
@@ -680,16 +621,6 @@ export function KnowledgeBase({
{/* Main Content */}
<div className='flex-1 overflow-auto'>
<div className='px-6 pb-6'>
{/* Hidden file input for document upload */}
<input
ref={fileInputRef}
type='file'
accept='.pdf,.doc,.docx,.txt,.csv,.xls,.xlsx'
onChange={handleFileUpload}
className='hidden'
multiple
/>
{/* Search and Create Section */}
<div className='mb-4 flex items-center justify-between pt-1'>
<SearchInput
@@ -700,17 +631,9 @@ export function KnowledgeBase({
<div className='flex items-center gap-3'>
{/* Add Documents Button */}
<PrimaryButton onClick={handleAddDocuments} disabled={isUploading}>
<PrimaryButton onClick={handleAddDocuments}>
<Plus className='h-3.5 w-3.5' />
{isUploading
? uploadProgress.stage === 'uploading'
? `Uploading ${uploadProgress.filesCompleted + 1}/${uploadProgress.totalFiles}...`
: uploadProgress.stage === 'processing'
? 'Processing...'
: uploadProgress.stage === 'completing'
? 'Completing...'
: 'Uploading...'
: 'Add Documents'}
Add Documents
</PrimaryButton>
</div>
</div>
@@ -1105,38 +1028,14 @@ export function KnowledgeBase({
</AlertDialogContent>
</AlertDialog>
{/* Toast Notification */}
{uploadError && (
<div className='slide-in-from-bottom-2 fixed right-4 bottom-4 z-50 max-w-md animate-in duration-300'>
<div className='flex cursor-pointer items-start gap-3 rounded-lg border bg-background p-4 shadow-lg'>
<div className='flex-shrink-0'>
<AlertCircle className='h-4 w-4 text-destructive' />
</div>
<div className='flex min-w-0 flex-1 flex-col gap-1'>
<div className='flex items-center gap-2'>
<span className='font-medium text-xs'>Error</span>
<span className='text-muted-foreground text-xs'>
{formatDistanceToNow(uploadError.timestamp, { addSuffix: true }).replace(
'less than a minute ago',
'<1 minute ago'
)}
</span>
</div>
<p className='overflow-wrap-anywhere hyphens-auto whitespace-normal break-normal text-foreground text-sm'>
{uploadError.message.length > 100
? `${uploadError.message.slice(0, 60)}...`
: uploadError.message}
</p>
</div>
<button
onClick={() => clearError()}
className='flex-shrink-0 rounded-sm opacity-70 hover:opacity-100 focus:outline-none focus:ring-2 focus:ring-ring'
>
<X className='h-4 w-4' />
</button>
</div>
</div>
)}
{/* Upload Modal */}
<UploadModal
open={showUploadModal}
onOpenChange={setShowUploadModal}
knowledgeBaseId={id}
chunkingConfig={knowledgeBase?.chunkingConfig}
onUploadComplete={refreshDocuments}
/>
{/* Bulk Action Bar */}
<ActionBar

View File

@@ -0,0 +1,284 @@
'use client'
import { useRef, useState } from 'react'
import { X } from 'lucide-react'
import { Button } from '@/components/ui/button'
import { Dialog, DialogContent, DialogHeader, DialogTitle } from '@/components/ui/dialog'
import { Label } from '@/components/ui/label'
import { createLogger } from '@/lib/logs/console-logger'
import { type TagData, TagInput } from '../../../components/tag-input/tag-input'
import { useKnowledgeUpload } from '../../../hooks/use-knowledge-upload'
const logger = createLogger('UploadModal')
const MAX_FILE_SIZE = 100 * 1024 * 1024 // 100MB
const ACCEPTED_FILE_TYPES = [
'application/pdf',
'application/msword',
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'text/plain',
'text/csv',
'application/vnd.ms-excel',
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
]
interface FileWithPreview extends File {
preview: string
}
interface UploadModalProps {
open: boolean
onOpenChange: (open: boolean) => void
knowledgeBaseId: string
chunkingConfig?: {
maxSize: number
minSize: number
overlap: number
}
onUploadComplete?: () => void
}
export function UploadModal({
open,
onOpenChange,
knowledgeBaseId,
chunkingConfig,
onUploadComplete,
}: UploadModalProps) {
const fileInputRef = useRef<HTMLInputElement>(null)
const [files, setFiles] = useState<FileWithPreview[]>([])
const [tags, setTags] = useState<TagData>({})
const [fileError, setFileError] = useState<string | null>(null)
const [isDragging, setIsDragging] = useState(false)
const { isUploading, uploadProgress, uploadFiles } = useKnowledgeUpload({
onUploadComplete: () => {
logger.info(`Successfully uploaded ${files.length} files`)
onUploadComplete?.()
handleClose()
},
})
const handleClose = () => {
if (isUploading) return // Prevent closing during upload
setFiles([])
setTags({})
setFileError(null)
setIsDragging(false)
onOpenChange(false)
}
const validateFile = (file: File): string | null => {
if (file.size > MAX_FILE_SIZE) {
return `File "${file.name}" is too large. Maximum size is 100MB.`
}
if (!ACCEPTED_FILE_TYPES.includes(file.type)) {
return `File "${file.name}" has an unsupported format. Please use PDF, DOC, DOCX, TXT, CSV, XLS, or XLSX files.`
}
return null
}
const processFiles = (fileList: FileList | File[]) => {
setFileError(null)
const newFiles: FileWithPreview[] = []
for (const file of Array.from(fileList)) {
const error = validateFile(file)
if (error) {
setFileError(error)
return
}
const fileWithPreview = Object.assign(file, {
preview: URL.createObjectURL(file),
})
newFiles.push(fileWithPreview)
}
setFiles((prev) => [...prev, ...newFiles])
}
const removeFile = (index: number) => {
setFiles((prev) => {
const newFiles = [...prev]
const removedFile = newFiles.splice(index, 1)[0]
if (removedFile.preview) {
URL.revokeObjectURL(removedFile.preview)
}
return newFiles
})
}
const handleFileChange = (e: React.ChangeEvent<HTMLInputElement>) => {
if (e.target.files) {
processFiles(e.target.files)
}
}
const handleDragOver = (e: React.DragEvent) => {
e.preventDefault()
e.stopPropagation()
setIsDragging(true)
}
const handleDragLeave = (e: React.DragEvent) => {
e.preventDefault()
e.stopPropagation()
setIsDragging(false)
}
const handleDrop = (e: React.DragEvent) => {
e.preventDefault()
e.stopPropagation()
setIsDragging(false)
if (e.dataTransfer.files) {
processFiles(e.dataTransfer.files)
}
}
const handleUpload = async () => {
if (files.length === 0) return
try {
// Create files with tags for upload
const filesWithTags = files.map((file) => {
// Add tags as custom properties to the file object
const fileWithTags = file as File & TagData
Object.assign(fileWithTags, tags)
return fileWithTags
})
await uploadFiles(filesWithTags, knowledgeBaseId, {
chunkSize: chunkingConfig?.maxSize || 1024,
minCharactersPerChunk: chunkingConfig?.minSize || 100,
chunkOverlap: chunkingConfig?.overlap || 200,
recipe: 'default',
})
} catch (error) {
logger.error('Error uploading files:', error)
}
}
return (
<Dialog open={open} onOpenChange={handleClose}>
<DialogContent className='flex max-h-[90vh] max-w-2xl flex-col overflow-hidden'>
<DialogHeader>
<DialogTitle>Upload Documents</DialogTitle>
</DialogHeader>
<div className='flex-1 space-y-6 overflow-auto'>
{/* Tag Input Section */}
<TagInput tags={tags} onTagsChange={setTags} disabled={isUploading} />
{/* File Upload Section */}
<div className='space-y-3'>
<Label>Select Files</Label>
{files.length === 0 ? (
<div
onDragOver={handleDragOver}
onDragLeave={handleDragLeave}
onDrop={handleDrop}
onClick={() => fileInputRef.current?.click()}
className={`relative flex cursor-pointer items-center justify-center rounded-lg border-2 border-dashed p-8 text-center transition-colors ${
isDragging
? 'border-primary bg-primary/5'
: 'border-muted-foreground/25 hover:border-muted-foreground/40 hover:bg-muted/10'
}`}
>
<input
ref={fileInputRef}
type='file'
accept={ACCEPTED_FILE_TYPES.join(',')}
onChange={handleFileChange}
className='hidden'
multiple
/>
<div className='space-y-2'>
<p className='font-medium text-sm'>
{isDragging ? 'Drop files here!' : 'Drop files here or click to browse'}
</p>
<p className='text-muted-foreground text-xs'>
Supports PDF, DOC, DOCX, TXT, CSV, XLS, XLSX (max 100MB each)
</p>
</div>
</div>
) : (
<div className='space-y-2'>
<div
onDragOver={handleDragOver}
onDragLeave={handleDragLeave}
onDrop={handleDrop}
onClick={() => fileInputRef.current?.click()}
className={`cursor-pointer rounded-md border border-dashed p-3 text-center transition-colors ${
isDragging
? 'border-primary bg-primary/5'
: 'border-muted-foreground/25 hover:border-muted-foreground/40'
}`}
>
<input
ref={fileInputRef}
type='file'
accept={ACCEPTED_FILE_TYPES.join(',')}
onChange={handleFileChange}
className='hidden'
multiple
/>
<p className='text-sm'>
{isDragging ? 'Drop more files here!' : 'Add more files'}
</p>
</div>
<div className='max-h-40 space-y-2 overflow-auto'>
{files.map((file, index) => (
<div
key={index}
className='flex items-center justify-between rounded-md border p-3'
>
<div className='min-w-0 flex-1'>
<p className='truncate font-medium text-sm'>{file.name}</p>
<p className='text-muted-foreground text-xs'>
{(file.size / 1024 / 1024).toFixed(2)} MB
</p>
</div>
<Button
type='button'
variant='ghost'
size='sm'
onClick={() => removeFile(index)}
disabled={isUploading}
className='h-8 w-8 p-0'
>
<X className='h-4 w-4' />
</Button>
</div>
))}
</div>
</div>
)}
{fileError && <p className='text-destructive text-sm'>{fileError}</p>}
</div>
</div>
{/* Footer */}
<div className='flex justify-end gap-3 border-t pt-4'>
<Button variant='outline' onClick={handleClose} disabled={isUploading}>
Cancel
</Button>
<Button onClick={handleUpload} disabled={files.length === 0 || isUploading}>
{isUploading
? uploadProgress.stage === 'uploading'
? `Uploading ${uploadProgress.filesCompleted + 1}/${uploadProgress.totalFiles}...`
: uploadProgress.stage === 'processing'
? 'Processing...'
: 'Uploading...'
: `Upload ${files.length} file${files.length !== 1 ? 's' : ''}`}
</Button>
</div>
</DialogContent>
</Dialog>
)
}

View File

@@ -15,6 +15,7 @@ import { createLogger } from '@/lib/logs/console-logger'
import { getDocumentIcon } from '@/app/workspace/[workspaceId]/knowledge/components/icons/document-icons'
import type { KnowledgeBaseData } from '@/stores/knowledge/store'
import { useKnowledgeUpload } from '../../hooks/use-knowledge-upload'
import { type TagData, TagInput } from '../tag-input/tag-input'
const logger = createLogger('CreateModal')
@@ -80,6 +81,7 @@ export function CreateModal({ open, onOpenChange, onKnowledgeBaseCreated }: Crea
const [fileError, setFileError] = useState<string | null>(null)
const [isDragging, setIsDragging] = useState(false)
const [dragCounter, setDragCounter] = useState(0) // Track drag events to handle nested elements
const [tags, setTags] = useState<TagData>({})
const scrollContainerRef = useRef<HTMLDivElement>(null)
const dropZoneRef = useRef<HTMLDivElement>(null)
@@ -273,7 +275,14 @@ export function CreateModal({ open, onOpenChange, onKnowledgeBaseCreated }: Crea
const newKnowledgeBase = result.data
if (files.length > 0) {
const uploadedFiles = await uploadFiles(files, newKnowledgeBase.id, {
// Add tags to files before upload
const filesWithTags = files.map((file) => {
const fileWithTags = file as File & TagData
Object.assign(fileWithTags, tags)
return fileWithTags
})
const uploadedFiles = await uploadFiles(filesWithTags, newKnowledgeBase.id, {
chunkSize: data.maxChunkSize,
minCharactersPerChunk: data.minChunkSize,
chunkOverlap: data.overlapSize,
@@ -297,6 +306,7 @@ export function CreateModal({ open, onOpenChange, onKnowledgeBaseCreated }: Crea
maxChunkSize: 1024,
overlapSize: 200,
})
setTags({})
// Clean up file previews
files.forEach((file) => URL.revokeObjectURL(file.preview))
@@ -463,6 +473,11 @@ export function CreateModal({ open, onOpenChange, onKnowledgeBaseCreated }: Crea
</div>
</div>
{/* Tag Input Section */}
<div className='mt-6'>
<TagInput tags={tags} onTagsChange={setTags} disabled={isSubmitting} />
</div>
{/* File Upload Section - Expands to fill remaining space */}
<div className='mt-6 flex flex-1 flex-col'>
<Label className='mb-2'>Upload Documents</Label>

View File

@@ -0,0 +1,185 @@
'use client'
import { useState } from 'react'
import { ChevronDown, ChevronRight, Plus, Settings, X } from 'lucide-react'
import { Button } from '@/components/ui/button'
import { Collapsible, CollapsibleContent, CollapsibleTrigger } from '@/components/ui/collapsible'
import { Input } from '@/components/ui/input'
import { Label } from '@/components/ui/label'
export interface TagData {
tag1?: string
tag2?: string
tag3?: string
tag4?: string
tag5?: string
tag6?: string
tag7?: string
}
interface TagInputProps {
tags: TagData
onTagsChange: (tags: TagData) => void
disabled?: boolean
className?: string
}
const TAG_LABELS = [
{ key: 'tag1' as keyof TagData, label: 'Tag 1', placeholder: 'Enter tag value' },
{ key: 'tag2' as keyof TagData, label: 'Tag 2', placeholder: 'Enter tag value' },
{ key: 'tag3' as keyof TagData, label: 'Tag 3', placeholder: 'Enter tag value' },
{ key: 'tag4' as keyof TagData, label: 'Tag 4', placeholder: 'Enter tag value' },
{ key: 'tag5' as keyof TagData, label: 'Tag 5', placeholder: 'Enter tag value' },
{ key: 'tag6' as keyof TagData, label: 'Tag 6', placeholder: 'Enter tag value' },
{ key: 'tag7' as keyof TagData, label: 'Tag 7', placeholder: 'Enter tag value' },
]
export function TagInput({ tags, onTagsChange, disabled = false, className = '' }: TagInputProps) {
const [isOpen, setIsOpen] = useState(false)
const [showAllTags, setShowAllTags] = useState(false)
const handleTagChange = (tagKey: keyof TagData, value: string) => {
onTagsChange({
...tags,
[tagKey]: value.trim() || undefined,
})
}
const clearTag = (tagKey: keyof TagData) => {
onTagsChange({
...tags,
[tagKey]: undefined,
})
}
const hasAnyTags = Object.values(tags).some((tag) => tag?.trim())
const visibleTags = showAllTags ? TAG_LABELS : TAG_LABELS.slice(0, 2)
return (
<div className={className}>
<Collapsible open={isOpen} onOpenChange={setIsOpen}>
<CollapsibleTrigger asChild>
<Button
type='button'
variant='ghost'
className='flex h-auto w-full justify-between p-0 hover:bg-transparent'
>
<div className='flex items-center gap-2'>
<Settings className='h-4 w-4 text-muted-foreground' />
<Label className='cursor-pointer font-medium text-sm'>Advanced Settings</Label>
{hasAnyTags && (
<span className='rounded-full bg-primary/10 px-2 py-0.5 text-primary text-xs'>
{Object.values(tags).filter((tag) => tag?.trim()).length} tag
{Object.values(tags).filter((tag) => tag?.trim()).length !== 1 ? 's' : ''}
</span>
)}
</div>
{isOpen ? (
<ChevronDown className='h-4 w-4 text-muted-foreground' />
) : (
<ChevronRight className='h-4 w-4 text-muted-foreground' />
)}
</Button>
</CollapsibleTrigger>
<CollapsibleContent className='space-y-4 pt-4'>
<div className='space-y-3'>
<div className='flex items-center justify-between'>
<Label className='font-medium text-sm'>Document Tags</Label>
{!showAllTags && (
<Button
type='button'
variant='ghost'
size='sm'
onClick={() => setShowAllTags(true)}
className='h-auto p-1 text-muted-foreground text-xs hover:text-foreground'
>
<Plus className='mr-1 h-3 w-3' />
More tags
</Button>
)}
</div>
<div className='grid grid-cols-1 gap-3 sm:grid-cols-2'>
{visibleTags.map(({ key, label, placeholder }) => (
<div key={key} className='space-y-1'>
<Label htmlFor={key} className='text-muted-foreground text-xs'>
{label}
</Label>
<div className='relative'>
<Input
id={key}
type='text'
value={tags[key] || ''}
onChange={(e) => handleTagChange(key, e.target.value)}
placeholder={placeholder}
disabled={disabled}
className='pr-8 text-sm'
/>
{tags[key] && (
<Button
type='button'
variant='ghost'
size='sm'
onClick={() => clearTag(key)}
disabled={disabled}
className='-translate-y-1/2 absolute top-1/2 right-1 h-6 w-6 p-0 hover:bg-muted'
>
<X className='h-3 w-3' />
</Button>
)}
</div>
</div>
))}
</div>
{showAllTags && (
<div className='flex justify-center'>
<Button
type='button'
variant='ghost'
size='sm'
onClick={() => setShowAllTags(false)}
className='h-auto p-1 text-muted-foreground text-xs hover:text-foreground'
>
Show fewer tags
</Button>
</div>
)}
{hasAnyTags && (
<div className='rounded-md bg-muted/50 p-3'>
<p className='mb-2 text-muted-foreground text-xs'>Active tags:</p>
<div className='flex flex-wrap gap-1'>
{Object.entries(tags).map(([key, value]) => {
if (!value?.trim()) return null
const tagLabel = TAG_LABELS.find((t) => t.key === key)?.label || key
return (
<span
key={key}
className='inline-flex items-center gap-1 rounded-md bg-primary/10 px-2 py-1 text-primary text-xs'
>
<span className='font-medium'>{tagLabel}:</span>
<span>{value}</span>
<Button
type='button'
variant='ghost'
size='sm'
onClick={() => clearTag(key as keyof TagData)}
disabled={disabled}
className='h-3 w-3 p-0 hover:bg-primary/20'
>
<X className='h-2 w-2' />
</Button>
</span>
)
})}
</div>
</div>
)}
</div>
</CollapsibleContent>
</Collapsible>
</div>
)
}

View File

@@ -8,6 +8,14 @@ export interface UploadedFile {
fileUrl: string
fileSize: number
mimeType: string
// Document tags
tag1?: string
tag2?: string
tag3?: string
tag4?: string
tag5?: string
tag6?: string
tag7?: string
}
export interface UploadProgress {
@@ -78,12 +86,21 @@ export function useKnowledgeUpload(options: UseKnowledgeUploadOptions = {}) {
filename: string,
fileUrl: string,
fileSize: number,
mimeType: string
mimeType: string,
originalFile?: File
): UploadedFile => ({
filename,
fileUrl,
fileSize,
mimeType,
// Include tags from original file if available
tag1: (originalFile as any)?.tag1,
tag2: (originalFile as any)?.tag2,
tag3: (originalFile as any)?.tag3,
tag4: (originalFile as any)?.tag4,
tag5: (originalFile as any)?.tag5,
tag6: (originalFile as any)?.tag6,
tag7: (originalFile as any)?.tag7,
})
const createErrorFromException = (error: unknown, defaultMessage: string): UploadError => {
@@ -196,7 +213,9 @@ export function useKnowledgeUpload(options: UseKnowledgeUploadOptions = {}) {
? presignedData.fileInfo.path
: `${window.location.origin}${presignedData.fileInfo.path}`
uploadedFiles.push(createUploadedFile(file.name, fullFileUrl, file.size, file.type))
uploadedFiles.push(
createUploadedFile(file.name, fullFileUrl, file.size, file.type, file)
)
} else {
// Fallback to traditional upload through API route
const formData = new FormData()
@@ -238,7 +257,8 @@ export function useKnowledgeUpload(options: UseKnowledgeUploadOptions = {}) {
? uploadResult.path
: `${window.location.origin}${uploadResult.path}`,
file.size,
file.type
file.type,
file
)
)
}
@@ -252,7 +272,17 @@ export function useKnowledgeUpload(options: UseKnowledgeUploadOptions = {}) {
// Start async document processing
const processPayload = {
documents: uploadedFiles,
documents: uploadedFiles.map((file) => ({
...file,
// Extract tags from file if they exist (added by upload modal)
tag1: (file as any).tag1,
tag2: (file as any).tag2,
tag3: (file as any).tag3,
tag4: (file as any).tag4,
tag5: (file as any).tag5,
tag6: (file as any).tag6,
tag7: (file as any).tag7,
})),
processingOptions: {
chunkSize: processingOptions.chunkSize || 1024,
minCharactersPerChunk: processingOptions.minCharactersPerChunk || 100,

View File

@@ -36,6 +36,15 @@ export const KnowledgeBlock: BlockConfig = {
topK: { type: 'number', required: false },
documentId: { type: 'string', required: false },
content: { type: 'string', required: false },
name: { type: 'string', required: false },
// Tag filters for search
tag1: { type: 'string', required: false },
tag2: { type: 'string', required: false },
tag3: { type: 'string', required: false },
tag4: { type: 'string', required: false },
tag5: { type: 'string', required: false },
tag6: { type: 'string', required: false },
tag7: { type: 'string', required: false },
},
outputs: {
results: 'json',
@@ -89,6 +98,69 @@ export const KnowledgeBlock: BlockConfig = {
placeholder: 'Enter number of results (default: 10)',
condition: { field: 'operation', value: 'search' },
},
{
id: 'tag1',
title: 'Tag 1 Filter',
type: 'short-input',
layout: 'half',
placeholder: 'Filter by tag 1',
condition: { field: 'operation', value: 'search' },
mode: 'advanced',
},
{
id: 'tag2',
title: 'Tag 2 Filter',
type: 'short-input',
layout: 'half',
placeholder: 'Filter by tag 2',
condition: { field: 'operation', value: 'search' },
mode: 'advanced',
},
{
id: 'tag3',
title: 'Tag 3 Filter',
type: 'short-input',
layout: 'half',
placeholder: 'Filter by tag 3',
condition: { field: 'operation', value: 'search' },
mode: 'advanced',
},
{
id: 'tag4',
title: 'Tag 4 Filter',
type: 'short-input',
layout: 'half',
placeholder: 'Filter by tag 4',
condition: { field: 'operation', value: 'search' },
mode: 'advanced',
},
{
id: 'tag5',
title: 'Tag 5 Filter',
type: 'short-input',
layout: 'half',
placeholder: 'Filter by tag 5',
condition: { field: 'operation', value: 'search' },
mode: 'advanced',
},
{
id: 'tag6',
title: 'Tag 6 Filter',
type: 'short-input',
layout: 'half',
placeholder: 'Filter by tag 6',
condition: { field: 'operation', value: 'search' },
mode: 'advanced',
},
{
id: 'tag7',
title: 'Tag 7 Filter',
type: 'short-input',
layout: 'half',
placeholder: 'Filter by tag 7',
condition: { field: 'operation', value: 'search' },
mode: 'advanced',
},
{
id: 'documentId',
title: 'Document',
@@ -123,5 +195,69 @@ export const KnowledgeBlock: BlockConfig = {
rows: 6,
condition: { field: 'operation', value: ['create_document'] },
},
// Tag inputs for Create Document (in advanced mode)
{
id: 'tag1',
title: 'Tag 1',
type: 'short-input',
layout: 'half',
placeholder: 'Enter tag 1 value',
condition: { field: 'operation', value: 'create_document' },
mode: 'advanced',
},
{
id: 'tag2',
title: 'Tag 2',
type: 'short-input',
layout: 'half',
placeholder: 'Enter tag 2 value',
condition: { field: 'operation', value: 'create_document' },
mode: 'advanced',
},
{
id: 'tag3',
title: 'Tag 3',
type: 'short-input',
layout: 'half',
placeholder: 'Enter tag 3 value',
condition: { field: 'operation', value: 'create_document' },
mode: 'advanced',
},
{
id: 'tag4',
title: 'Tag 4',
type: 'short-input',
layout: 'half',
placeholder: 'Enter tag 4 value',
condition: { field: 'operation', value: 'create_document' },
mode: 'advanced',
},
{
id: 'tag5',
title: 'Tag 5',
type: 'short-input',
layout: 'half',
placeholder: 'Enter tag 5 value',
condition: { field: 'operation', value: 'create_document' },
mode: 'advanced',
},
{
id: 'tag6',
title: 'Tag 6',
type: 'short-input',
layout: 'half',
placeholder: 'Enter tag 6 value',
condition: { field: 'operation', value: 'create_document' },
mode: 'advanced',
},
{
id: 'tag7',
title: 'Tag 7',
type: 'short-input',
layout: 'half',
placeholder: 'Enter tag 7 value',
condition: { field: 'operation', value: 'create_document' },
mode: 'advanced',
},
],
}

View File

@@ -0,0 +1,30 @@
DROP INDEX "emb_metadata_gin_idx";--> statement-breakpoint
ALTER TABLE "document" ADD COLUMN "tag1" text;--> statement-breakpoint
ALTER TABLE "document" ADD COLUMN "tag2" text;--> statement-breakpoint
ALTER TABLE "document" ADD COLUMN "tag3" text;--> statement-breakpoint
ALTER TABLE "document" ADD COLUMN "tag4" text;--> statement-breakpoint
ALTER TABLE "document" ADD COLUMN "tag5" text;--> statement-breakpoint
ALTER TABLE "document" ADD COLUMN "tag6" text;--> statement-breakpoint
ALTER TABLE "document" ADD COLUMN "tag7" text;--> statement-breakpoint
ALTER TABLE "embedding" ADD COLUMN "tag1" text;--> statement-breakpoint
ALTER TABLE "embedding" ADD COLUMN "tag2" text;--> statement-breakpoint
ALTER TABLE "embedding" ADD COLUMN "tag3" text;--> statement-breakpoint
ALTER TABLE "embedding" ADD COLUMN "tag4" text;--> statement-breakpoint
ALTER TABLE "embedding" ADD COLUMN "tag5" text;--> statement-breakpoint
ALTER TABLE "embedding" ADD COLUMN "tag6" text;--> statement-breakpoint
ALTER TABLE "embedding" ADD COLUMN "tag7" text;--> statement-breakpoint
CREATE INDEX "doc_tag1_idx" ON "document" USING btree ("tag1");--> statement-breakpoint
CREATE INDEX "doc_tag2_idx" ON "document" USING btree ("tag2");--> statement-breakpoint
CREATE INDEX "doc_tag3_idx" ON "document" USING btree ("tag3");--> statement-breakpoint
CREATE INDEX "doc_tag4_idx" ON "document" USING btree ("tag4");--> statement-breakpoint
CREATE INDEX "doc_tag5_idx" ON "document" USING btree ("tag5");--> statement-breakpoint
CREATE INDEX "doc_tag6_idx" ON "document" USING btree ("tag6");--> statement-breakpoint
CREATE INDEX "doc_tag7_idx" ON "document" USING btree ("tag7");--> statement-breakpoint
CREATE INDEX "emb_tag1_idx" ON "embedding" USING btree ("tag1");--> statement-breakpoint
CREATE INDEX "emb_tag2_idx" ON "embedding" USING btree ("tag2");--> statement-breakpoint
CREATE INDEX "emb_tag3_idx" ON "embedding" USING btree ("tag3");--> statement-breakpoint
CREATE INDEX "emb_tag4_idx" ON "embedding" USING btree ("tag4");--> statement-breakpoint
CREATE INDEX "emb_tag5_idx" ON "embedding" USING btree ("tag5");--> statement-breakpoint
CREATE INDEX "emb_tag6_idx" ON "embedding" USING btree ("tag6");--> statement-breakpoint
CREATE INDEX "emb_tag7_idx" ON "embedding" USING btree ("tag7");--> statement-breakpoint
ALTER TABLE "embedding" DROP COLUMN "metadata";

File diff suppressed because it is too large Load Diff

View File

@@ -365,6 +365,13 @@
"when": 1752019053066,
"tag": "0052_fluffy_shinobi_shaw",
"breakpoints": true
},
{
"idx": 53,
"version": "7",
"when": 1752093722331,
"tag": "0053_gigantic_gabe_jones",
"breakpoints": true
}
]
}

View File

@@ -816,6 +816,15 @@ export const document = pgTable(
enabled: boolean('enabled').notNull().default(true), // Enable/disable from knowledge base
deletedAt: timestamp('deleted_at'), // Soft delete
// Document tags for filtering (inherited by all chunks)
tag1: text('tag1'),
tag2: text('tag2'),
tag3: text('tag3'),
tag4: text('tag4'),
tag5: text('tag5'),
tag6: text('tag6'),
tag7: text('tag7'),
// Timestamps
uploadedAt: timestamp('uploaded_at').notNull().defaultNow(),
},
@@ -831,6 +840,14 @@ export const document = pgTable(
table.knowledgeBaseId,
table.processingStatus
),
// Tag indexes for filtering
tag1Idx: index('doc_tag1_idx').on(table.tag1),
tag2Idx: index('doc_tag2_idx').on(table.tag2),
tag3Idx: index('doc_tag3_idx').on(table.tag3),
tag4Idx: index('doc_tag4_idx').on(table.tag4),
tag5Idx: index('doc_tag5_idx').on(table.tag5),
tag6Idx: index('doc_tag6_idx').on(table.tag6),
tag7Idx: index('doc_tag7_idx').on(table.tag7),
})
)
@@ -860,8 +877,14 @@ export const embedding = pgTable(
startOffset: integer('start_offset').notNull(),
endOffset: integer('end_offset').notNull(),
// Rich metadata for advanced filtering
metadata: jsonb('metadata').notNull().default('{}'),
// Tag columns inherited from document for efficient filtering
tag1: text('tag1'),
tag2: text('tag2'),
tag3: text('tag3'),
tag4: text('tag4'),
tag5: text('tag5'),
tag6: text('tag6'),
tag7: text('tag7'),
// Chunk state - enable/disable from knowledge base
enabled: boolean('enabled').notNull().default(true),
@@ -900,8 +923,14 @@ export const embedding = pgTable(
ef_construction: 64,
}),
// GIN index for JSONB metadata queries
metadataGinIdx: index('emb_metadata_gin_idx').using('gin', table.metadata),
// Tag indexes for efficient filtering
tag1Idx: index('emb_tag1_idx').on(table.tag1),
tag2Idx: index('emb_tag2_idx').on(table.tag2),
tag3Idx: index('emb_tag3_idx').on(table.tag3),
tag4Idx: index('emb_tag4_idx').on(table.tag4),
tag5Idx: index('emb_tag5_idx').on(table.tag5),
tag6Idx: index('emb_tag6_idx').on(table.tag6),
tag7Idx: index('emb_tag7_idx').on(table.tag7),
// Full-text search index
contentFtsIdx: index('emb_content_fts_idx').using('gin', table.contentTsv),

View File

@@ -44,6 +44,14 @@ export interface DocumentData {
processingError?: string | null
enabled: boolean
uploadedAt: string
// Document tags
tag1?: string | null
tag2?: string | null
tag3?: string | null
tag4?: string | null
tag5?: string | null
tag6?: string | null
tag7?: string | null
}
export interface ChunkData {
@@ -55,7 +63,13 @@ export interface ChunkData {
enabled: boolean
startOffset: number
endOffset: number
metadata: Record<string, unknown>
tag1?: string | null
tag2?: string | null
tag3?: string | null
tag4?: string | null
tag5?: string | null
tag6?: string | null
tag7?: string | null
createdAt: string
updatedAt: string
}

View File

@@ -22,6 +22,41 @@ export const knowledgeCreateDocumentTool: ToolConfig<any, KnowledgeCreateDocumen
required: true,
description: 'Content of the document',
},
tag1: {
type: 'string',
required: false,
description: 'Tag 1 value for the document',
},
tag2: {
type: 'string',
required: false,
description: 'Tag 2 value for the document',
},
tag3: {
type: 'string',
required: false,
description: 'Tag 3 value for the document',
},
tag4: {
type: 'string',
required: false,
description: 'Tag 4 value for the document',
},
tag5: {
type: 'string',
required: false,
description: 'Tag 5 value for the document',
},
tag6: {
type: 'string',
required: false,
description: 'Tag 6 value for the document',
},
tag7: {
type: 'string',
required: false,
description: 'Tag 7 value for the document',
},
},
request: {
url: (params) => `/api/knowledge/${params.knowledgeBaseId}/documents`,
@@ -65,6 +100,14 @@ export const knowledgeCreateDocumentTool: ToolConfig<any, KnowledgeCreateDocumen
fileUrl: dataUri,
fileSize: contentBytes,
mimeType: 'text/plain',
// Include tags if provided
tag1: params.tag1 || undefined,
tag2: params.tag2 || undefined,
tag3: params.tag3 || undefined,
tag4: params.tag4 || undefined,
tag5: params.tag5 || undefined,
tag6: params.tag6 || undefined,
tag7: params.tag7 || undefined,
},
]

View File

@@ -23,6 +23,41 @@ export const knowledgeSearchTool: ToolConfig<any, KnowledgeSearchResponse> = {
required: false,
description: 'Number of most similar results to return (1-100)',
},
tag1: {
type: 'string',
required: false,
description: 'Filter by tag 1 value',
},
tag2: {
type: 'string',
required: false,
description: 'Filter by tag 2 value',
},
tag3: {
type: 'string',
required: false,
description: 'Filter by tag 3 value',
},
tag4: {
type: 'string',
required: false,
description: 'Filter by tag 4 value',
},
tag5: {
type: 'string',
required: false,
description: 'Filter by tag 5 value',
},
tag6: {
type: 'string',
required: false,
description: 'Filter by tag 6 value',
},
tag7: {
type: 'string',
required: false,
description: 'Filter by tag 7 value',
},
},
request: {
url: () => '/api/knowledge/search',
@@ -43,12 +78,23 @@ export const knowledgeSearchTool: ToolConfig<any, KnowledgeSearchResponse> = {
.filter((id) => id.length > 0)
}
// Build filters object from tag parameters
const filters: Record<string, string> = {}
if (params.tag1) filters.tag1 = params.tag1.toString()
if (params.tag2) filters.tag2 = params.tag2.toString()
if (params.tag3) filters.tag3 = params.tag3.toString()
if (params.tag4) filters.tag4 = params.tag4.toString()
if (params.tag5) filters.tag5 = params.tag5.toString()
if (params.tag6) filters.tag6 = params.tag6.toString()
if (params.tag7) filters.tag7 = params.tag7.toString()
const requestBody = {
knowledgeBaseIds,
query: params.query,
topK: params.topK
? Math.max(1, Math.min(100, Number.parseInt(params.topK.toString()) || 10))
: 10,
...(Object.keys(filters).length > 0 && { filters }),
...(workflowId && { workflowId }),
}