feat(knowledge): add connector tools and expand document metadata (#3452)

* feat(knowledge): add connector tools and expand document metadata

* fix(knowledge): address PR review feedback on new tools

* fix(knowledge): remove unused params from get_document transform
This commit is contained in:
Waleed
2026-03-06 17:58:33 -08:00
committed by GitHub
parent b93c87c521
commit e6ca3b3311
10 changed files with 586 additions and 2 deletions

View File

@@ -58,6 +58,8 @@ export interface DocumentData {
boolean3?: boolean | null
// Connector fields
connectorId?: string | null
sourceUrl?: string | null
externalId?: string | null
}
export interface EmbeddingData {
@@ -287,6 +289,8 @@ export async function checkDocumentWriteAccess(
boolean3: document.boolean3,
// Connector fields
connectorId: document.connectorId,
sourceUrl: document.sourceUrl,
externalId: document.externalId,
})
.from(document)
.where(and(eq(document.id, documentId), isNull(document.deletedAt)))

View File

@@ -10,7 +10,10 @@ export const KnowledgeBlock: BlockConfig = {
bestPractices: `
- Clarify which tags are available for the knowledge base to understand whether to use tag filters on a search.
- Use List Documents to enumerate documents before operating on them.
- Use Get Document to retrieve full details including tags, connector metadata, and processing status.
- Use List Chunks to inspect a document's contents before updating or deleting chunks.
- Use List Connectors to see which external sources are syncing documents into the knowledge base.
- Use Get Connector to check sync health and review recent sync logs.
`,
bgColor: '#00B0B0',
icon: PackageSearchIcon,
@@ -24,6 +27,7 @@ export const KnowledgeBlock: BlockConfig = {
options: [
{ label: 'Search', id: 'search' },
{ label: 'List Documents', id: 'list_documents' },
{ label: 'Get Document', id: 'get_document' },
{ label: 'Create Document', id: 'create_document' },
{ label: 'Delete Document', id: 'delete_document' },
{ label: 'List Chunks', id: 'list_chunks' },
@@ -31,6 +35,9 @@ export const KnowledgeBlock: BlockConfig = {
{ label: 'Update Chunk', id: 'update_chunk' },
{ label: 'Delete Chunk', id: 'delete_chunk' },
{ label: 'List Tags', id: 'list_tags' },
{ label: 'List Connectors', id: 'list_connectors' },
{ label: 'Get Connector', id: 'get_connector' },
{ label: 'Trigger Sync', id: 'trigger_sync' },
],
value: () => 'search',
},
@@ -125,7 +132,14 @@ export const KnowledgeBlock: BlockConfig = {
mode: 'basic',
condition: {
field: 'operation',
value: ['upload_chunk', 'delete_document', 'list_chunks', 'update_chunk', 'delete_chunk'],
value: [
'get_document',
'upload_chunk',
'delete_document',
'list_chunks',
'update_chunk',
'delete_chunk',
],
},
},
// Document selector — advanced mode (manual ID input)
@@ -139,7 +153,14 @@ export const KnowledgeBlock: BlockConfig = {
mode: 'advanced',
condition: {
field: 'operation',
value: ['upload_chunk', 'delete_document', 'list_chunks', 'update_chunk', 'delete_chunk'],
value: [
'get_document',
'upload_chunk',
'delete_document',
'list_chunks',
'update_chunk',
'delete_chunk',
],
},
},
@@ -208,6 +229,16 @@ export const KnowledgeBlock: BlockConfig = {
condition: { field: 'operation', value: 'update_chunk' },
},
// --- Connector operations ---
{
id: 'connectorId',
title: 'Connector ID',
type: 'short-input',
placeholder: 'Enter connector ID',
required: true,
condition: { field: 'operation', value: ['get_connector', 'trigger_sync'] },
},
// --- List Chunks ---
{
id: 'chunkSearch',
@@ -235,10 +266,14 @@ export const KnowledgeBlock: BlockConfig = {
'knowledge_create_document',
'knowledge_list_tags',
'knowledge_list_documents',
'knowledge_get_document',
'knowledge_delete_document',
'knowledge_list_chunks',
'knowledge_update_chunk',
'knowledge_delete_chunk',
'knowledge_list_connectors',
'knowledge_get_connector',
'knowledge_trigger_sync',
],
config: {
tool: (params) => {
@@ -253,6 +288,8 @@ export const KnowledgeBlock: BlockConfig = {
return 'knowledge_list_tags'
case 'list_documents':
return 'knowledge_list_documents'
case 'get_document':
return 'knowledge_get_document'
case 'delete_document':
return 'knowledge_delete_document'
case 'list_chunks':
@@ -261,6 +298,12 @@ export const KnowledgeBlock: BlockConfig = {
return 'knowledge_update_chunk'
case 'delete_chunk':
return 'knowledge_delete_chunk'
case 'list_connectors':
return 'knowledge_list_connectors'
case 'get_connector':
return 'knowledge_get_connector'
case 'trigger_sync':
return 'knowledge_trigger_sync'
default:
return 'knowledge_search'
}
@@ -273,6 +316,7 @@ export const KnowledgeBlock: BlockConfig = {
params.knowledgeBaseId = knowledgeBaseId
const docOps = [
'get_document',
'upload_chunk',
'delete_document',
'list_chunks',
@@ -296,6 +340,15 @@ export const KnowledgeBlock: BlockConfig = {
params.chunkId = chunkId
}
const connectorOps = ['get_connector', 'trigger_sync']
if (connectorOps.includes(params.operation)) {
const connectorId = params.connectorId ? String(params.connectorId).trim() : ''
if (!connectorId) {
throw new Error(`Connector ID is required for ${params.operation} operation`)
}
params.connectorId = connectorId
}
// Map list_chunks sub-block fields to tool params
if (params.operation === 'list_chunks') {
if (params.chunkSearch) params.search = params.chunkSearch
@@ -329,6 +382,7 @@ export const KnowledgeBlock: BlockConfig = {
documentTags: { type: 'string', description: 'Document tags' },
chunkSearch: { type: 'string', description: 'Search filter for chunks' },
chunkEnabledFilter: { type: 'string', description: 'Filter chunks by enabled status' },
connectorId: { type: 'string', description: 'Connector identifier' },
},
outputs: {
results: { type: 'json', description: 'Search results' },

View File

@@ -0,0 +1,119 @@
import type { KnowledgeGetConnectorResponse } from '@/tools/knowledge/types'
import type { ToolConfig } from '@/tools/types'
export const knowledgeGetConnectorTool: ToolConfig<any, KnowledgeGetConnectorResponse> = {
id: 'knowledge_get_connector',
name: 'Knowledge Get Connector',
description:
'Get detailed connector information including recent sync logs for monitoring sync health',
version: '1.0.0',
params: {
knowledgeBaseId: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description: 'ID of the knowledge base the connector belongs to',
},
connectorId: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description: 'ID of the connector to retrieve',
},
},
request: {
url: (params) =>
`/api/knowledge/${params.knowledgeBaseId}/connectors/${params.connectorId}`,
method: 'GET',
headers: () => ({
'Content-Type': 'application/json',
}),
},
transformResponse: async (response): Promise<KnowledgeGetConnectorResponse> => {
const result = await response.json()
const data = result.data || {}
return {
success: result.success ?? true,
output: {
connector: {
id: data.id,
connectorType: data.connectorType,
status: data.status,
syncIntervalMinutes: data.syncIntervalMinutes,
lastSyncAt: data.lastSyncAt ?? null,
lastSyncError: data.lastSyncError ?? null,
lastSyncDocCount: data.lastSyncDocCount ?? null,
nextSyncAt: data.nextSyncAt ?? null,
consecutiveFailures: data.consecutiveFailures ?? 0,
createdAt: data.createdAt ?? null,
updatedAt: data.updatedAt ?? null,
},
syncLogs: (data.syncLogs || []).map(
(log: {
id: string
status: string
startedAt: string | null
completedAt: string | null
docsAdded: number | null
docsUpdated: number | null
docsDeleted: number | null
docsUnchanged: number | null
errorMessage: string | null
}) => ({
id: log.id,
status: log.status,
startedAt: log.startedAt ?? null,
completedAt: log.completedAt ?? null,
docsAdded: log.docsAdded ?? null,
docsUpdated: log.docsUpdated ?? null,
docsDeleted: log.docsDeleted ?? null,
docsUnchanged: log.docsUnchanged ?? null,
errorMessage: log.errorMessage ?? null,
})
),
},
}
},
outputs: {
connector: {
type: 'object',
description: 'Connector details',
properties: {
id: { type: 'string', description: 'Connector ID' },
connectorType: { type: 'string', description: 'Type of connector' },
status: { type: 'string', description: 'Connector status (active, paused, syncing)' },
syncIntervalMinutes: { type: 'number', description: 'Sync interval in minutes' },
lastSyncAt: { type: 'string', description: 'Timestamp of last sync' },
lastSyncError: { type: 'string', description: 'Error from last sync if failed' },
lastSyncDocCount: { type: 'number', description: 'Docs synced in last sync' },
nextSyncAt: { type: 'string', description: 'Next scheduled sync timestamp' },
consecutiveFailures: { type: 'number', description: 'Consecutive sync failures' },
createdAt: { type: 'string', description: 'Creation timestamp' },
updatedAt: { type: 'string', description: 'Last update timestamp' },
},
},
syncLogs: {
type: 'array',
description: 'Recent sync log entries',
items: {
type: 'object',
properties: {
id: { type: 'string', description: 'Sync log ID' },
status: { type: 'string', description: 'Sync status' },
startedAt: { type: 'string', description: 'Sync start time' },
completedAt: { type: 'string', description: 'Sync completion time' },
docsAdded: { type: 'number', description: 'Documents added' },
docsUpdated: { type: 'number', description: 'Documents updated' },
docsDeleted: { type: 'number', description: 'Documents deleted' },
docsUnchanged: { type: 'number', description: 'Documents unchanged' },
errorMessage: { type: 'string', description: 'Error message if sync failed' },
},
},
},
},
}

View File

@@ -0,0 +1,124 @@
import type { KnowledgeGetDocumentResponse } from '@/tools/knowledge/types'
import type { ToolConfig } from '@/tools/types'
export const knowledgeGetDocumentTool: ToolConfig<any, KnowledgeGetDocumentResponse> = {
id: 'knowledge_get_document',
name: 'Knowledge Get Document',
description:
'Get full details of a single document including tags, connector metadata, and processing status',
version: '1.0.0',
params: {
knowledgeBaseId: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description: 'ID of the knowledge base the document belongs to',
},
documentId: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description: 'ID of the document to retrieve',
},
},
request: {
url: (params) =>
`/api/knowledge/${params.knowledgeBaseId}/documents/${params.documentId}`,
method: 'GET',
headers: () => ({
'Content-Type': 'application/json',
}),
},
transformResponse: async (response): Promise<KnowledgeGetDocumentResponse> => {
const result = await response.json()
const doc = result.data || {}
const tagSlots = [
'tag1',
'tag2',
'tag3',
'tag4',
'tag5',
'tag6',
'tag7',
'number1',
'number2',
'number3',
'number4',
'number5',
'date1',
'date2',
'boolean1',
'boolean2',
'boolean3',
]
const tags: Record<string, unknown> = {}
for (const slot of tagSlots) {
if (doc[slot] !== null && doc[slot] !== undefined) {
tags[slot] = doc[slot]
}
}
return {
success: result.success ?? true,
output: {
id: doc.id,
filename: doc.filename,
fileSize: doc.fileSize ?? 0,
mimeType: doc.mimeType ?? null,
enabled: doc.enabled ?? true,
processingStatus: doc.processingStatus ?? null,
processingError: doc.processingError ?? null,
chunkCount: doc.chunkCount ?? 0,
tokenCount: doc.tokenCount ?? 0,
characterCount: doc.characterCount ?? 0,
uploadedAt: doc.uploadedAt ?? null,
updatedAt: doc.updatedAt ?? null,
connectorId: doc.connectorId ?? null,
sourceUrl: doc.sourceUrl ?? null,
externalId: doc.externalId ?? null,
tags,
},
}
},
outputs: {
id: { type: 'string', description: 'Document ID' },
filename: { type: 'string', description: 'Document filename' },
fileSize: { type: 'number', description: 'File size in bytes' },
mimeType: { type: 'string', description: 'MIME type of the document' },
enabled: { type: 'boolean', description: 'Whether the document is enabled' },
processingStatus: {
type: 'string',
description: 'Processing status (pending, processing, completed, failed)',
},
processingError: {
type: 'string',
description: 'Error message if processing failed',
},
chunkCount: { type: 'number', description: 'Number of chunks in the document' },
tokenCount: { type: 'number', description: 'Total token count across chunks' },
characterCount: { type: 'number', description: 'Total character count' },
uploadedAt: { type: 'string', description: 'Upload timestamp' },
updatedAt: { type: 'string', description: 'Last update timestamp' },
connectorId: {
type: 'string',
description: 'Connector ID if document was synced from an external source',
},
sourceUrl: {
type: 'string',
description: 'Original URL in the source system if synced from a connector',
},
externalId: {
type: 'string',
description: 'External ID from the source system',
},
tags: {
type: 'object',
description: 'Tag values keyed by tag slot (tag1-7, number1-5, date1-2, boolean1-3)',
},
},
}

View File

@@ -1,10 +1,14 @@
import { knowledgeCreateDocumentTool } from '@/tools/knowledge/create_document'
import { knowledgeDeleteChunkTool } from '@/tools/knowledge/delete_chunk'
import { knowledgeDeleteDocumentTool } from '@/tools/knowledge/delete_document'
import { knowledgeGetConnectorTool } from '@/tools/knowledge/get_connector'
import { knowledgeGetDocumentTool } from '@/tools/knowledge/get_document'
import { knowledgeListChunksTool } from '@/tools/knowledge/list_chunks'
import { knowledgeListConnectorsTool } from '@/tools/knowledge/list_connectors'
import { knowledgeListDocumentsTool } from '@/tools/knowledge/list_documents'
import { knowledgeListTagsTool } from '@/tools/knowledge/list_tags'
import { knowledgeSearchTool } from '@/tools/knowledge/search'
import { knowledgeTriggerSyncTool } from '@/tools/knowledge/trigger_sync'
import { knowledgeUpdateChunkTool } from '@/tools/knowledge/update_chunk'
import { knowledgeUploadChunkTool } from '@/tools/knowledge/upload_chunk'
@@ -15,7 +19,11 @@ export {
knowledgeListTagsTool,
knowledgeListDocumentsTool,
knowledgeDeleteDocumentTool,
knowledgeGetDocumentTool,
knowledgeListChunksTool,
knowledgeUpdateChunkTool,
knowledgeDeleteChunkTool,
knowledgeListConnectorsTool,
knowledgeGetConnectorTool,
knowledgeTriggerSyncTool,
}

View File

@@ -0,0 +1,113 @@
import type { KnowledgeListConnectorsResponse } from '@/tools/knowledge/types'
import type { ToolConfig } from '@/tools/types'
export const knowledgeListConnectorsTool: ToolConfig<any, KnowledgeListConnectorsResponse> = {
id: 'knowledge_list_connectors',
name: 'Knowledge List Connectors',
description:
'List all connectors for a knowledge base, showing sync status, type, and document counts',
version: '1.0.0',
params: {
knowledgeBaseId: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description: 'ID of the knowledge base to list connectors for',
},
},
request: {
url: (params) => `/api/knowledge/${params.knowledgeBaseId}/connectors`,
method: 'GET',
headers: () => ({
'Content-Type': 'application/json',
}),
},
transformResponse: async (response, params): Promise<KnowledgeListConnectorsResponse> => {
const result = await response.json()
const connectors = result.data || []
return {
success: result.success ?? true,
output: {
knowledgeBaseId: params?.knowledgeBaseId ?? '',
connectors: connectors.map(
(c: {
id: string
connectorType: string
status: string
syncIntervalMinutes: number
lastSyncAt: string | null
lastSyncError: string | null
lastSyncDocCount: number | null
nextSyncAt: string | null
consecutiveFailures: number
createdAt: string | null
updatedAt: string | null
}) => ({
id: c.id,
connectorType: c.connectorType,
status: c.status,
syncIntervalMinutes: c.syncIntervalMinutes,
lastSyncAt: c.lastSyncAt ?? null,
lastSyncError: c.lastSyncError ?? null,
lastSyncDocCount: c.lastSyncDocCount ?? null,
nextSyncAt: c.nextSyncAt ?? null,
consecutiveFailures: c.consecutiveFailures ?? 0,
createdAt: c.createdAt ?? null,
updatedAt: c.updatedAt ?? null,
})
),
totalConnectors: connectors.length,
},
}
},
outputs: {
knowledgeBaseId: {
type: 'string',
description: 'ID of the knowledge base',
},
connectors: {
type: 'array',
description: 'Array of connectors for the knowledge base',
items: {
type: 'object',
properties: {
id: { type: 'string', description: 'Connector ID' },
connectorType: {
type: 'string',
description: 'Type of connector (e.g. notion, github, confluence)',
},
status: {
type: 'string',
description: 'Connector status (active, paused, syncing)',
},
syncIntervalMinutes: {
type: 'number',
description: 'Sync interval in minutes (0 = manual only)',
},
lastSyncAt: { type: 'string', description: 'Timestamp of last sync' },
lastSyncError: { type: 'string', description: 'Error from last sync if failed' },
lastSyncDocCount: {
type: 'number',
description: 'Number of documents synced in last sync',
},
nextSyncAt: { type: 'string', description: 'Timestamp of next scheduled sync' },
consecutiveFailures: {
type: 'number',
description: 'Number of consecutive sync failures',
},
createdAt: { type: 'string', description: 'Creation timestamp' },
updatedAt: { type: 'string', description: 'Last update timestamp' },
},
},
},
totalConnectors: {
type: 'number',
description: 'Total number of connectors',
},
},
}

View File

@@ -78,6 +78,9 @@ export const knowledgeListDocumentsTool: ToolConfig<any, KnowledgeListDocumentsR
tokenCount: number
uploadedAt: string
updatedAt: string
connectorId: string | null
connectorType: string | null
sourceUrl: string | null
}) => ({
id: doc.id,
filename: doc.filename,
@@ -89,6 +92,9 @@ export const knowledgeListDocumentsTool: ToolConfig<any, KnowledgeListDocumentsR
tokenCount: doc.tokenCount ?? 0,
uploadedAt: doc.uploadedAt ?? null,
updatedAt: doc.updatedAt ?? null,
connectorId: doc.connectorId ?? null,
connectorType: doc.connectorType ?? null,
sourceUrl: doc.sourceUrl ?? null,
})
),
totalDocuments: pagination.total ?? documents.length,
@@ -122,6 +128,18 @@ export const knowledgeListDocumentsTool: ToolConfig<any, KnowledgeListDocumentsR
tokenCount: { type: 'number', description: 'Total token count across chunks' },
uploadedAt: { type: 'string', description: 'Upload timestamp' },
updatedAt: { type: 'string', description: 'Last update timestamp' },
connectorId: {
type: 'string',
description: 'Connector ID if document was synced from an external source',
},
connectorType: {
type: 'string',
description: 'Connector type (e.g. notion, github, confluence) if synced',
},
sourceUrl: {
type: 'string',
description: 'Original URL in the source system if synced from a connector',
},
},
},
},

View File

@@ -0,0 +1,56 @@
import type { KnowledgeTriggerSyncResponse } from '@/tools/knowledge/types'
import type { ToolConfig } from '@/tools/types'
export const knowledgeTriggerSyncTool: ToolConfig<any, KnowledgeTriggerSyncResponse> = {
id: 'knowledge_trigger_sync',
name: 'Knowledge Trigger Sync',
description: 'Trigger a manual sync for a knowledge base connector',
version: '1.0.0',
params: {
knowledgeBaseId: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description: 'ID of the knowledge base the connector belongs to',
},
connectorId: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description: 'ID of the connector to trigger sync for',
},
},
request: {
url: (params) =>
`/api/knowledge/${params.knowledgeBaseId}/connectors/${params.connectorId}/sync`,
method: 'POST',
headers: () => ({
'Content-Type': 'application/json',
}),
},
transformResponse: async (response, params): Promise<KnowledgeTriggerSyncResponse> => {
const result = await response.json()
return {
success: result.success ?? true,
output: {
connectorId: params?.connectorId ?? '',
message: result.message ?? 'Sync triggered',
},
}
},
outputs: {
connectorId: {
type: 'string',
description: 'ID of the connector that was synced',
},
message: {
type: 'string',
description: 'Status message from the sync trigger',
},
},
}

View File

@@ -135,6 +135,9 @@ export interface KnowledgeDocumentSummary {
tokenCount: number
uploadedAt: string | null
updatedAt: string | null
connectorId: string | null
connectorType: string | null
sourceUrl: string | null
}
export interface KnowledgeListDocumentsResponse {
@@ -206,3 +209,80 @@ export interface KnowledgeDeleteChunkResponse {
}
error?: string
}
export interface KnowledgeGetDocumentResponse {
success: boolean
output: {
id: string
filename: string
fileSize: number
mimeType: string | null
enabled: boolean
processingStatus: string | null
processingError: string | null
chunkCount: number
tokenCount: number
characterCount: number
uploadedAt: string | null
updatedAt: string | null
connectorId: string | null
sourceUrl: string | null
externalId: string | null
tags: Record<string, unknown>
}
error?: string
}
export interface KnowledgeConnectorSummary {
id: string
connectorType: string
status: string
syncIntervalMinutes: number
lastSyncAt: string | null
lastSyncError: string | null
lastSyncDocCount: number | null
nextSyncAt: string | null
consecutiveFailures: number
createdAt: string | null
updatedAt: string | null
}
export interface KnowledgeListConnectorsResponse {
success: boolean
output: {
knowledgeBaseId: string
connectors: KnowledgeConnectorSummary[]
totalConnectors: number
}
error?: string
}
export interface KnowledgeSyncLogEntry {
id: string
status: string
startedAt: string | null
completedAt: string | null
docsAdded: number | null
docsUpdated: number | null
docsDeleted: number | null
docsUnchanged: number | null
errorMessage: string | null
}
export interface KnowledgeGetConnectorResponse {
success: boolean
output: {
connector: KnowledgeConnectorSummary
syncLogs: KnowledgeSyncLogEntry[]
}
error?: string
}
export interface KnowledgeTriggerSyncResponse {
success: boolean
output: {
connectorId: string
message: string
}
error?: string
}

View File

@@ -1155,10 +1155,14 @@ import {
knowledgeCreateDocumentTool,
knowledgeDeleteChunkTool,
knowledgeDeleteDocumentTool,
knowledgeGetConnectorTool,
knowledgeGetDocumentTool,
knowledgeListChunksTool,
knowledgeListConnectorsTool,
knowledgeListDocumentsTool,
knowledgeListTagsTool,
knowledgeSearchTool,
knowledgeTriggerSyncTool,
knowledgeUpdateChunkTool,
knowledgeUploadChunkTool,
} from '@/tools/knowledge'
@@ -3598,10 +3602,14 @@ export const tools: Record<string, ToolConfig> = {
knowledge_create_document: knowledgeCreateDocumentTool,
knowledge_list_tags: knowledgeListTagsTool,
knowledge_list_documents: knowledgeListDocumentsTool,
knowledge_get_document: knowledgeGetDocumentTool,
knowledge_delete_document: knowledgeDeleteDocumentTool,
knowledge_list_chunks: knowledgeListChunksTool,
knowledge_update_chunk: knowledgeUpdateChunkTool,
knowledge_delete_chunk: knowledgeDeleteChunkTool,
knowledge_list_connectors: knowledgeListConnectorsTool,
knowledge_get_connector: knowledgeGetConnectorTool,
knowledge_trigger_sync: knowledgeTriggerSyncTool,
search_tool: searchTool,
elevenlabs_tts: elevenLabsTtsTool,
stt_whisper: whisperSttTool,