diff --git a/apps/sim/app/api/knowledge/[id]/connectors/route.ts b/apps/sim/app/api/knowledge/[id]/connectors/route.ts index 9bd505ed8..1a4be3d1d 100644 --- a/apps/sim/app/api/knowledge/[id]/connectors/route.ts +++ b/apps/sim/app/api/knowledge/[id]/connectors/route.ts @@ -117,11 +117,12 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{ } let finalSourceConfig: Record = sourceConfig + const tagSlotMapping: Record = {} + if (connectorConfig.tagDefinitions?.length) { const disabledIds = new Set((sourceConfig.disabledTagIds as string[] | undefined) ?? []) const enabledDefs = connectorConfig.tagDefinitions.filter((td) => !disabledIds.has(td.id)) - const tagSlotMapping: Record = {} const skippedTags: string[] = [] for (const td of enabledDefs) { const slot = await getNextAvailableSlot(knowledgeBaseId, td.fieldType) @@ -130,15 +131,6 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{ logger.warn(`[${requestId}] No available ${td.fieldType} slots for "${td.displayName}"`) continue } - await createTagDefinition( - { - knowledgeBaseId, - tagSlot: slot, - displayName: td.displayName, - fieldType: td.fieldType, - }, - requestId - ) tagSlotMapping[td.id] = slot } @@ -157,17 +149,33 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{ const nextSyncAt = syncIntervalMinutes > 0 ? new Date(now.getTime() + syncIntervalMinutes * 60 * 1000) : null - await db.insert(knowledgeConnector).values({ - id: connectorId, - knowledgeBaseId, - connectorType, - credentialId, - sourceConfig: finalSourceConfig, - syncIntervalMinutes, - status: 'active', - nextSyncAt, - createdAt: now, - updatedAt: now, + await db.transaction(async (tx) => { + for (const [semanticId, slot] of Object.entries(tagSlotMapping)) { + const td = connectorConfig.tagDefinitions!.find((d) => d.id === semanticId)! + await createTagDefinition( + { + knowledgeBaseId, + tagSlot: slot, + displayName: td.displayName, + fieldType: td.fieldType, + }, + requestId, + tx + ) + } + + await tx.insert(knowledgeConnector).values({ + id: connectorId, + knowledgeBaseId, + connectorType, + credentialId, + sourceConfig: finalSourceConfig, + syncIntervalMinutes, + status: 'active', + nextSyncAt, + createdAt: now, + updatedAt: now, + }) }) logger.info(`[${requestId}] Created connector ${connectorId} for KB ${knowledgeBaseId}`) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx index eda60181e..e02ff5018 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx @@ -745,6 +745,7 @@ export function Document({ setEnabledFilter('all') setIsFilterPopoverOpen(false) setSelectedChunks(new Set()) + goToPage(1) }} > All @@ -755,6 +756,7 @@ export function Document({ setEnabledFilter('enabled') setIsFilterPopoverOpen(false) setSelectedChunks(new Set()) + goToPage(1) }} > Enabled @@ -765,6 +767,7 @@ export function Document({ setEnabledFilter('disabled') setIsFilterPopoverOpen(false) setSelectedChunks(new Set()) + goToPage(1) }} > Disabled diff --git a/apps/sim/connectors/jira/jira.ts b/apps/sim/connectors/jira/jira.ts index 7d56b3922..a044e75d1 100644 --- a/apps/sim/connectors/jira/jira.ts +++ b/apps/sim/connectors/jira/jira.ts @@ -8,13 +8,6 @@ const logger = createLogger('JiraConnector') const PAGE_SIZE = 50 -/** - * Escapes a value for use inside JQL double-quoted strings. - */ -function escapeJql(value: string): string { - return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"') -} - /** * Computes a SHA-256 hash of the given content. */ @@ -147,9 +140,9 @@ export const jiraConnector: ConnectorConfig = { const cloudId = await getJiraCloudId(domain, accessToken) - let jql = `project = "${escapeJql(projectKey)}" ORDER BY updated DESC` + let jql = `project = ${projectKey} ORDER BY updated DESC` if (jqlFilter.trim()) { - jql = `project = "${escapeJql(projectKey)}" AND (${jqlFilter.trim()}) ORDER BY updated DESC` + jql = `project = ${projectKey} AND (${jqlFilter.trim()}) ORDER BY updated DESC` } const startAt = cursor ? Number(cursor) : 0 @@ -251,19 +244,13 @@ export const jiraConnector: ConnectorConfig = { return { valid: false, error: 'Max issues must be a positive number' } } - const jql = sourceConfig.jql as string | undefined - if (jql?.trim()) { - if (/\b(delete|drop|truncate|insert|update|alter|create|grant|revoke)\b/i.test(jql)) { - return { valid: false, error: 'Invalid JQL filter' } - } - } + const jqlFilter = (sourceConfig.jql as string | undefined)?.trim() || '' try { const cloudId = await getJiraCloudId(domain, accessToken) - // Verify the project exists by running a minimal search const params = new URLSearchParams() - params.append('jql', `project = "${escapeJql(projectKey)}"`) + params.append('jql', `project = ${projectKey}`) params.append('maxResults', '0') const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/search?${params.toString()}` @@ -287,6 +274,29 @@ export const jiraConnector: ConnectorConfig = { return { valid: false, error: `Failed to validate: ${response.status} - ${errorText}` } } + if (jqlFilter) { + const filterParams = new URLSearchParams() + filterParams.append('jql', `project = ${projectKey} AND (${jqlFilter})`) + filterParams.append('maxResults', '0') + + const filterUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/search?${filterParams.toString()}` + const filterResponse = await fetchWithRetry( + filterUrl, + { + method: 'GET', + headers: { + Accept: 'application/json', + Authorization: `Bearer ${accessToken}`, + }, + }, + VALIDATE_RETRY_OPTIONS + ) + + if (!filterResponse.ok) { + return { valid: false, error: 'Invalid JQL filter. Check syntax and field names.' } + } + } + return { valid: true } } catch (error) { const message = error instanceof Error ? error.message : 'Failed to validate configuration' diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.ts b/apps/sim/lib/knowledge/connectors/sync-engine.ts index 4d6bed2eb..d5bd7e6aa 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.ts @@ -275,7 +275,7 @@ export async function executeSync( } } - if (options?.fullSync || connector.syncMode === 'incremental') { + if (options?.fullSync || connector.syncMode === 'full') { for (const existing of existingDocs) { if (existing.externalId && !seenExternalIds.has(existing.externalId)) { await db diff --git a/apps/sim/lib/knowledge/tags/service.ts b/apps/sim/lib/knowledge/tags/service.ts index 96959e6d2..c2c45fa76 100644 --- a/apps/sim/lib/knowledge/tags/service.ts +++ b/apps/sim/lib/knowledge/tags/service.ts @@ -3,6 +3,7 @@ import { db } from '@sim/db' import { document, embedding, knowledgeBaseTagDefinitions } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { and, eq, isNotNull, isNull, sql } from 'drizzle-orm' +import type { DbOrTx } from '@/lib/db/types' import { getSlotsForFieldType, SUPPORTED_FIELD_TYPES } from '@/lib/knowledge/constants' import type { BulkTagDefinitionsData, DocumentTagDefinition } from '@/lib/knowledge/tags/types' import type { @@ -485,8 +486,10 @@ export async function deleteTagDefinition( */ export async function createTagDefinition( data: CreateTagDefinitionData, - requestId: string + requestId: string, + txDb?: DbOrTx ): Promise { + const dbInstance = txDb ?? db const tagDefinitionId = randomUUID() const now = new Date() @@ -500,7 +503,7 @@ export async function createTagDefinition( updatedAt: now, } - await db.insert(knowledgeBaseTagDefinitions).values(newDefinition) + await dbInstance.insert(knowledgeBaseTagDefinitions).values(newDefinition) logger.info( `[${requestId}] Created tag definition: ${data.displayName} -> ${data.tagSlot} in KB ${data.knowledgeBaseId}` diff --git a/packages/db/migrations/0155_talented_ben_parker.sql b/packages/db/migrations/0155_bitter_maginty.sql similarity index 98% rename from packages/db/migrations/0155_talented_ben_parker.sql rename to packages/db/migrations/0155_bitter_maginty.sql index 9ed3cf740..02dad1ce4 100644 --- a/packages/db/migrations/0155_talented_ben_parker.sql +++ b/packages/db/migrations/0155_bitter_maginty.sql @@ -4,7 +4,7 @@ CREATE TABLE "knowledge_connector" ( "connector_type" text NOT NULL, "credential_id" text NOT NULL, "source_config" json NOT NULL, - "sync_mode" text DEFAULT 'incremental' NOT NULL, + "sync_mode" text DEFAULT 'full' NOT NULL, "sync_interval_minutes" integer DEFAULT 1440 NOT NULL, "status" text DEFAULT 'active' NOT NULL, "last_sync_at" timestamp, diff --git a/packages/db/migrations/meta/0155_snapshot.json b/packages/db/migrations/meta/0155_snapshot.json index 05614038a..f49f2ac6d 100644 --- a/packages/db/migrations/meta/0155_snapshot.json +++ b/packages/db/migrations/meta/0155_snapshot.json @@ -1,5 +1,5 @@ { - "id": "4209f841-3ebf-469f-ad98-2df8a84766a1", + "id": "0c80d49c-beb2-4792-a9c1-91ce6ef7eb11", "prevId": "49f580f7-7eba-4431-bdf4-61db0e606546", "version": "7", "dialect": "postgresql", @@ -4292,7 +4292,7 @@ "type": "text", "primaryKey": false, "notNull": true, - "default": "'incremental'" + "default": "'full'" }, "sync_interval_minutes": { "name": "sync_interval_minutes", diff --git a/packages/db/migrations/meta/_journal.json b/packages/db/migrations/meta/_journal.json index d4a736939..b07bd7b2c 100644 --- a/packages/db/migrations/meta/_journal.json +++ b/packages/db/migrations/meta/_journal.json @@ -1083,8 +1083,8 @@ { "idx": 155, "version": "7", - "when": 1771305981173, - "tag": "0155_talented_ben_parker", + "when": 1771314071508, + "tag": "0155_bitter_maginty", "breakpoints": true } ] diff --git a/packages/db/schema.ts b/packages/db/schema.ts index 03cd051eb..d68e27e25 100644 --- a/packages/db/schema.ts +++ b/packages/db/schema.ts @@ -2270,7 +2270,7 @@ export const knowledgeConnector = pgTable( connectorType: text('connector_type').notNull(), credentialId: text('credential_id').notNull(), sourceConfig: json('source_config').notNull(), - syncMode: text('sync_mode').notNull().default('incremental'), + syncMode: text('sync_mode').notNull().default('full'), syncIntervalMinutes: integer('sync_interval_minutes').notNull().default(1440), status: text('status').notNull().default('active'), lastSyncAt: timestamp('last_sync_at'),