feat(knowledge): connectors, user exclusions, expanded tools & airtable integration

packages/db/migrations/0155_talented_ben_parker.sql

@@ -0,0 +1,45 @@
+CREATE TABLE "knowledge_connector" (
+	"id" text PRIMARY KEY NOT NULL,
+	"knowledge_base_id" text NOT NULL,
+	"connector_type" text NOT NULL,
+	"credential_id" text NOT NULL,
+	"source_config" json NOT NULL,
+	"sync_mode" text DEFAULT 'incremental' NOT NULL,
+	"sync_interval_minutes" integer DEFAULT 1440 NOT NULL,
+	"status" text DEFAULT 'active' NOT NULL,
+	"last_sync_at" timestamp,
+	"last_sync_error" text,
+	"last_sync_doc_count" integer,
+	"next_sync_at" timestamp,
+	"consecutive_failures" integer DEFAULT 0 NOT NULL,
+	"created_at" timestamp DEFAULT now() NOT NULL,
+	"updated_at" timestamp DEFAULT now() NOT NULL,
+	"deleted_at" timestamp
+);
+--> statement-breakpoint
+CREATE TABLE "knowledge_connector_sync_log" (
+	"id" text PRIMARY KEY NOT NULL,
+	"connector_id" text NOT NULL,
+	"status" text NOT NULL,
+	"started_at" timestamp DEFAULT now() NOT NULL,
+	"completed_at" timestamp,
+	"docs_added" integer DEFAULT 0 NOT NULL,
+	"docs_updated" integer DEFAULT 0 NOT NULL,
+	"docs_deleted" integer DEFAULT 0 NOT NULL,
+	"docs_unchanged" integer DEFAULT 0 NOT NULL,
+	"error_message" text
+);
+--> statement-breakpoint
+ALTER TABLE "document" ADD COLUMN "user_excluded" boolean DEFAULT false NOT NULL;--> statement-breakpoint
+ALTER TABLE "document" ADD COLUMN "connector_id" text;--> statement-breakpoint
+ALTER TABLE "document" ADD COLUMN "external_id" text;--> statement-breakpoint
+ALTER TABLE "document" ADD COLUMN "content_hash" text;--> statement-breakpoint
+ALTER TABLE "document" ADD COLUMN "source_url" text;--> statement-breakpoint
+ALTER TABLE "knowledge_connector" ADD CONSTRAINT "knowledge_connector_knowledge_base_id_knowledge_base_id_fk" FOREIGN KEY ("knowledge_base_id") REFERENCES "public"."knowledge_base"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+ALTER TABLE "knowledge_connector_sync_log" ADD CONSTRAINT "knowledge_connector_sync_log_connector_id_knowledge_connector_id_fk" FOREIGN KEY ("connector_id") REFERENCES "public"."knowledge_connector"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+CREATE INDEX "kc_knowledge_base_id_idx" ON "knowledge_connector" USING btree ("knowledge_base_id");--> statement-breakpoint
+CREATE INDEX "kc_status_next_sync_idx" ON "knowledge_connector" USING btree ("status","next_sync_at");--> statement-breakpoint
+CREATE INDEX "kcsl_connector_id_idx" ON "knowledge_connector_sync_log" USING btree ("connector_id");--> statement-breakpoint
+ALTER TABLE "document" ADD CONSTRAINT "document_connector_id_knowledge_connector_id_fk" FOREIGN KEY ("connector_id") REFERENCES "public"."knowledge_connector"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint
+CREATE UNIQUE INDEX "doc_connector_external_id_idx" ON "document" USING btree ("connector_id","external_id") WHERE "document"."deleted_at" IS NULL;--> statement-breakpoint
+CREATE INDEX "doc_connector_id_idx" ON "document" USING btree ("connector_id");

packages/db/migrations/meta/_journal.json

@@ -1079,6 +1079,13 @@
       "when": 1770869658697,
       "tag": "0154_bumpy_living_mummy",
       "breakpoints": true
+    },
+    {
+      "idx": 155,
+      "version": "7",
+      "when": 1771305981173,
+      "tag": "0155_talented_ben_parker",
+      "breakpoints": true
     }
   ]
 }

packages/db/schema.ts

@@ -1215,6 +1215,7 @@ export const document = pgTable(
     // Document state
     enabled: boolean('enabled').notNull().default(true), // Enable/disable from knowledge base
     deletedAt: timestamp('deleted_at'), // Soft delete
+    userExcluded: boolean('user_excluded').notNull().default(false), // User explicitly excluded — skip on sync
 
     // Document tags for filtering (inherited by all chunks)
     // Text tags (7 slots)
@@ -1239,6 +1240,14 @@ export const document = pgTable(
     boolean2: boolean('boolean2'),
     boolean3: boolean('boolean3'),
 
+    // Connector-sourced document fields
+    connectorId: text('connector_id').references(() => knowledgeConnector.id, {
+      onDelete: 'set null',
+    }),
+    externalId: text('external_id'),
+    contentHash: text('content_hash'),
+    sourceUrl: text('source_url'),
+
     // Timestamps
     uploadedAt: timestamp('uploaded_at').notNull().defaultNow(),
   },
@@ -1252,6 +1261,12 @@ export const document = pgTable(
       table.knowledgeBaseId,
       table.processingStatus
     ),
+    // Connector document uniqueness (partial — only non-deleted rows)
+    connectorExternalIdIdx: uniqueIndex('doc_connector_external_id_idx')
+      .on(table.connectorId, table.externalId)
+      .where(sql`${table.deletedAt} IS NULL`),
+    // Sync engine: load all active docs for a connector
+    connectorIdIdx: index('doc_connector_id_idx').on(table.connectorId),
     // Text tag indexes
     tag1Idx: index('doc_tag1_idx').on(table.tag1),
     tag2Idx: index('doc_tag2_idx').on(table.tag2),
@@ -2240,3 +2255,60 @@ export const asyncJobs = pgTable(
     ),
   })
 )
+
+/**
+ * Knowledge Connector - persistent link to an external source (Confluence, Google Drive, etc.)
+ * that syncs documents into a knowledge base.
+ */
+export const knowledgeConnector = pgTable(
+  'knowledge_connector',
+  {
+    id: text('id').primaryKey(),
+    knowledgeBaseId: text('knowledge_base_id')
+      .notNull()
+      .references(() => knowledgeBase.id, { onDelete: 'cascade' }),
+    connectorType: text('connector_type').notNull(),
+    credentialId: text('credential_id').notNull(),
+    sourceConfig: json('source_config').notNull(),
+    syncMode: text('sync_mode').notNull().default('incremental'),
+    syncIntervalMinutes: integer('sync_interval_minutes').notNull().default(1440),
+    status: text('status').notNull().default('active'),
+    lastSyncAt: timestamp('last_sync_at'),
+    lastSyncError: text('last_sync_error'),
+    lastSyncDocCount: integer('last_sync_doc_count'),
+    nextSyncAt: timestamp('next_sync_at'),
+    consecutiveFailures: integer('consecutive_failures').notNull().default(0),
+    createdAt: timestamp('created_at').notNull().defaultNow(),
+    updatedAt: timestamp('updated_at').notNull().defaultNow(),
+    deletedAt: timestamp('deleted_at'),
+  },
+  (table) => ({
+    knowledgeBaseIdIdx: index('kc_knowledge_base_id_idx').on(table.knowledgeBaseId),
+    // Cron scheduler: WHERE status='active' AND nextSyncAt <= now AND deletedAt IS NULL
+    statusNextSyncIdx: index('kc_status_next_sync_idx').on(table.status, table.nextSyncAt),
+  })
+)
+
+/**
+ * Knowledge Connector Sync Log - audit trail for connector sync operations.
+ */
+export const knowledgeConnectorSyncLog = pgTable(
+  'knowledge_connector_sync_log',
+  {
+    id: text('id').primaryKey(),
+    connectorId: text('connector_id')
+      .notNull()
+      .references(() => knowledgeConnector.id, { onDelete: 'cascade' }),
+    status: text('status').notNull(),
+    startedAt: timestamp('started_at').notNull().defaultNow(),
+    completedAt: timestamp('completed_at'),
+    docsAdded: integer('docs_added').notNull().default(0),
+    docsUpdated: integer('docs_updated').notNull().default(0),
+    docsDeleted: integer('docs_deleted').notNull().default(0),
+    docsUnchanged: integer('docs_unchanged').notNull().default(0),
+    errorMessage: text('error_message'),
+  },
+  (table) => ({
+    connectorIdIdx: index('kcsl_connector_id_idx').on(table.connectorId),
+  })
+)