From 82b874c0271bb5bff186830f29a52d4b4673d6d1 Mon Sep 17 00:00:00 2001 From: waleed Date: Mon, 16 Feb 2026 22:43:12 -0800 Subject: [PATCH] improvements --- .claude/commands/add-connector.md | 21 ++++++ apps/sim/connectors/airtable/airtable.ts | 35 ++++++---- apps/sim/connectors/confluence/confluence.ts | 11 ++-- apps/sim/connectors/github/github.ts | 25 ++++--- .../connectors/google-drive/google-drive.ts | 41 +++++++----- apps/sim/connectors/jira/jira.ts | 21 +++--- apps/sim/connectors/linear/linear.ts | 3 +- apps/sim/connectors/notion/notion.ts | 65 +++++++++++-------- apps/sim/lib/knowledge/documents/service.ts | 8 ++- apps/sim/lib/knowledge/documents/utils.ts | 10 +++ 10 files changed, 158 insertions(+), 82 deletions(-) diff --git a/.claude/commands/add-connector.md b/.claude/commands/add-connector.md index a587d1642..3161078e6 100644 --- a/.claude/commands/add-connector.md +++ b/.claude/commands/add-connector.md @@ -28,6 +28,7 @@ connectors/{service}/ ```typescript import { createLogger } from '@sim/logger' import { {Service}Icon } from '@/components/icons' +import { fetchWithRetry } from '@/lib/knowledge/documents/utils' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' const logger = createLogger('{Service}Connector') @@ -179,6 +180,25 @@ mapTags: (metadata: Record): Record => { } ``` +## External API Calls — Use `fetchWithRetry` + +All external API calls must use `fetchWithRetry` from `@/lib/knowledge/documents/utils` instead of raw `fetch()`. This provides exponential backoff with retries on 429/502/503/504 errors. It returns a standard `Response` — all `.ok`, `.json()`, `.text()` checks work unchanged. + +For `validateConfig` (user-facing, called on save), pass `VALIDATE_RETRY_OPTIONS` to cap wait time at ~7s. Background operations (`listDocuments`, `getDocument`) use the built-in defaults (5 retries, ~31s max). + +```typescript +import { VALIDATE_RETRY_OPTIONS, fetchWithRetry } from '@/lib/knowledge/documents/utils' + +// Background sync — use defaults +const response = await fetchWithRetry(url, { + method: 'GET', + headers: { Authorization: `Bearer ${accessToken}` }, +}) + +// validateConfig — tighter retry budget +const response = await fetchWithRetry(url, { ... }, VALIDATE_RETRY_OPTIONS) +``` + ## sourceUrl If `ExternalDocument.sourceUrl` is set, the sync engine stores it on the document record. Always construct the full URL (not a relative path). @@ -235,6 +255,7 @@ See `apps/sim/connectors/confluence/confluence.ts` for a complete example with: - [ ] `tagDefinitions` declared for each semantic key returned by `mapTags` - [ ] `mapTags` implemented if source has useful metadata (labels, dates, versions) - [ ] `validateConfig` verifies the source is accessible +- [ ] All external API calls use `fetchWithRetry` (not raw `fetch`) - [ ] All optional config fields validated in `validateConfig` - [ ] Icon exists in `components/icons.tsx` (or asked user to provide SVG) - [ ] Registered in `connectors/registry.ts` diff --git a/apps/sim/connectors/airtable/airtable.ts b/apps/sim/connectors/airtable/airtable.ts index d707fef1d..b871830d0 100644 --- a/apps/sim/connectors/airtable/airtable.ts +++ b/apps/sim/connectors/airtable/airtable.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { AirtableIcon } from '@/components/icons' +import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' const logger = createLogger('AirtableConnector') @@ -162,7 +163,7 @@ export const airtableConnector: ConnectorConfig = { view: viewId ?? 'default', }) - const response = await fetch(url, { + const response = await fetchWithRetry(url, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}`, @@ -211,7 +212,7 @@ export const airtableConnector: ConnectorConfig = { const encodedTable = encodeURIComponent(tableIdOrName) const url = `${AIRTABLE_API}/${baseId}/${encodedTable}/${externalId}` - const response = await fetch(url, { + const response = await fetchWithRetry(url, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}`, @@ -251,12 +252,16 @@ export const airtableConnector: ConnectorConfig = { // Verify base and table are accessible by fetching 1 record const encodedTable = encodeURIComponent(tableIdOrName) const url = `${AIRTABLE_API}/${baseId}/${encodedTable}?pageSize=1` - const response = await fetch(url, { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, + const response = await fetchWithRetry( + url, + { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + }, }, - }) + VALIDATE_RETRY_OPTIONS + ) if (!response.ok) { const errorText = await response.text() @@ -273,12 +278,16 @@ export const airtableConnector: ConnectorConfig = { const viewId = sourceConfig.viewId as string | undefined if (viewId) { const viewUrl = `${AIRTABLE_API}/${baseId}/${encodedTable}?pageSize=1&view=${encodeURIComponent(viewId)}` - const viewResponse = await fetch(viewUrl, { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, + const viewResponse = await fetchWithRetry( + viewUrl, + { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + }, }, - }) + VALIDATE_RETRY_OPTIONS + ) if (!viewResponse.ok) { return { valid: false, error: `View "${viewId}" not found in table "${tableIdOrName}"` } } @@ -354,7 +363,7 @@ async function fetchFieldNames( try { const url = `${AIRTABLE_API}/meta/bases/${baseId}/tables` - const response = await fetch(url, { + const response = await fetchWithRetry(url, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}`, diff --git a/apps/sim/connectors/confluence/confluence.ts b/apps/sim/connectors/confluence/confluence.ts index e4ce62615..8d773abdf 100644 --- a/apps/sim/connectors/confluence/confluence.ts +++ b/apps/sim/connectors/confluence/confluence.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { ConfluenceIcon } from '@/components/icons' +import { fetchWithRetry } from '@/lib/knowledge/documents/utils' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { getConfluenceCloudId } from '@/tools/confluence/utils' @@ -45,7 +46,7 @@ async function fetchLabelsForPages( pageIds.map(async (pageId) => { try { const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/pages/${pageId}/labels` - const response = await fetch(url, { + const response = await fetchWithRetry(url, { method: 'GET', headers: { Accept: 'application/json', @@ -226,7 +227,7 @@ export const confluenceConnector: ConnectorConfig = { const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/pages/${externalId}?body-format=storage` - const response = await fetch(url, { + const response = await fetchWithRetry(url, { method: 'GET', headers: { Accept: 'application/json', @@ -342,7 +343,7 @@ async function listDocumentsV2( logger.info(`Listing ${endpoint} in space ${spaceKey} (ID: ${spaceId})`) - const response = await fetch(url, { + const response = await fetchWithRetry(url, { method: 'GET', headers: { Accept: 'application/json', @@ -543,7 +544,7 @@ async function listDocumentsViaCql( logger.info(`Searching Confluence via CQL: ${cql}`, { start, limit }) - const response = await fetch(url, { + const response = await fetchWithRetry(url, { method: 'GET', headers: { Accept: 'application/json', @@ -588,7 +589,7 @@ async function resolveSpaceId( ): Promise { const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/spaces?keys=${encodeURIComponent(spaceKey)}&limit=1` - const response = await fetch(url, { + const response = await fetchWithRetry(url, { method: 'GET', headers: { Accept: 'application/json', diff --git a/apps/sim/connectors/github/github.ts b/apps/sim/connectors/github/github.ts index 6f122a61b..27b69cf0f 100644 --- a/apps/sim/connectors/github/github.ts +++ b/apps/sim/connectors/github/github.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { GithubIcon } from '@/components/icons' +import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' const logger = createLogger('GitHubConnector') @@ -73,7 +74,7 @@ async function fetchTree( ): Promise { const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/git/trees/${encodeURIComponent(branch)}?recursive=1` - const response = await fetch(url, { + const response = await fetchWithRetry(url, { method: 'GET', headers: { Accept: 'application/vnd.github+json', @@ -108,7 +109,7 @@ async function fetchBlobContent( ): Promise { const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/git/blobs/${sha}` - const response = await fetch(url, { + const response = await fetchWithRetry(url, { method: 'GET', headers: { Accept: 'application/vnd.github+json', @@ -282,7 +283,7 @@ export const githubConnector: ConnectorConfig = { try { const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/contents/${encodeURIComponent(path)}?ref=${encodeURIComponent(branch)}` - const response = await fetch(url, { + const response = await fetchWithRetry(url, { method: 'GET', headers: { Accept: 'application/vnd.github+json', @@ -358,14 +359,18 @@ export const githubConnector: ConnectorConfig = { try { // Verify repo and branch are accessible const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/branches/${encodeURIComponent(branch)}` - const response = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/vnd.github+json', - Authorization: `Bearer ${accessToken}`, - 'X-GitHub-Api-Version': '2022-11-28', + const response = await fetchWithRetry( + url, + { + method: 'GET', + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${accessToken}`, + 'X-GitHub-Api-Version': '2022-11-28', + }, }, - }) + VALIDATE_RETRY_OPTIONS + ) if (response.status === 404) { return { diff --git a/apps/sim/connectors/google-drive/google-drive.ts b/apps/sim/connectors/google-drive/google-drive.ts index f339e0594..82dd9d1ee 100644 --- a/apps/sim/connectors/google-drive/google-drive.ts +++ b/apps/sim/connectors/google-drive/google-drive.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { GoogleDriveIcon } from '@/components/icons' +import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' const logger = createLogger('GoogleDriveConnector') @@ -61,7 +62,7 @@ async function exportGoogleWorkspaceFile( const url = `https://www.googleapis.com/drive/v3/files/${fileId}/export?mimeType=${encodeURIComponent(exportMimeType)}` - const response = await fetch(url, { + const response = await fetchWithRetry(url, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}` }, }) @@ -76,7 +77,7 @@ async function exportGoogleWorkspaceFile( async function downloadTextFile(accessToken: string, fileId: string): Promise { const url = `https://www.googleapis.com/drive/v3/files/${fileId}?alt=media` - const response = await fetch(url, { + const response = await fetchWithRetry(url, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}` }, }) @@ -266,7 +267,7 @@ export const googleDriveConnector: ConnectorConfig = { logger.info('Listing Google Drive files', { query, cursor: cursor ?? 'initial' }) - const response = await fetch(url, { + const response = await fetchWithRetry(url, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}`, @@ -310,7 +311,7 @@ export const googleDriveConnector: ConnectorConfig = { 'id,name,mimeType,modifiedTime,createdTime,webViewLink,parents,owners,size,starred,trashed' const url = `https://www.googleapis.com/drive/v3/files/${externalId}?fields=${encodeURIComponent(fields)}&supportsAllDrives=true` - const response = await fetch(url, { + const response = await fetchWithRetry(url, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}`, @@ -346,13 +347,17 @@ export const googleDriveConnector: ConnectorConfig = { if (folderId?.trim()) { // Verify the folder exists and is accessible const url = `https://www.googleapis.com/drive/v3/files/${folderId.trim()}?fields=id,name,mimeType&supportsAllDrives=true` - const response = await fetch(url, { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', + const response = await fetchWithRetry( + url, + { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, }, - }) + VALIDATE_RETRY_OPTIONS + ) if (!response.ok) { if (response.status === 404) { @@ -368,13 +373,17 @@ export const googleDriveConnector: ConnectorConfig = { } else { // Verify basic Drive access by listing one file const url = 'https://www.googleapis.com/drive/v3/files?pageSize=1&fields=files(id)' - const response = await fetch(url, { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', + const response = await fetchWithRetry( + url, + { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, }, - }) + VALIDATE_RETRY_OPTIONS + ) if (!response.ok) { return { valid: false, error: `Failed to access Google Drive: ${response.status}` } diff --git a/apps/sim/connectors/jira/jira.ts b/apps/sim/connectors/jira/jira.ts index 185cf9574..aaa464869 100644 --- a/apps/sim/connectors/jira/jira.ts +++ b/apps/sim/connectors/jira/jira.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { JiraIcon } from '@/components/icons' +import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { extractAdfText, getJiraCloudId } from '@/tools/jira/utils' @@ -159,7 +160,7 @@ export const jiraConnector: ConnectorConfig = { logger.info(`Listing Jira issues for project ${projectKey}`, { startAt }) - const response = await fetch(url, { + const response = await fetchWithRetry(url, { method: 'GET', headers: { Accept: 'application/json', @@ -210,7 +211,7 @@ export const jiraConnector: ConnectorConfig = { const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${externalId}?${params.toString()}` - const response = await fetch(url, { + const response = await fetchWithRetry(url, { method: 'GET', headers: { Accept: 'application/json', @@ -259,13 +260,17 @@ export const jiraConnector: ConnectorConfig = { params.append('maxResults', '0') const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/search?${params.toString()}` - const response = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, + const response = await fetchWithRetry( + url, + { + method: 'GET', + headers: { + Accept: 'application/json', + Authorization: `Bearer ${accessToken}`, + }, }, - }) + VALIDATE_RETRY_OPTIONS + ) if (!response.ok) { const errorText = await response.text() diff --git a/apps/sim/connectors/linear/linear.ts b/apps/sim/connectors/linear/linear.ts index b39c16977..99073352d 100644 --- a/apps/sim/connectors/linear/linear.ts +++ b/apps/sim/connectors/linear/linear.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { LinearIcon } from '@/components/icons' +import { fetchWithRetry } from '@/lib/knowledge/documents/utils' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' const logger = createLogger('LinearConnector') @@ -46,7 +47,7 @@ async function linearGraphQL( query: string, variables?: Record ): Promise> { - const response = await fetch(LINEAR_API, { + const response = await fetchWithRetry(LINEAR_API, { method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/apps/sim/connectors/notion/notion.ts b/apps/sim/connectors/notion/notion.ts index ac0799275..563453a5a 100644 --- a/apps/sim/connectors/notion/notion.ts +++ b/apps/sim/connectors/notion/notion.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { NotionIcon } from '@/components/icons' +import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' const logger = createLogger('NotionConnector') @@ -97,7 +98,7 @@ async function fetchAllBlocks( const params = new URLSearchParams({ page_size: '100' }) if (cursor) params.append('start_cursor', cursor) - const response = await fetch( + const response = await fetchWithRetry( `${NOTION_BASE_URL}/blocks/${pageId}/children?${params.toString()}`, { method: 'GET', @@ -262,7 +263,7 @@ export const notionConnector: ConnectorConfig = { _sourceConfig: Record, externalId: string ): Promise => { - const response = await fetch(`${NOTION_BASE_URL}/pages/${externalId}`, { + const response = await fetchWithRetry(`${NOTION_BASE_URL}/pages/${externalId}`, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}`, @@ -304,39 +305,51 @@ export const notionConnector: ConnectorConfig = { // Verify the token works if (scope === 'database' && databaseId) { // Verify database is accessible - const response = await fetch(`${NOTION_BASE_URL}/databases/${databaseId}`, { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - 'Notion-Version': NOTION_API_VERSION, + const response = await fetchWithRetry( + `${NOTION_BASE_URL}/databases/${databaseId}`, + { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + 'Notion-Version': NOTION_API_VERSION, + }, }, - }) + VALIDATE_RETRY_OPTIONS + ) if (!response.ok) { return { valid: false, error: `Cannot access database: ${response.status}` } } } else if (scope === 'page' && rootPageId) { // Verify page is accessible - const response = await fetch(`${NOTION_BASE_URL}/pages/${rootPageId}`, { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - 'Notion-Version': NOTION_API_VERSION, + const response = await fetchWithRetry( + `${NOTION_BASE_URL}/pages/${rootPageId}`, + { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + 'Notion-Version': NOTION_API_VERSION, + }, }, - }) + VALIDATE_RETRY_OPTIONS + ) if (!response.ok) { return { valid: false, error: `Cannot access page: ${response.status}` } } } else { // Workspace scope — just verify token works - const response = await fetch(`${NOTION_BASE_URL}/search`, { - method: 'POST', - headers: { - Authorization: `Bearer ${accessToken}`, - 'Notion-Version': NOTION_API_VERSION, - 'Content-Type': 'application/json', + const response = await fetchWithRetry( + `${NOTION_BASE_URL}/search`, + { + method: 'POST', + headers: { + Authorization: `Bearer ${accessToken}`, + 'Notion-Version': NOTION_API_VERSION, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ page_size: 1 }), }, - body: JSON.stringify({ page_size: 1 }), - }) + VALIDATE_RETRY_OPTIONS + ) if (!response.ok) { const errorText = await response.text() return { valid: false, error: `Cannot access Notion workspace: ${errorText}` } @@ -401,7 +414,7 @@ async function listFromWorkspace( logger.info('Listing Notion pages from workspace', { searchQuery, cursor }) - const response = await fetch(`${NOTION_BASE_URL}/search`, { + const response = await fetchWithRetry(`${NOTION_BASE_URL}/search`, { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, @@ -450,7 +463,7 @@ async function listFromDatabase( logger.info('Querying Notion database', { databaseId, cursor }) - const response = await fetch(`${NOTION_BASE_URL}/databases/${databaseId}/query`, { + const response = await fetchWithRetry(`${NOTION_BASE_URL}/databases/${databaseId}/query`, { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, @@ -497,7 +510,7 @@ async function listFromParentPage( logger.info('Listing child pages under root page', { rootPageId, cursor }) - const response = await fetch( + const response = await fetchWithRetry( `${NOTION_BASE_URL}/blocks/${rootPageId}/children?${params.toString()}`, { method: 'GET', @@ -531,7 +544,7 @@ async function listFromParentPage( if (maxPages > 0 && documents.length >= maxPages) break try { - const pageResponse = await fetch(`${NOTION_BASE_URL}/pages/${pageId}`, { + const pageResponse = await fetchWithRetry(`${NOTION_BASE_URL}/pages/${pageId}`, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}`, diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index 8af312487..af6cc5a1f 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -1795,22 +1795,24 @@ export async function deleteDocument( documentId: string, requestId: string ): Promise<{ success: boolean; message: string }> { - const doc = await db + const docs = await db .select({ connectorId: document.connectorId }) .from(document) .where(eq(document.id, documentId)) .limit(1) + const isConnectorDoc = docs.length > 0 && docs[0].connectorId !== null + await db .update(document) .set({ deletedAt: new Date(), - ...(doc[0]?.connectorId ? { userExcluded: true } : {}), + ...(isConnectorDoc ? { userExcluded: true } : {}), }) .where(eq(document.id, documentId)) logger.info(`[${requestId}] Document deleted: ${documentId}`, { - userExcluded: Boolean(doc[0]?.connectorId), + userExcluded: isConnectorDoc, }) return { diff --git a/apps/sim/lib/knowledge/documents/utils.ts b/apps/sim/lib/knowledge/documents/utils.ts index 7aae3187a..9c40f3439 100644 --- a/apps/sim/lib/knowledge/documents/utils.ts +++ b/apps/sim/lib/knowledge/documents/utils.ts @@ -129,6 +129,16 @@ export async function retryWithExponentialBackoff( throw lastError || new Error('Retry operation failed') } +/** + * Tighter retry options for user-facing operations (e.g. validateConfig). + * Caps total wait at ~7s instead of ~31s to avoid API route timeouts. + */ +export const VALIDATE_RETRY_OPTIONS: RetryOptions = { + maxRetries: 3, + initialDelayMs: 1000, + maxDelayMs: 10000, +} + /** * Wrapper for fetch requests with retry logic */