improvements

2026-02-17 18:02:09 -05:00 · 2026-02-16 22:43:12 -08:00
parent 3eaf5babf4
commit 82b874c027
10 changed files with 158 additions and 82 deletions
--- a/apps/sim/connectors/airtable/airtable.ts
+++ b/apps/sim/connectors/airtable/airtable.ts
@@ -1,5 +1,6 @@
 import { createLogger } from '@sim/logger'
 import { AirtableIcon } from '@/components/icons'
+import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'

 const logger = createLogger('AirtableConnector')
@@ -162,7 +163,7 @@ export const airtableConnector: ConnectorConfig = {
      view: viewId ?? 'default',
    })

-    const response = await fetch(url, {
+    const response = await fetchWithRetry(url, {
      method: 'GET',
      headers: {
        Authorization: `Bearer ${accessToken}`,
@@ -211,7 +212,7 @@ export const airtableConnector: ConnectorConfig = {
    const encodedTable = encodeURIComponent(tableIdOrName)
    const url = `${AIRTABLE_API}/${baseId}/${encodedTable}/${externalId}`

-    const response = await fetch(url, {
+    const response = await fetchWithRetry(url, {
      method: 'GET',
      headers: {
        Authorization: `Bearer ${accessToken}`,
@@ -251,12 +252,16 @@ export const airtableConnector: ConnectorConfig = {
      // Verify base and table are accessible by fetching 1 record
      const encodedTable = encodeURIComponent(tableIdOrName)
      const url = `${AIRTABLE_API}/${baseId}/${encodedTable}?pageSize=1`
-      const response = await fetch(url, {
-        method: 'GET',
-        headers: {
-          Authorization: `Bearer ${accessToken}`,
+      const response = await fetchWithRetry(
+        url,
+        {
+          method: 'GET',
+          headers: {
+            Authorization: `Bearer ${accessToken}`,
+          },
        },
-      })
+        VALIDATE_RETRY_OPTIONS
+      )

      if (!response.ok) {
        const errorText = await response.text()
@@ -273,12 +278,16 @@ export const airtableConnector: ConnectorConfig = {
      const viewId = sourceConfig.viewId as string | undefined
      if (viewId) {
        const viewUrl = `${AIRTABLE_API}/${baseId}/${encodedTable}?pageSize=1&view=${encodeURIComponent(viewId)}`
-        const viewResponse = await fetch(viewUrl, {
-          method: 'GET',
-          headers: {
-            Authorization: `Bearer ${accessToken}`,
+        const viewResponse = await fetchWithRetry(
+          viewUrl,
+          {
+            method: 'GET',
+            headers: {
+              Authorization: `Bearer ${accessToken}`,
+            },
          },
-        })
+          VALIDATE_RETRY_OPTIONS
+        )
        if (!viewResponse.ok) {
          return { valid: false, error: `View "${viewId}" not found in table "${tableIdOrName}"` }
        }
@@ -354,7 +363,7 @@ async function fetchFieldNames(

  try {
    const url = `${AIRTABLE_API}/meta/bases/${baseId}/tables`
-    const response = await fetch(url, {
+    const response = await fetchWithRetry(url, {
      method: 'GET',
      headers: {
        Authorization: `Bearer ${accessToken}`,
--- a/apps/sim/connectors/confluence/confluence.ts
+++ b/apps/sim/connectors/confluence/confluence.ts
@@ -1,5 +1,6 @@
 import { createLogger } from '@sim/logger'
 import { ConfluenceIcon } from '@/components/icons'
+import { fetchWithRetry } from '@/lib/knowledge/documents/utils'
 import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
 import { getConfluenceCloudId } from '@/tools/confluence/utils'

@@ -45,7 +46,7 @@ async function fetchLabelsForPages(
    pageIds.map(async (pageId) => {
      try {
        const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/pages/${pageId}/labels`
-        const response = await fetch(url, {
+        const response = await fetchWithRetry(url, {
          method: 'GET',
          headers: {
            Accept: 'application/json',
@@ -226,7 +227,7 @@ export const confluenceConnector: ConnectorConfig = {

    const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/pages/${externalId}?body-format=storage`

-    const response = await fetch(url, {
+    const response = await fetchWithRetry(url, {
      method: 'GET',
      headers: {
        Accept: 'application/json',
@@ -342,7 +343,7 @@ async function listDocumentsV2(

  logger.info(`Listing ${endpoint} in space ${spaceKey} (ID: ${spaceId})`)

-  const response = await fetch(url, {
+  const response = await fetchWithRetry(url, {
    method: 'GET',
    headers: {
      Accept: 'application/json',
@@ -543,7 +544,7 @@ async function listDocumentsViaCql(

  logger.info(`Searching Confluence via CQL: ${cql}`, { start, limit })

-  const response = await fetch(url, {
+  const response = await fetchWithRetry(url, {
    method: 'GET',
    headers: {
      Accept: 'application/json',
@@ -588,7 +589,7 @@ async function resolveSpaceId(
 ): Promise<string> {
  const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/spaces?keys=${encodeURIComponent(spaceKey)}&limit=1`

-  const response = await fetch(url, {
+  const response = await fetchWithRetry(url, {
    method: 'GET',
    headers: {
      Accept: 'application/json',
--- a/apps/sim/connectors/github/github.ts
+++ b/apps/sim/connectors/github/github.ts
@@ -1,5 +1,6 @@
 import { createLogger } from '@sim/logger'
 import { GithubIcon } from '@/components/icons'
+import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'

 const logger = createLogger('GitHubConnector')
@@ -73,7 +74,7 @@ async function fetchTree(
 ): Promise<TreeItem[]> {
  const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/git/trees/${encodeURIComponent(branch)}?recursive=1`

-  const response = await fetch(url, {
+  const response = await fetchWithRetry(url, {
    method: 'GET',
    headers: {
      Accept: 'application/vnd.github+json',
@@ -108,7 +109,7 @@ async function fetchBlobContent(
 ): Promise<string> {
  const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/git/blobs/${sha}`

-  const response = await fetch(url, {
+  const response = await fetchWithRetry(url, {
    method: 'GET',
    headers: {
      Accept: 'application/vnd.github+json',
@@ -282,7 +283,7 @@ export const githubConnector: ConnectorConfig = {

    try {
      const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/contents/${encodeURIComponent(path)}?ref=${encodeURIComponent(branch)}`
-      const response = await fetch(url, {
+      const response = await fetchWithRetry(url, {
        method: 'GET',
        headers: {
          Accept: 'application/vnd.github+json',
@@ -358,14 +359,18 @@ export const githubConnector: ConnectorConfig = {
    try {
      // Verify repo and branch are accessible
      const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/branches/${encodeURIComponent(branch)}`
-      const response = await fetch(url, {
-        method: 'GET',
-        headers: {
-          Accept: 'application/vnd.github+json',
-          Authorization: `Bearer ${accessToken}`,
-          'X-GitHub-Api-Version': '2022-11-28',
+      const response = await fetchWithRetry(
+        url,
+        {
+          method: 'GET',
+          headers: {
+            Accept: 'application/vnd.github+json',
+            Authorization: `Bearer ${accessToken}`,
+            'X-GitHub-Api-Version': '2022-11-28',
+          },
        },
-      })
+        VALIDATE_RETRY_OPTIONS
+      )

      if (response.status === 404) {
        return {
--- a/apps/sim/connectors/google-drive/google-drive.ts
+++ b/apps/sim/connectors/google-drive/google-drive.ts
@@ -1,5 +1,6 @@
 import { createLogger } from '@sim/logger'
 import { GoogleDriveIcon } from '@/components/icons'
+import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'

 const logger = createLogger('GoogleDriveConnector')
@@ -61,7 +62,7 @@ async function exportGoogleWorkspaceFile(

  const url = `https://www.googleapis.com/drive/v3/files/${fileId}/export?mimeType=${encodeURIComponent(exportMimeType)}`

-  const response = await fetch(url, {
+  const response = await fetchWithRetry(url, {
    method: 'GET',
    headers: { Authorization: `Bearer ${accessToken}` },
  })
@@ -76,7 +77,7 @@ async function exportGoogleWorkspaceFile(
 async function downloadTextFile(accessToken: string, fileId: string): Promise<string> {
  const url = `https://www.googleapis.com/drive/v3/files/${fileId}?alt=media`

-  const response = await fetch(url, {
+  const response = await fetchWithRetry(url, {
    method: 'GET',
    headers: { Authorization: `Bearer ${accessToken}` },
  })
@@ -266,7 +267,7 @@ export const googleDriveConnector: ConnectorConfig = {

    logger.info('Listing Google Drive files', { query, cursor: cursor ?? 'initial' })

-    const response = await fetch(url, {
+    const response = await fetchWithRetry(url, {
      method: 'GET',
      headers: {
        Authorization: `Bearer ${accessToken}`,
@@ -310,7 +311,7 @@ export const googleDriveConnector: ConnectorConfig = {
      'id,name,mimeType,modifiedTime,createdTime,webViewLink,parents,owners,size,starred,trashed'
    const url = `https://www.googleapis.com/drive/v3/files/${externalId}?fields=${encodeURIComponent(fields)}&supportsAllDrives=true`

-    const response = await fetch(url, {
+    const response = await fetchWithRetry(url, {
      method: 'GET',
      headers: {
        Authorization: `Bearer ${accessToken}`,
@@ -346,13 +347,17 @@ export const googleDriveConnector: ConnectorConfig = {
      if (folderId?.trim()) {
        // Verify the folder exists and is accessible
        const url = `https://www.googleapis.com/drive/v3/files/${folderId.trim()}?fields=id,name,mimeType&supportsAllDrives=true`
-        const response = await fetch(url, {
-          method: 'GET',
-          headers: {
-            Authorization: `Bearer ${accessToken}`,
-            Accept: 'application/json',
+        const response = await fetchWithRetry(
+          url,
+          {
+            method: 'GET',
+            headers: {
+              Authorization: `Bearer ${accessToken}`,
+              Accept: 'application/json',
+            },
          },
-        })
+          VALIDATE_RETRY_OPTIONS
+        )

        if (!response.ok) {
          if (response.status === 404) {
@@ -368,13 +373,17 @@ export const googleDriveConnector: ConnectorConfig = {
      } else {
        // Verify basic Drive access by listing one file
        const url = 'https://www.googleapis.com/drive/v3/files?pageSize=1&fields=files(id)'
-        const response = await fetch(url, {
-          method: 'GET',
-          headers: {
-            Authorization: `Bearer ${accessToken}`,
-            Accept: 'application/json',
+        const response = await fetchWithRetry(
+          url,
+          {
+            method: 'GET',
+            headers: {
+              Authorization: `Bearer ${accessToken}`,
+              Accept: 'application/json',
+            },
          },
-        })
+          VALIDATE_RETRY_OPTIONS
+        )

        if (!response.ok) {
          return { valid: false, error: `Failed to access Google Drive: ${response.status}` }
--- a/apps/sim/connectors/jira/jira.ts
+++ b/apps/sim/connectors/jira/jira.ts
@@ -1,5 +1,6 @@
 import { createLogger } from '@sim/logger'
 import { JiraIcon } from '@/components/icons'
+import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
 import { extractAdfText, getJiraCloudId } from '@/tools/jira/utils'

@@ -159,7 +160,7 @@ export const jiraConnector: ConnectorConfig = {

    logger.info(`Listing Jira issues for project ${projectKey}`, { startAt })

-    const response = await fetch(url, {
+    const response = await fetchWithRetry(url, {
      method: 'GET',
      headers: {
        Accept: 'application/json',
@@ -210,7 +211,7 @@ export const jiraConnector: ConnectorConfig = {

    const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${externalId}?${params.toString()}`

-    const response = await fetch(url, {
+    const response = await fetchWithRetry(url, {
      method: 'GET',
      headers: {
        Accept: 'application/json',
@@ -259,13 +260,17 @@ export const jiraConnector: ConnectorConfig = {
      params.append('maxResults', '0')

      const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/search?${params.toString()}`
-      const response = await fetch(url, {
-        method: 'GET',
-        headers: {
-          Accept: 'application/json',
-          Authorization: `Bearer ${accessToken}`,
+      const response = await fetchWithRetry(
+        url,
+        {
+          method: 'GET',
+          headers: {
+            Accept: 'application/json',
+            Authorization: `Bearer ${accessToken}`,
+          },
        },
-      })
+        VALIDATE_RETRY_OPTIONS
+      )

      if (!response.ok) {
        const errorText = await response.text()
--- a/apps/sim/connectors/linear/linear.ts
+++ b/apps/sim/connectors/linear/linear.ts
@@ -1,5 +1,6 @@
 import { createLogger } from '@sim/logger'
 import { LinearIcon } from '@/components/icons'
+import { fetchWithRetry } from '@/lib/knowledge/documents/utils'
 import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'

 const logger = createLogger('LinearConnector')
@@ -46,7 +47,7 @@ async function linearGraphQL(
  query: string,
  variables?: Record<string, unknown>
 ): Promise<Record<string, unknown>> {
-  const response = await fetch(LINEAR_API, {
+  const response = await fetchWithRetry(LINEAR_API, {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
--- a/apps/sim/connectors/notion/notion.ts
+++ b/apps/sim/connectors/notion/notion.ts
@@ -1,5 +1,6 @@
 import { createLogger } from '@sim/logger'
 import { NotionIcon } from '@/components/icons'
+import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'

 const logger = createLogger('NotionConnector')
@@ -97,7 +98,7 @@ async function fetchAllBlocks(
    const params = new URLSearchParams({ page_size: '100' })
    if (cursor) params.append('start_cursor', cursor)

-    const response = await fetch(
+    const response = await fetchWithRetry(
      `${NOTION_BASE_URL}/blocks/${pageId}/children?${params.toString()}`,
      {
        method: 'GET',
@@ -262,7 +263,7 @@ export const notionConnector: ConnectorConfig = {
    _sourceConfig: Record<string, unknown>,
    externalId: string
  ): Promise<ExternalDocument | null> => {
-    const response = await fetch(`${NOTION_BASE_URL}/pages/${externalId}`, {
+    const response = await fetchWithRetry(`${NOTION_BASE_URL}/pages/${externalId}`, {
      method: 'GET',
      headers: {
        Authorization: `Bearer ${accessToken}`,
@@ -304,39 +305,51 @@ export const notionConnector: ConnectorConfig = {
      // Verify the token works
      if (scope === 'database' && databaseId) {
        // Verify database is accessible
-        const response = await fetch(`${NOTION_BASE_URL}/databases/${databaseId}`, {
-          method: 'GET',
-          headers: {
-            Authorization: `Bearer ${accessToken}`,
-            'Notion-Version': NOTION_API_VERSION,
+        const response = await fetchWithRetry(
+          `${NOTION_BASE_URL}/databases/${databaseId}`,
+          {
+            method: 'GET',
+            headers: {
+              Authorization: `Bearer ${accessToken}`,
+              'Notion-Version': NOTION_API_VERSION,
+            },
          },
-        })
+          VALIDATE_RETRY_OPTIONS
+        )
        if (!response.ok) {
          return { valid: false, error: `Cannot access database: ${response.status}` }
        }
      } else if (scope === 'page' && rootPageId) {
        // Verify page is accessible
-        const response = await fetch(`${NOTION_BASE_URL}/pages/${rootPageId}`, {
-          method: 'GET',
-          headers: {
-            Authorization: `Bearer ${accessToken}`,
-            'Notion-Version': NOTION_API_VERSION,
+        const response = await fetchWithRetry(
+          `${NOTION_BASE_URL}/pages/${rootPageId}`,
+          {
+            method: 'GET',
+            headers: {
+              Authorization: `Bearer ${accessToken}`,
+              'Notion-Version': NOTION_API_VERSION,
+            },
          },
-        })
+          VALIDATE_RETRY_OPTIONS
+        )
        if (!response.ok) {
          return { valid: false, error: `Cannot access page: ${response.status}` }
        }
      } else {
        // Workspace scope — just verify token works
-        const response = await fetch(`${NOTION_BASE_URL}/search`, {
-          method: 'POST',
-          headers: {
-            Authorization: `Bearer ${accessToken}`,
-            'Notion-Version': NOTION_API_VERSION,
-            'Content-Type': 'application/json',
+        const response = await fetchWithRetry(
+          `${NOTION_BASE_URL}/search`,
+          {
+            method: 'POST',
+            headers: {
+              Authorization: `Bearer ${accessToken}`,
+              'Notion-Version': NOTION_API_VERSION,
+              'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({ page_size: 1 }),
          },
-          body: JSON.stringify({ page_size: 1 }),
-        })
+          VALIDATE_RETRY_OPTIONS
+        )
        if (!response.ok) {
          const errorText = await response.text()
          return { valid: false, error: `Cannot access Notion workspace: ${errorText}` }
@@ -401,7 +414,7 @@ async function listFromWorkspace(

  logger.info('Listing Notion pages from workspace', { searchQuery, cursor })

-  const response = await fetch(`${NOTION_BASE_URL}/search`, {
+  const response = await fetchWithRetry(`${NOTION_BASE_URL}/search`, {
    method: 'POST',
    headers: {
      Authorization: `Bearer ${accessToken}`,
@@ -450,7 +463,7 @@ async function listFromDatabase(

  logger.info('Querying Notion database', { databaseId, cursor })

-  const response = await fetch(`${NOTION_BASE_URL}/databases/${databaseId}/query`, {
+  const response = await fetchWithRetry(`${NOTION_BASE_URL}/databases/${databaseId}/query`, {
    method: 'POST',
    headers: {
      Authorization: `Bearer ${accessToken}`,
@@ -497,7 +510,7 @@ async function listFromParentPage(

  logger.info('Listing child pages under root page', { rootPageId, cursor })

-  const response = await fetch(
+  const response = await fetchWithRetry(
    `${NOTION_BASE_URL}/blocks/${rootPageId}/children?${params.toString()}`,
    {
      method: 'GET',
@@ -531,7 +544,7 @@ async function listFromParentPage(
    if (maxPages > 0 && documents.length >= maxPages) break

    try {
-      const pageResponse = await fetch(`${NOTION_BASE_URL}/pages/${pageId}`, {
+      const pageResponse = await fetchWithRetry(`${NOTION_BASE_URL}/pages/${pageId}`, {
        method: 'GET',
        headers: {
          Authorization: `Bearer ${accessToken}`,
--- a/apps/sim/lib/knowledge/documents/service.ts
+++ b/apps/sim/lib/knowledge/documents/service.ts
@@ -1795,22 +1795,24 @@ export async function deleteDocument(
  documentId: string,
  requestId: string
 ): Promise<{ success: boolean; message: string }> {
-  const doc = await db
+  const docs = await db
    .select({ connectorId: document.connectorId })
    .from(document)
    .where(eq(document.id, documentId))
    .limit(1)

+  const isConnectorDoc = docs.length > 0 && docs[0].connectorId !== null
+
  await db
    .update(document)
    .set({
      deletedAt: new Date(),
-      ...(doc[0]?.connectorId ? { userExcluded: true } : {}),
+      ...(isConnectorDoc ? { userExcluded: true } : {}),
    })
    .where(eq(document.id, documentId))

  logger.info(`[${requestId}] Document deleted: ${documentId}`, {
-    userExcluded: Boolean(doc[0]?.connectorId),
+    userExcluded: isConnectorDoc,
  })

  return {
--- a/apps/sim/lib/knowledge/documents/utils.ts
+++ b/apps/sim/lib/knowledge/documents/utils.ts
@@ -129,6 +129,16 @@ export async function retryWithExponentialBackoff<T>(
  throw lastError || new Error('Retry operation failed')
 }

+/**
+ * Tighter retry options for user-facing operations (e.g. validateConfig).
+ * Caps total wait at ~7s instead of ~31s to avoid API route timeouts.
+ */
+export const VALIDATE_RETRY_OPTIONS: RetryOptions = {
+  maxRetries: 3,
+  initialDelayMs: 1000,
+  maxDelayMs: 10000,
+}
+
 /**
 * Wrapper for fetch requests with retry logic
 */