github/sim
1
0
Fork 0
mirror of https://github.com/simstudioai/sim.git synced 2026-02-18 02:11:59 -05:00
Code Issues Packages Projects Releases Wiki Activity

improvements

Browse Source
This commit is contained in:
waleed
2026-02-16 22:43:12 -08:00
parent 3eaf5babf4
commit 82b874c027
10 changed files with 158 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
.claude/commands/add-connector.md
View File

@@ -28,6 +28,7 @@ connectors/{service}/
```typescript
import { createLogger } from '@sim/logger'
import { {Service}Icon } from '@/components/icons'
import { fetchWithRetry } from '@/lib/knowledge/documents/utils'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
const logger = createLogger('{Service}Connector')
@@ -179,6 +180,25 @@ mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
}
```
## External API Calls — Use `fetchWithRetry`
All external API calls must use `fetchWithRetry` from `@/lib/knowledge/documents/utils` instead of raw `fetch()`. This provides exponential backoff with retries on 429/502/503/504 errors. It returns a standard `Response` — all `.ok`, `.json()`, `.text()` checks work unchanged.
For `validateConfig` (user-facing, called on save), pass `VALIDATE_RETRY_OPTIONS` to cap wait time at ~7s. Background operations (`listDocuments`, `getDocument`) use the built-in defaults (5 retries, ~31s max).
```typescript
import { VALIDATE_RETRY_OPTIONS, fetchWithRetry } from '@/lib/knowledge/documents/utils'
// Background sync — use defaults
const response = await fetchWithRetry(url, {
method: 'GET',
headers: { Authorization: `Bearer ${accessToken}` },
})
// validateConfig — tighter retry budget
const response = await fetchWithRetry(url, { ... }, VALIDATE_RETRY_OPTIONS)
```
## sourceUrl
If `ExternalDocument.sourceUrl` is set, the sync engine stores it on the document record. Always construct the full URL (not a relative path).
@@ -235,6 +255,7 @@ See `apps/sim/connectors/confluence/confluence.ts` for a complete example with:
- [ ] `tagDefinitions` declared for each semantic key returned by `mapTags`
- [ ] `mapTags` implemented if source has useful metadata (labels, dates, versions)
- [ ] `validateConfig` verifies the source is accessible
- [ ] All external API calls use `fetchWithRetry` (not raw `fetch`)
- [ ] All optional config fields validated in `validateConfig`
- [ ] Icon exists in `components/icons.tsx` (or asked user to provide SVG)
- [ ] Registered in `connectors/registry.ts`

35
apps/sim/connectors/airtable/airtable.ts
View File

@@ -1,5 +1,6 @@
import { createLogger } from '@sim/logger'
import { AirtableIcon } from '@/components/icons'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
const logger = createLogger('AirtableConnector')
@@ -162,7 +163,7 @@ export const airtableConnector: ConnectorConfig = {
view: viewId ?? 'default',
})
const response = await fetch(url, {
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
@@ -211,7 +212,7 @@ export const airtableConnector: ConnectorConfig = {
const encodedTable = encodeURIComponent(tableIdOrName)
const url = `${AIRTABLE_API}/${baseId}/${encodedTable}/${externalId}`
const response = await fetch(url, {
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
@@ -251,12 +252,16 @@ export const airtableConnector: ConnectorConfig = {
// Verify base and table are accessible by fetching 1 record
const encodedTable = encodeURIComponent(tableIdOrName)
const url = `${AIRTABLE_API}/${baseId}/${encodedTable}?pageSize=1`
const response = await fetch(url, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
const response = await fetchWithRetry(
url,
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
},
},
})
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
const errorText = await response.text()
@@ -273,12 +278,16 @@ export const airtableConnector: ConnectorConfig = {
const viewId = sourceConfig.viewId as string | undefined
if (viewId) {
const viewUrl = `${AIRTABLE_API}/${baseId}/${encodedTable}?pageSize=1&view=${encodeURIComponent(viewId)}`
const viewResponse = await fetch(viewUrl, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
const viewResponse = await fetchWithRetry(
viewUrl,
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
},
},
})
VALIDATE_RETRY_OPTIONS
)
if (!viewResponse.ok) {
return { valid: false, error: `View "${viewId}" not found in table "${tableIdOrName}"` }
}
@@ -354,7 +363,7 @@ async function fetchFieldNames(
try {
const url = `${AIRTABLE_API}/meta/bases/${baseId}/tables`
const response = await fetch(url, {
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,

11
apps/sim/connectors/confluence/confluence.ts
View File

@@ -1,5 +1,6 @@
import { createLogger } from '@sim/logger'
import { ConfluenceIcon } from '@/components/icons'
import { fetchWithRetry } from '@/lib/knowledge/documents/utils'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { getConfluenceCloudId } from '@/tools/confluence/utils'
@@ -45,7 +46,7 @@ async function fetchLabelsForPages(
pageIds.map(async (pageId) => {
try {
const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/pages/${pageId}/labels`
const response = await fetch(url, {
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Accept: 'application/json',
@@ -226,7 +227,7 @@ export const confluenceConnector: ConnectorConfig = {
const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/pages/${externalId}?body-format=storage`
const response = await fetch(url, {
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Accept: 'application/json',
@@ -342,7 +343,7 @@ async function listDocumentsV2(
logger.info(`Listing ${endpoint} in space ${spaceKey} (ID: ${spaceId})`)
const response = await fetch(url, {
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Accept: 'application/json',
@@ -543,7 +544,7 @@ async function listDocumentsViaCql(
logger.info(`Searching Confluence via CQL: ${cql}`, { start, limit })
const response = await fetch(url, {
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Accept: 'application/json',
@@ -588,7 +589,7 @@ async function resolveSpaceId(
): Promise<string> {
const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/spaces?keys=${encodeURIComponent(spaceKey)}&limit=1`
const response = await fetch(url, {
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Accept: 'application/json',

25
apps/sim/connectors/github/github.ts
View File

@@ -1,5 +1,6 @@
import { createLogger } from '@sim/logger'
import { GithubIcon } from '@/components/icons'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
const logger = createLogger('GitHubConnector')
@@ -73,7 +74,7 @@ async function fetchTree(
): Promise<TreeItem[]> {
const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/git/trees/${encodeURIComponent(branch)}?recursive=1`
const response = await fetch(url, {
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Accept: 'application/vnd.github+json',
@@ -108,7 +109,7 @@ async function fetchBlobContent(
): Promise<string> {
const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/git/blobs/${sha}`
const response = await fetch(url, {
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Accept: 'application/vnd.github+json',
@@ -282,7 +283,7 @@ export const githubConnector: ConnectorConfig = {
try {
const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/contents/${encodeURIComponent(path)}?ref=${encodeURIComponent(branch)}`
const response = await fetch(url, {
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Accept: 'application/vnd.github+json',
@@ -358,14 +359,18 @@ export const githubConnector: ConnectorConfig = {
try {
// Verify repo and branch are accessible
const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/branches/${encodeURIComponent(branch)}`
const response = await fetch(url, {
method: 'GET',
headers: {
Accept: 'application/vnd.github+json',
Authorization: `Bearer ${accessToken}`,
'X-GitHub-Api-Version': '2022-11-28',
const response = await fetchWithRetry(
url,
{
method: 'GET',
headers: {
Accept: 'application/vnd.github+json',
Authorization: `Bearer ${accessToken}`,
'X-GitHub-Api-Version': '2022-11-28',
},
},
})
VALIDATE_RETRY_OPTIONS
)
if (response.status === 404) {
return {

41
apps/sim/connectors/google-drive/google-drive.ts
View File

@@ -1,5 +1,6 @@
import { createLogger } from '@sim/logger'
import { GoogleDriveIcon } from '@/components/icons'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
const logger = createLogger('GoogleDriveConnector')
@@ -61,7 +62,7 @@ async function exportGoogleWorkspaceFile(
const url = `https://www.googleapis.com/drive/v3/files/${fileId}/export?mimeType=${encodeURIComponent(exportMimeType)}`
const response = await fetch(url, {
const response = await fetchWithRetry(url, {
method: 'GET',
headers: { Authorization: `Bearer ${accessToken}` },
})
@@ -76,7 +77,7 @@ async function exportGoogleWorkspaceFile(
async function downloadTextFile(accessToken: string, fileId: string): Promise<string> {
const url = `https://www.googleapis.com/drive/v3/files/${fileId}?alt=media`
const response = await fetch(url, {
const response = await fetchWithRetry(url, {
method: 'GET',
headers: { Authorization: `Bearer ${accessToken}` },
})
@@ -266,7 +267,7 @@ export const googleDriveConnector: ConnectorConfig = {
logger.info('Listing Google Drive files', { query, cursor: cursor ?? 'initial' })
const response = await fetch(url, {
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
@@ -310,7 +311,7 @@ export const googleDriveConnector: ConnectorConfig = {
'id,name,mimeType,modifiedTime,createdTime,webViewLink,parents,owners,size,starred,trashed'
const url = `https://www.googleapis.com/drive/v3/files/${externalId}?fields=${encodeURIComponent(fields)}&supportsAllDrives=true`
const response = await fetch(url, {
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
@@ -346,13 +347,17 @@ export const googleDriveConnector: ConnectorConfig = {
if (folderId?.trim()) {
// Verify the folder exists and is accessible
const url = `https://www.googleapis.com/drive/v3/files/${folderId.trim()}?fields=id,name,mimeType&supportsAllDrives=true`
const response = await fetch(url, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
const response = await fetchWithRetry(
url,
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
},
})
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
if (response.status === 404) {
@@ -368,13 +373,17 @@ export const googleDriveConnector: ConnectorConfig = {
} else {
// Verify basic Drive access by listing one file
const url = 'https://www.googleapis.com/drive/v3/files?pageSize=1&fields=files(id)'
const response = await fetch(url, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
const response = await fetchWithRetry(
url,
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
},
})
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
return { valid: false, error: `Failed to access Google Drive: ${response.status}` }

21
apps/sim/connectors/jira/jira.ts
View File

@@ -1,5 +1,6 @@
import { createLogger } from '@sim/logger'
import { JiraIcon } from '@/components/icons'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { extractAdfText, getJiraCloudId } from '@/tools/jira/utils'
@@ -159,7 +160,7 @@ export const jiraConnector: ConnectorConfig = {
logger.info(`Listing Jira issues for project ${projectKey}`, { startAt })
const response = await fetch(url, {
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Accept: 'application/json',
@@ -210,7 +211,7 @@ export const jiraConnector: ConnectorConfig = {
const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${externalId}?${params.toString()}`
const response = await fetch(url, {
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Accept: 'application/json',
@@ -259,13 +260,17 @@ export const jiraConnector: ConnectorConfig = {
params.append('maxResults', '0')
const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/search?${params.toString()}`
const response = await fetch(url, {
method: 'GET',
headers: {
Accept: 'application/json',
Authorization: `Bearer ${accessToken}`,
const response = await fetchWithRetry(
url,
{
method: 'GET',
headers: {
Accept: 'application/json',
Authorization: `Bearer ${accessToken}`,
},
},
})
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
const errorText = await response.text()

3
apps/sim/connectors/linear/linear.ts
View File

@@ -1,5 +1,6 @@
import { createLogger } from '@sim/logger'
import { LinearIcon } from '@/components/icons'
import { fetchWithRetry } from '@/lib/knowledge/documents/utils'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
const logger = createLogger('LinearConnector')
@@ -46,7 +47,7 @@ async function linearGraphQL(
query: string,
variables?: Record<string, unknown>
): Promise<Record<string, unknown>> {
const response = await fetch(LINEAR_API, {
const response = await fetchWithRetry(LINEAR_API, {
method: 'POST',
headers: {
'Content-Type': 'application/json',

65
apps/sim/connectors/notion/notion.ts
View File

@@ -1,5 +1,6 @@
import { createLogger } from '@sim/logger'
import { NotionIcon } from '@/components/icons'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
const logger = createLogger('NotionConnector')
@@ -97,7 +98,7 @@ async function fetchAllBlocks(
const params = new URLSearchParams({ page_size: '100' })
if (cursor) params.append('start_cursor', cursor)
const response = await fetch(
const response = await fetchWithRetry(
`${NOTION_BASE_URL}/blocks/${pageId}/children?${params.toString()}`,
{
method: 'GET',
@@ -262,7 +263,7 @@ export const notionConnector: ConnectorConfig = {
_sourceConfig: Record<string, unknown>,
externalId: string
): Promise<ExternalDocument | null> => {
const response = await fetch(`${NOTION_BASE_URL}/pages/${externalId}`, {
const response = await fetchWithRetry(`${NOTION_BASE_URL}/pages/${externalId}`, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
@@ -304,39 +305,51 @@ export const notionConnector: ConnectorConfig = {
// Verify the token works
if (scope === 'database' && databaseId) {
// Verify database is accessible
const response = await fetch(`${NOTION_BASE_URL}/databases/${databaseId}`, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
'Notion-Version': NOTION_API_VERSION,
const response = await fetchWithRetry(
`${NOTION_BASE_URL}/databases/${databaseId}`,
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
'Notion-Version': NOTION_API_VERSION,
},
},
})
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
return { valid: false, error: `Cannot access database: ${response.status}` }
}
} else if (scope === 'page' && rootPageId) {
// Verify page is accessible
const response = await fetch(`${NOTION_BASE_URL}/pages/${rootPageId}`, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
'Notion-Version': NOTION_API_VERSION,
const response = await fetchWithRetry(
`${NOTION_BASE_URL}/pages/${rootPageId}`,
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
'Notion-Version': NOTION_API_VERSION,
},
},
})
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
return { valid: false, error: `Cannot access page: ${response.status}` }
}
} else {
// Workspace scope — just verify token works
const response = await fetch(`${NOTION_BASE_URL}/search`, {
method: 'POST',
headers: {
Authorization: `Bearer ${accessToken}`,
'Notion-Version': NOTION_API_VERSION,
'Content-Type': 'application/json',
const response = await fetchWithRetry(
`${NOTION_BASE_URL}/search`,
{
method: 'POST',
headers: {
Authorization: `Bearer ${accessToken}`,
'Notion-Version': NOTION_API_VERSION,
'Content-Type': 'application/json',
},
body: JSON.stringify({ page_size: 1 }),
},
body: JSON.stringify({ page_size: 1 }),
})
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
const errorText = await response.text()
return { valid: false, error: `Cannot access Notion workspace: ${errorText}` }
@@ -401,7 +414,7 @@ async function listFromWorkspace(
logger.info('Listing Notion pages from workspace', { searchQuery, cursor })
const response = await fetch(`${NOTION_BASE_URL}/search`, {
const response = await fetchWithRetry(`${NOTION_BASE_URL}/search`, {
method: 'POST',
headers: {
Authorization: `Bearer ${accessToken}`,
@@ -450,7 +463,7 @@ async function listFromDatabase(
logger.info('Querying Notion database', { databaseId, cursor })
const response = await fetch(`${NOTION_BASE_URL}/databases/${databaseId}/query`, {
const response = await fetchWithRetry(`${NOTION_BASE_URL}/databases/${databaseId}/query`, {
method: 'POST',
headers: {
Authorization: `Bearer ${accessToken}`,
@@ -497,7 +510,7 @@ async function listFromParentPage(
logger.info('Listing child pages under root page', { rootPageId, cursor })
const response = await fetch(
const response = await fetchWithRetry(
`${NOTION_BASE_URL}/blocks/${rootPageId}/children?${params.toString()}`,
{
method: 'GET',
@@ -531,7 +544,7 @@ async function listFromParentPage(
if (maxPages > 0 && documents.length >= maxPages) break
try {
const pageResponse = await fetch(`${NOTION_BASE_URL}/pages/${pageId}`, {
const pageResponse = await fetchWithRetry(`${NOTION_BASE_URL}/pages/${pageId}`, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,

8
apps/sim/lib/knowledge/documents/service.ts
View File

@@ -1795,22 +1795,24 @@ export async function deleteDocument(
documentId: string,
requestId: string
): Promise<{ success: boolean; message: string }> {
const doc = await db
const docs = await db
.select({ connectorId: document.connectorId })
.from(document)
.where(eq(document.id, documentId))
.limit(1)
const isConnectorDoc = docs.length > 0 && docs[0].connectorId !== null
await db
.update(document)
.set({
deletedAt: new Date(),
...(doc[0]?.connectorId ? { userExcluded: true } : {}),
...(isConnectorDoc ? { userExcluded: true } : {}),
})
.where(eq(document.id, documentId))
logger.info(`[${requestId}] Document deleted: ${documentId}`, {
userExcluded: Boolean(doc[0]?.connectorId),
userExcluded: isConnectorDoc,
})
return {

10
apps/sim/lib/knowledge/documents/utils.ts
View File

@@ -129,6 +129,16 @@ export async function retryWithExponentialBackoff<T>(
throw lastError || new Error('Retry operation failed')
}
/**
* Tighter retry options for user-facing operations (e.g. validateConfig).
* Caps total wait at ~7s instead of ~31s to avoid API route timeouts.
*/
export const VALIDATE_RETRY_OPTIONS: RetryOptions = {
maxRetries: 3,
initialDelayMs: 1000,
maxDelayMs: 10000,
}
/**
* Wrapper for fetch requests with retry logic
*/