mirror of
https://github.com/simstudioai/sim.git
synced 2026-04-06 03:00:16 -04:00
feat(search): added operations to search modal in main app, updated retrieval in docs to use RRF (#2889)
This commit is contained in:
@@ -86,27 +86,112 @@ export async function GET(request: NextRequest) {
|
||||
)
|
||||
.limit(candidateLimit)
|
||||
|
||||
const seenIds = new Set<string>()
|
||||
const mergedResults = []
|
||||
const knownLocales = ['en', 'es', 'fr', 'de', 'ja', 'zh']
|
||||
|
||||
for (let i = 0; i < Math.max(vectorResults.length, keywordResults.length); i++) {
|
||||
if (i < vectorResults.length && !seenIds.has(vectorResults[i].chunkId)) {
|
||||
mergedResults.push(vectorResults[i])
|
||||
seenIds.add(vectorResults[i].chunkId)
|
||||
}
|
||||
if (i < keywordResults.length && !seenIds.has(keywordResults[i].chunkId)) {
|
||||
mergedResults.push(keywordResults[i])
|
||||
seenIds.add(keywordResults[i].chunkId)
|
||||
const vectorRankMap = new Map<string, number>()
|
||||
vectorResults.forEach((r, idx) => vectorRankMap.set(r.chunkId, idx + 1))
|
||||
|
||||
const keywordRankMap = new Map<string, number>()
|
||||
keywordResults.forEach((r, idx) => keywordRankMap.set(r.chunkId, idx + 1))
|
||||
|
||||
const allChunkIds = new Set([
|
||||
...vectorResults.map((r) => r.chunkId),
|
||||
...keywordResults.map((r) => r.chunkId),
|
||||
])
|
||||
|
||||
const k = 60
|
||||
type ResultWithRRF = (typeof vectorResults)[0] & { rrfScore: number }
|
||||
const scoredResults: ResultWithRRF[] = []
|
||||
|
||||
for (const chunkId of allChunkIds) {
|
||||
const vectorRank = vectorRankMap.get(chunkId) ?? Number.POSITIVE_INFINITY
|
||||
const keywordRank = keywordRankMap.get(chunkId) ?? Number.POSITIVE_INFINITY
|
||||
|
||||
const rrfScore = 1 / (k + vectorRank) + 1 / (k + keywordRank)
|
||||
|
||||
const result =
|
||||
vectorResults.find((r) => r.chunkId === chunkId) ||
|
||||
keywordResults.find((r) => r.chunkId === chunkId)
|
||||
|
||||
if (result) {
|
||||
scoredResults.push({ ...result, rrfScore })
|
||||
}
|
||||
}
|
||||
|
||||
const filteredResults = mergedResults.slice(0, limit)
|
||||
const searchResults = filteredResults.map((result) => {
|
||||
scoredResults.sort((a, b) => b.rrfScore - a.rrfScore)
|
||||
|
||||
const localeFilteredResults = scoredResults.filter((result) => {
|
||||
const firstPart = result.sourceDocument.split('/')[0]
|
||||
if (knownLocales.includes(firstPart)) {
|
||||
return firstPart === locale
|
||||
}
|
||||
return locale === 'en'
|
||||
})
|
||||
|
||||
const queryLower = query.toLowerCase()
|
||||
const getTitleBoost = (result: ResultWithRRF): number => {
|
||||
const fileName = result.sourceDocument
|
||||
.replace('.mdx', '')
|
||||
.split('/')
|
||||
.pop()
|
||||
?.toLowerCase()
|
||||
?.replace(/_/g, ' ')
|
||||
|
||||
if (fileName === queryLower) return 0.01
|
||||
if (fileName?.includes(queryLower)) return 0.005
|
||||
return 0
|
||||
}
|
||||
|
||||
localeFilteredResults.sort((a, b) => {
|
||||
return b.rrfScore + getTitleBoost(b) - (a.rrfScore + getTitleBoost(a))
|
||||
})
|
||||
|
||||
const pageMap = new Map<string, ResultWithRRF>()
|
||||
|
||||
for (const result of localeFilteredResults) {
|
||||
const pageKey = result.sourceDocument
|
||||
const existing = pageMap.get(pageKey)
|
||||
|
||||
if (!existing || result.rrfScore > existing.rrfScore) {
|
||||
pageMap.set(pageKey, result)
|
||||
}
|
||||
}
|
||||
|
||||
const deduplicatedResults = Array.from(pageMap.values())
|
||||
.sort((a, b) => b.rrfScore + getTitleBoost(b) - (a.rrfScore + getTitleBoost(a)))
|
||||
.slice(0, limit)
|
||||
|
||||
const searchResults = deduplicatedResults.map((result) => {
|
||||
const title = result.headerText || result.sourceDocument.replace('.mdx', '')
|
||||
|
||||
const pathParts = result.sourceDocument
|
||||
.replace('.mdx', '')
|
||||
.split('/')
|
||||
.map((part) => part.charAt(0).toUpperCase() + part.slice(1))
|
||||
.filter((part) => part !== 'index' && !knownLocales.includes(part))
|
||||
.map((part) => {
|
||||
return part
|
||||
.replace(/_/g, ' ')
|
||||
.split(' ')
|
||||
.map((word) => {
|
||||
const acronyms = [
|
||||
'api',
|
||||
'mcp',
|
||||
'sdk',
|
||||
'url',
|
||||
'http',
|
||||
'json',
|
||||
'xml',
|
||||
'html',
|
||||
'css',
|
||||
'ai',
|
||||
]
|
||||
if (acronyms.includes(word.toLowerCase())) {
|
||||
return word.toUpperCase()
|
||||
}
|
||||
return word.charAt(0).toUpperCase() + word.slice(1)
|
||||
})
|
||||
.join(' ')
|
||||
})
|
||||
|
||||
return {
|
||||
id: result.chunkId,
|
||||
|
||||
@@ -692,7 +692,8 @@ const WorkflowContent = React.memo(() => {
|
||||
parentId?: string,
|
||||
extent?: 'parent',
|
||||
autoConnectEdge?: Edge,
|
||||
triggerMode?: boolean
|
||||
triggerMode?: boolean,
|
||||
presetSubBlockValues?: Record<string, unknown>
|
||||
) => {
|
||||
setPendingSelection([id])
|
||||
setSelectedEdges(new Map())
|
||||
@@ -722,6 +723,14 @@ const WorkflowContent = React.memo(() => {
|
||||
}
|
||||
}
|
||||
|
||||
// Apply preset subblock values (e.g., from tool-operation search)
|
||||
if (presetSubBlockValues) {
|
||||
if (!subBlockValues[id]) {
|
||||
subBlockValues[id] = {}
|
||||
}
|
||||
Object.assign(subBlockValues[id], presetSubBlockValues)
|
||||
}
|
||||
|
||||
collaborativeBatchAddBlocks(
|
||||
[block],
|
||||
autoConnectEdge ? [autoConnectEdge] : [],
|
||||
@@ -1489,7 +1498,7 @@ const WorkflowContent = React.memo(() => {
|
||||
return
|
||||
}
|
||||
|
||||
const { type, enableTriggerMode } = event.detail
|
||||
const { type, enableTriggerMode, presetOperation } = event.detail
|
||||
|
||||
if (!type) return
|
||||
if (type === 'connectionBlock') return
|
||||
@@ -1552,7 +1561,8 @@ const WorkflowContent = React.memo(() => {
|
||||
undefined,
|
||||
undefined,
|
||||
autoConnectEdge,
|
||||
enableTriggerMode
|
||||
enableTriggerMode,
|
||||
presetOperation ? { operation: presetOperation } : undefined
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ import { useParams, useRouter } from 'next/navigation'
|
||||
import { Dialog, DialogPortal, DialogTitle } from '@/components/ui/dialog'
|
||||
import { useBrandConfig } from '@/lib/branding/branding'
|
||||
import { cn } from '@/lib/core/utils/cn'
|
||||
import { getToolOperationsIndex } from '@/lib/search/tool-operations'
|
||||
import { getTriggersForSidebar, hasTriggerCapability } from '@/lib/workflows/triggers/trigger-utils'
|
||||
import { searchItems } from '@/app/workspace/[workspaceId]/w/components/sidebar/components/search-modal/search-utils'
|
||||
import { SIDEBAR_SCROLL_EVENT } from '@/app/workspace/[workspaceId]/w/components/sidebar/sidebar'
|
||||
@@ -81,10 +82,12 @@ type SearchItem = {
|
||||
color?: string
|
||||
href?: string
|
||||
shortcut?: string
|
||||
type: 'block' | 'trigger' | 'tool' | 'workflow' | 'workspace' | 'page' | 'doc'
|
||||
type: 'block' | 'trigger' | 'tool' | 'tool-operation' | 'workflow' | 'workspace' | 'page' | 'doc'
|
||||
isCurrent?: boolean
|
||||
blockType?: string
|
||||
config?: any
|
||||
operationId?: string
|
||||
aliases?: string[]
|
||||
}
|
||||
|
||||
interface SearchResultItemProps {
|
||||
@@ -101,7 +104,11 @@ const SearchResultItem = memo(function SearchResultItem({
|
||||
onItemClick,
|
||||
}: SearchResultItemProps) {
|
||||
const Icon = item.icon
|
||||
const showColoredIcon = item.type === 'block' || item.type === 'trigger' || item.type === 'tool'
|
||||
const showColoredIcon =
|
||||
item.type === 'block' ||
|
||||
item.type === 'trigger' ||
|
||||
item.type === 'tool' ||
|
||||
item.type === 'tool-operation'
|
||||
const isWorkflow = item.type === 'workflow'
|
||||
const isWorkspace = item.type === 'workspace'
|
||||
|
||||
@@ -278,6 +285,24 @@ export const SearchModal = memo(function SearchModal({
|
||||
)
|
||||
}, [open, isOnWorkflowPage, filterBlocks])
|
||||
|
||||
const toolOperations = useMemo(() => {
|
||||
if (!open || !isOnWorkflowPage) return []
|
||||
|
||||
const allowedBlockTypes = new Set(tools.map((t) => t.type))
|
||||
|
||||
return getToolOperationsIndex()
|
||||
.filter((op) => allowedBlockTypes.has(op.blockType))
|
||||
.map((op) => ({
|
||||
id: op.id,
|
||||
name: `${op.serviceName}: ${op.operationName}`,
|
||||
icon: op.icon,
|
||||
bgColor: op.bgColor,
|
||||
blockType: op.blockType,
|
||||
operationId: op.operationId,
|
||||
aliases: op.aliases,
|
||||
}))
|
||||
}, [open, isOnWorkflowPage, tools])
|
||||
|
||||
const pages = useMemo(
|
||||
(): PageItem[] => [
|
||||
{
|
||||
@@ -396,6 +421,19 @@ export const SearchModal = memo(function SearchModal({
|
||||
})
|
||||
})
|
||||
|
||||
toolOperations.forEach((op) => {
|
||||
items.push({
|
||||
id: op.id,
|
||||
name: op.name,
|
||||
icon: op.icon,
|
||||
bgColor: op.bgColor,
|
||||
type: 'tool-operation',
|
||||
blockType: op.blockType,
|
||||
operationId: op.operationId,
|
||||
aliases: op.aliases,
|
||||
})
|
||||
})
|
||||
|
||||
docs.forEach((doc) => {
|
||||
items.push({
|
||||
id: doc.id,
|
||||
@@ -407,10 +445,10 @@ export const SearchModal = memo(function SearchModal({
|
||||
})
|
||||
|
||||
return items
|
||||
}, [workspaces, workflows, pages, blocks, triggers, tools, docs])
|
||||
}, [workspaces, workflows, pages, blocks, triggers, tools, toolOperations, docs])
|
||||
|
||||
const sectionOrder = useMemo<SearchItem['type'][]>(
|
||||
() => ['block', 'tool', 'trigger', 'workflow', 'workspace', 'page', 'doc'],
|
||||
() => ['block', 'tool', 'tool-operation', 'trigger', 'workflow', 'workspace', 'page', 'doc'],
|
||||
[]
|
||||
)
|
||||
|
||||
@@ -457,6 +495,7 @@ export const SearchModal = memo(function SearchModal({
|
||||
page: [],
|
||||
trigger: [],
|
||||
block: [],
|
||||
'tool-operation': [],
|
||||
tool: [],
|
||||
doc: [],
|
||||
}
|
||||
@@ -512,6 +551,17 @@ export const SearchModal = memo(function SearchModal({
|
||||
window.dispatchEvent(event)
|
||||
}
|
||||
break
|
||||
case 'tool-operation':
|
||||
if (item.blockType && item.operationId) {
|
||||
const event = new CustomEvent('add-block-from-toolbar', {
|
||||
detail: {
|
||||
type: item.blockType,
|
||||
presetOperation: item.operationId,
|
||||
},
|
||||
})
|
||||
window.dispatchEvent(event)
|
||||
}
|
||||
break
|
||||
case 'workspace':
|
||||
if (item.isCurrent) {
|
||||
break
|
||||
@@ -592,6 +642,7 @@ export const SearchModal = memo(function SearchModal({
|
||||
page: 'Pages',
|
||||
trigger: 'Triggers',
|
||||
block: 'Blocks',
|
||||
'tool-operation': 'Tool Operations',
|
||||
tool: 'Tools',
|
||||
doc: 'Docs',
|
||||
}
|
||||
|
||||
@@ -8,17 +8,19 @@ export interface SearchableItem {
|
||||
name: string
|
||||
description?: string
|
||||
type: string
|
||||
aliases?: string[]
|
||||
[key: string]: any
|
||||
}
|
||||
|
||||
export interface SearchResult<T extends SearchableItem> {
|
||||
item: T
|
||||
score: number
|
||||
matchType: 'exact' | 'prefix' | 'word-boundary' | 'substring' | 'description'
|
||||
matchType: 'exact' | 'prefix' | 'alias' | 'word-boundary' | 'substring' | 'description'
|
||||
}
|
||||
|
||||
const SCORE_EXACT_MATCH = 10000
|
||||
const SCORE_PREFIX_MATCH = 5000
|
||||
const SCORE_ALIAS_MATCH = 3000
|
||||
const SCORE_WORD_BOUNDARY = 1000
|
||||
const SCORE_SUBSTRING_MATCH = 100
|
||||
const DESCRIPTION_WEIGHT = 0.3
|
||||
@@ -67,6 +69,39 @@ function calculateFieldScore(
|
||||
return { score: 0, matchType: null }
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if query matches any alias in the item's aliases array
|
||||
* Returns the alias score if a match is found, 0 otherwise
|
||||
*/
|
||||
function calculateAliasScore(
|
||||
query: string,
|
||||
aliases?: string[]
|
||||
): { score: number; matchType: 'alias' | null } {
|
||||
if (!aliases || aliases.length === 0) {
|
||||
return { score: 0, matchType: null }
|
||||
}
|
||||
|
||||
const normalizedQuery = query.toLowerCase().trim()
|
||||
|
||||
for (const alias of aliases) {
|
||||
const normalizedAlias = alias.toLowerCase().trim()
|
||||
|
||||
if (normalizedAlias === normalizedQuery) {
|
||||
return { score: SCORE_ALIAS_MATCH, matchType: 'alias' }
|
||||
}
|
||||
|
||||
if (normalizedAlias.startsWith(normalizedQuery)) {
|
||||
return { score: SCORE_ALIAS_MATCH * 0.8, matchType: 'alias' }
|
||||
}
|
||||
|
||||
if (normalizedQuery.includes(normalizedAlias) || normalizedAlias.includes(normalizedQuery)) {
|
||||
return { score: SCORE_ALIAS_MATCH * 0.6, matchType: 'alias' }
|
||||
}
|
||||
}
|
||||
|
||||
return { score: 0, matchType: null }
|
||||
}
|
||||
|
||||
/**
|
||||
* Search items using tiered matching algorithm
|
||||
* Returns items sorted by relevance (highest score first)
|
||||
@@ -90,15 +125,20 @@ export function searchItems<T extends SearchableItem>(
|
||||
? calculateFieldScore(normalizedQuery, item.description)
|
||||
: { score: 0, matchType: null }
|
||||
|
||||
const aliasMatch = calculateAliasScore(normalizedQuery, item.aliases)
|
||||
|
||||
const nameScore = nameMatch.score
|
||||
const descScore = descMatch.score * DESCRIPTION_WEIGHT
|
||||
const aliasScore = aliasMatch.score
|
||||
|
||||
const bestScore = Math.max(nameScore, descScore)
|
||||
const bestScore = Math.max(nameScore, descScore, aliasScore)
|
||||
|
||||
if (bestScore > 0) {
|
||||
let matchType: SearchResult<T>['matchType'] = 'substring'
|
||||
if (nameScore >= descScore) {
|
||||
if (nameScore >= descScore && nameScore >= aliasScore) {
|
||||
matchType = nameMatch.matchType || 'substring'
|
||||
} else if (aliasScore >= descScore) {
|
||||
matchType = 'alias'
|
||||
} else {
|
||||
matchType = 'description'
|
||||
}
|
||||
@@ -125,6 +165,8 @@ export function getMatchTypeLabel(matchType: SearchResult<any>['matchType']): st
|
||||
return 'Exact match'
|
||||
case 'prefix':
|
||||
return 'Starts with'
|
||||
case 'alias':
|
||||
return 'Similar to'
|
||||
case 'word-boundary':
|
||||
return 'Word match'
|
||||
case 'substring':
|
||||
|
||||
@@ -29,13 +29,11 @@ export class DocsChunker {
|
||||
private readonly baseUrl: string
|
||||
|
||||
constructor(options: DocsChunkerOptions = {}) {
|
||||
// Use the existing TextChunker for chunking logic
|
||||
this.textChunker = new TextChunker({
|
||||
chunkSize: options.chunkSize ?? 300, // Max 300 tokens per chunk
|
||||
minCharactersPerChunk: options.minCharactersPerChunk ?? 1,
|
||||
chunkOverlap: options.chunkOverlap ?? 50,
|
||||
})
|
||||
// Use localhost docs in development, production docs otherwise
|
||||
this.baseUrl = options.baseUrl ?? 'https://docs.sim.ai'
|
||||
}
|
||||
|
||||
@@ -74,24 +72,18 @@ export class DocsChunker {
|
||||
const content = await fs.readFile(filePath, 'utf-8')
|
||||
const relativePath = path.relative(basePath, filePath)
|
||||
|
||||
// Parse frontmatter and content
|
||||
const { data: frontmatter, content: markdownContent } = this.parseFrontmatter(content)
|
||||
|
||||
// Extract headers from the content
|
||||
const headers = this.extractHeaders(markdownContent)
|
||||
|
||||
// Generate document URL
|
||||
const documentUrl = this.generateDocumentUrl(relativePath)
|
||||
|
||||
// Split content into chunks
|
||||
const textChunks = await this.splitContent(markdownContent)
|
||||
|
||||
// Generate embeddings for all chunks at once (batch processing)
|
||||
logger.info(`Generating embeddings for ${textChunks.length} chunks in ${relativePath}`)
|
||||
const embeddings = textChunks.length > 0 ? await generateEmbeddings(textChunks) : []
|
||||
const embeddingModel = 'text-embedding-3-small'
|
||||
|
||||
// Convert to DocChunk objects with header context and embeddings
|
||||
const chunks: DocChunk[] = []
|
||||
let currentPosition = 0
|
||||
|
||||
@@ -100,7 +92,6 @@ export class DocsChunker {
|
||||
const chunkStart = currentPosition
|
||||
const chunkEnd = currentPosition + chunkText.length
|
||||
|
||||
// Find the most relevant header for this chunk
|
||||
const relevantHeader = this.findRelevantHeader(headers, chunkStart)
|
||||
|
||||
const chunk: DocChunk = {
|
||||
@@ -186,11 +177,21 @@ export class DocsChunker {
|
||||
|
||||
/**
|
||||
* Generate document URL from relative path
|
||||
* Handles index.mdx files specially - they are served at the parent directory path
|
||||
*/
|
||||
private generateDocumentUrl(relativePath: string): string {
|
||||
// Convert file path to URL path
|
||||
// e.g., "tools/knowledge.mdx" -> "/tools/knowledge"
|
||||
const urlPath = relativePath.replace(/\.mdx$/, '').replace(/\\/g, '/') // Handle Windows paths
|
||||
// e.g., "triggers/index.mdx" -> "/triggers" (NOT "/triggers/index")
|
||||
let urlPath = relativePath.replace(/\.mdx$/, '').replace(/\\/g, '/') // Handle Windows paths
|
||||
|
||||
// In fumadocs, index.mdx files are served at the parent directory path
|
||||
// e.g., "triggers/index" -> "triggers"
|
||||
if (urlPath.endsWith('/index')) {
|
||||
urlPath = urlPath.slice(0, -6) // Remove "/index"
|
||||
} else if (urlPath === 'index') {
|
||||
urlPath = '' // Root index.mdx
|
||||
}
|
||||
|
||||
return `${this.baseUrl}/${urlPath}`
|
||||
}
|
||||
@@ -201,7 +202,6 @@ export class DocsChunker {
|
||||
private findRelevantHeader(headers: HeaderInfo[], position: number): HeaderInfo | null {
|
||||
if (headers.length === 0) return null
|
||||
|
||||
// Find the last header that comes before this position
|
||||
let relevantHeader: HeaderInfo | null = null
|
||||
|
||||
for (const header of headers) {
|
||||
@@ -219,23 +219,18 @@ export class DocsChunker {
|
||||
* Split content into chunks using the existing TextChunker with table awareness
|
||||
*/
|
||||
private async splitContent(content: string): Promise<string[]> {
|
||||
// Clean the content first
|
||||
const cleanedContent = this.cleanContent(content)
|
||||
|
||||
// Detect table boundaries to avoid splitting them
|
||||
const tableBoundaries = this.detectTableBoundaries(cleanedContent)
|
||||
|
||||
// Use the existing TextChunker
|
||||
const chunks = await this.textChunker.chunk(cleanedContent)
|
||||
|
||||
// Post-process chunks to ensure tables aren't split
|
||||
const processedChunks = this.mergeTableChunks(
|
||||
chunks.map((chunk) => chunk.text),
|
||||
tableBoundaries,
|
||||
cleanedContent
|
||||
)
|
||||
|
||||
// Ensure no chunk exceeds 300 tokens
|
||||
const finalChunks = this.enforceSizeLimit(processedChunks)
|
||||
|
||||
return finalChunks
|
||||
@@ -273,7 +268,6 @@ export class DocsChunker {
|
||||
const [, frontmatterText, markdownContent] = match
|
||||
const data: Frontmatter = {}
|
||||
|
||||
// Simple YAML parsing for title and description
|
||||
const lines = frontmatterText.split('\n')
|
||||
for (const line of lines) {
|
||||
const colonIndex = line.indexOf(':')
|
||||
@@ -294,7 +288,6 @@ export class DocsChunker {
|
||||
* Estimate token count (rough approximation)
|
||||
*/
|
||||
private estimateTokens(text: string): number {
|
||||
// Rough approximation: 1 token ≈ 4 characters
|
||||
return Math.ceil(text.length / 4)
|
||||
}
|
||||
|
||||
@@ -311,17 +304,13 @@ export class DocsChunker {
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i].trim()
|
||||
|
||||
// Detect table start (markdown table row with pipes)
|
||||
if (line.includes('|') && line.split('|').length >= 3 && !inTable) {
|
||||
// Check if next line is table separator (contains dashes and pipes)
|
||||
const nextLine = lines[i + 1]?.trim()
|
||||
if (nextLine?.includes('|') && nextLine.includes('-')) {
|
||||
inTable = true
|
||||
tableStart = i
|
||||
}
|
||||
}
|
||||
// Detect table end (empty line or non-table content)
|
||||
else if (inTable && (!line.includes('|') || line === '' || line.startsWith('#'))) {
|
||||
} else if (inTable && (!line.includes('|') || line === '' || line.startsWith('#'))) {
|
||||
tables.push({
|
||||
start: this.getCharacterPosition(lines, tableStart),
|
||||
end: this.getCharacterPosition(lines, i - 1) + lines[i - 1]?.length || 0,
|
||||
@@ -330,7 +319,6 @@ export class DocsChunker {
|
||||
}
|
||||
}
|
||||
|
||||
// Handle table at end of content
|
||||
if (inTable && tableStart >= 0) {
|
||||
tables.push({
|
||||
start: this.getCharacterPosition(lines, tableStart),
|
||||
@@ -367,7 +355,6 @@ export class DocsChunker {
|
||||
const chunkStart = originalContent.indexOf(chunk, currentPosition)
|
||||
const chunkEnd = chunkStart + chunk.length
|
||||
|
||||
// Check if this chunk intersects with any table
|
||||
const intersectsTable = tableBoundaries.some(
|
||||
(table) =>
|
||||
(chunkStart >= table.start && chunkStart <= table.end) ||
|
||||
@@ -376,7 +363,6 @@ export class DocsChunker {
|
||||
)
|
||||
|
||||
if (intersectsTable) {
|
||||
// Find which table(s) this chunk intersects with
|
||||
const affectedTables = tableBoundaries.filter(
|
||||
(table) =>
|
||||
(chunkStart >= table.start && chunkStart <= table.end) ||
|
||||
@@ -384,12 +370,10 @@ export class DocsChunker {
|
||||
(chunkStart <= table.start && chunkEnd >= table.end)
|
||||
)
|
||||
|
||||
// Create a chunk that includes the complete table(s)
|
||||
const minStart = Math.min(chunkStart, ...affectedTables.map((t) => t.start))
|
||||
const maxEnd = Math.max(chunkEnd, ...affectedTables.map((t) => t.end))
|
||||
const completeChunk = originalContent.slice(minStart, maxEnd)
|
||||
|
||||
// Only add if we haven't already included this content
|
||||
if (!mergedChunks.some((existing) => existing.includes(completeChunk.trim()))) {
|
||||
mergedChunks.push(completeChunk.trim())
|
||||
}
|
||||
@@ -400,7 +384,7 @@ export class DocsChunker {
|
||||
currentPosition = chunkEnd
|
||||
}
|
||||
|
||||
return mergedChunks.filter((chunk) => chunk.length > 50) // Filter out tiny chunks
|
||||
return mergedChunks.filter((chunk) => chunk.length > 50)
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -413,10 +397,8 @@ export class DocsChunker {
|
||||
const tokens = this.estimateTokens(chunk)
|
||||
|
||||
if (tokens <= 300) {
|
||||
// Chunk is within limit
|
||||
finalChunks.push(chunk)
|
||||
} else {
|
||||
// Chunk is too large - split it
|
||||
const lines = chunk.split('\n')
|
||||
let currentChunk = ''
|
||||
|
||||
@@ -426,7 +408,6 @@ export class DocsChunker {
|
||||
if (this.estimateTokens(testChunk) <= 300) {
|
||||
currentChunk = testChunk
|
||||
} else {
|
||||
// Adding this line would exceed limit
|
||||
if (currentChunk.trim()) {
|
||||
finalChunks.push(currentChunk.trim())
|
||||
}
|
||||
@@ -434,7 +415,6 @@ export class DocsChunker {
|
||||
}
|
||||
}
|
||||
|
||||
// Add final chunk if it has content
|
||||
if (currentChunk.trim()) {
|
||||
finalChunks.push(currentChunk.trim())
|
||||
}
|
||||
|
||||
@@ -326,32 +326,32 @@ export const env = createEnv({
|
||||
|
||||
NEXT_PUBLIC_E2B_ENABLED: z.string().optional(),
|
||||
NEXT_PUBLIC_COPILOT_TRAINING_ENABLED: z.string().optional(),
|
||||
NEXT_PUBLIC_ENABLE_PLAYGROUND: z.string().optional(), // Enable component playground at /playground
|
||||
NEXT_PUBLIC_ENABLE_PLAYGROUND: z.string().optional(), // Enable component playground at /playground
|
||||
NEXT_PUBLIC_DOCUMENTATION_URL: z.string().url().optional(), // Custom documentation URL
|
||||
NEXT_PUBLIC_TERMS_URL: z.string().url().optional(), // Custom terms of service URL
|
||||
NEXT_PUBLIC_PRIVACY_URL: z.string().url().optional(), // Custom privacy policy URL
|
||||
|
||||
// Theme Customization
|
||||
NEXT_PUBLIC_BRAND_PRIMARY_COLOR: z.string().regex(/^#[0-9A-Fa-f]{6}$/).optional(), // Primary brand color (hex format, e.g., "#701ffc")
|
||||
NEXT_PUBLIC_BRAND_PRIMARY_HOVER_COLOR: z.string().regex(/^#[0-9A-Fa-f]{6}$/).optional(), // Primary brand hover state (hex format)
|
||||
NEXT_PUBLIC_BRAND_PRIMARY_HOVER_COLOR: z.string().regex(/^#[0-9A-Fa-f]{6}$/).optional(), // Primary brand hover state (hex format)
|
||||
NEXT_PUBLIC_BRAND_ACCENT_COLOR: z.string().regex(/^#[0-9A-Fa-f]{6}$/).optional(), // Accent brand color (hex format)
|
||||
NEXT_PUBLIC_BRAND_ACCENT_HOVER_COLOR: z.string().regex(/^#[0-9A-Fa-f]{6}$/).optional(), // Accent brand hover state (hex format)
|
||||
NEXT_PUBLIC_BRAND_BACKGROUND_COLOR: z.string().regex(/^#[0-9A-Fa-f]{6}$/).optional(), // Brand background color (hex format)
|
||||
|
||||
// Feature Flags
|
||||
NEXT_PUBLIC_TRIGGER_DEV_ENABLED: z.boolean().optional(), // Client-side gate for async executions UI
|
||||
NEXT_PUBLIC_SSO_ENABLED: z.boolean().optional(), // Enable SSO login UI components
|
||||
NEXT_PUBLIC_CREDENTIAL_SETS_ENABLED: z.boolean().optional(), // Enable credential sets (email polling) on self-hosted
|
||||
NEXT_PUBLIC_ACCESS_CONTROL_ENABLED: z.boolean().optional(), // Enable access control (permission groups) on self-hosted
|
||||
NEXT_PUBLIC_ORGANIZATIONS_ENABLED: z.boolean().optional(), // Enable organizations on self-hosted (bypasses plan requirements)
|
||||
NEXT_PUBLIC_DISABLE_INVITATIONS: z.boolean().optional(), // Disable workspace invitations globally (for self-hosted deployments)
|
||||
NEXT_PUBLIC_TRIGGER_DEV_ENABLED: z.boolean().optional(), // Client-side gate for async executions UI
|
||||
NEXT_PUBLIC_SSO_ENABLED: z.boolean().optional(), // Enable SSO login UI components
|
||||
NEXT_PUBLIC_CREDENTIAL_SETS_ENABLED: z.boolean().optional(), // Enable credential sets (email polling) on self-hosted
|
||||
NEXT_PUBLIC_ACCESS_CONTROL_ENABLED: z.boolean().optional(), // Enable access control (permission groups) on self-hosted
|
||||
NEXT_PUBLIC_ORGANIZATIONS_ENABLED: z.boolean().optional(), // Enable organizations on self-hosted (bypasses plan requirements)
|
||||
NEXT_PUBLIC_DISABLE_INVITATIONS: z.boolean().optional(), // Disable workspace invitations globally (for self-hosted deployments)
|
||||
NEXT_PUBLIC_EMAIL_PASSWORD_SIGNUP_ENABLED: z.boolean().optional().default(true), // Control visibility of email/password login forms
|
||||
},
|
||||
|
||||
// Variables available on both server and client
|
||||
shared: {
|
||||
NODE_ENV: z.enum(['development', 'test', 'production']).optional(), // Runtime environment
|
||||
NEXT_TELEMETRY_DISABLED: z.string().optional(), // Disable Next.js telemetry collection
|
||||
NEXT_TELEMETRY_DISABLED: z.string().optional(), // Disable Next.js telemetry collection
|
||||
},
|
||||
|
||||
experimental__runtimeEnv: {
|
||||
|
||||
193
apps/sim/lib/search/tool-operations.ts
Normal file
193
apps/sim/lib/search/tool-operations.ts
Normal file
@@ -0,0 +1,193 @@
|
||||
import type { ComponentType } from 'react'
|
||||
import { getAllBlocks } from '@/blocks'
|
||||
import type { BlockConfig, SubBlockConfig } from '@/blocks/types'
|
||||
|
||||
/**
|
||||
* Represents a searchable tool operation extracted from block configurations.
|
||||
* Each operation maps to a specific tool that can be invoked when the block
|
||||
* is configured with that operation selected.
|
||||
*/
|
||||
export interface ToolOperationItem {
|
||||
/** Unique identifier combining block type and operation ID (e.g., "slack_send") */
|
||||
id: string
|
||||
/** The block type this operation belongs to (e.g., "slack") */
|
||||
blockType: string
|
||||
/** The operation dropdown value (e.g., "send") */
|
||||
operationId: string
|
||||
/** Human-readable service name from the block (e.g., "Slack") */
|
||||
serviceName: string
|
||||
/** Human-readable operation name from the dropdown label (e.g., "Send Message") */
|
||||
operationName: string
|
||||
/** The block's icon component */
|
||||
icon: ComponentType<{ className?: string }>
|
||||
/** The block's background color */
|
||||
bgColor: string
|
||||
/** Search aliases for common synonyms */
|
||||
aliases: string[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps common action verbs to their synonyms for better search matching.
|
||||
* When a user searches for "post message", it should match "send message".
|
||||
* Based on analysis of 1000+ tool operations in the codebase.
|
||||
*/
|
||||
const ACTION_VERB_ALIASES: Record<string, string[]> = {
|
||||
get: ['read', 'fetch', 'retrieve', 'load', 'obtain'],
|
||||
read: ['get', 'fetch', 'retrieve', 'load'],
|
||||
create: ['make', 'new', 'add', 'generate', 'insert'],
|
||||
add: ['create', 'insert', 'append', 'include'],
|
||||
update: ['edit', 'modify', 'change', 'patch', 'set'],
|
||||
set: ['update', 'configure', 'assign'],
|
||||
delete: ['remove', 'trash', 'destroy', 'erase'],
|
||||
remove: ['delete', 'clear', 'drop', 'unset'],
|
||||
list: ['show', 'display', 'view', 'browse', 'enumerate'],
|
||||
search: ['find', 'query', 'lookup', 'locate'],
|
||||
query: ['search', 'find', 'lookup'],
|
||||
send: ['post', 'write', 'deliver', 'transmit', 'publish'],
|
||||
write: ['send', 'post', 'compose'],
|
||||
download: ['export', 'save', 'pull', 'fetch'],
|
||||
upload: ['import', 'push', 'transfer', 'attach'],
|
||||
execute: ['run', 'invoke', 'trigger', 'perform', 'start'],
|
||||
check: ['verify', 'validate', 'test', 'inspect'],
|
||||
cancel: ['abort', 'stop', 'terminate', 'revoke'],
|
||||
archive: ['store', 'backup', 'preserve'],
|
||||
copy: ['duplicate', 'clone', 'replicate'],
|
||||
move: ['transfer', 'relocate', 'migrate'],
|
||||
share: ['publish', 'distribute', 'broadcast'],
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates search aliases for an operation name by finding synonyms
|
||||
* for action verbs in the operation name.
|
||||
*/
|
||||
function generateAliases(operationName: string): string[] {
|
||||
const aliases: string[] = []
|
||||
const lowerName = operationName.toLowerCase()
|
||||
|
||||
for (const [verb, synonyms] of Object.entries(ACTION_VERB_ALIASES)) {
|
||||
if (lowerName.includes(verb)) {
|
||||
for (const synonym of synonyms) {
|
||||
aliases.push(lowerName.replace(verb, synonym))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return aliases
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the operation dropdown subblock from a block's configuration.
|
||||
* Returns null if no operation dropdown exists.
|
||||
*/
|
||||
function findOperationDropdown(block: BlockConfig): SubBlockConfig | null {
|
||||
return (
|
||||
block.subBlocks.find(
|
||||
(sb) => sb.id === 'operation' && sb.type === 'dropdown' && Array.isArray(sb.options)
|
||||
) ?? null
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the tool ID for a given operation using the block's tool config.
|
||||
* Falls back to checking tools.access if no config.tool function exists.
|
||||
*/
|
||||
function resolveToolId(block: BlockConfig, operationId: string): string | null {
|
||||
if (!block.tools) return null
|
||||
|
||||
if (block.tools.config?.tool) {
|
||||
try {
|
||||
return block.tools.config.tool({ operation: operationId })
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
if (block.tools.access?.length === 1) {
|
||||
return block.tools.access[0]
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds an index of all tool operations from the block registry.
|
||||
* This index is used by the search modal to enable operation-level discovery.
|
||||
*
|
||||
* The function iterates through all blocks that have:
|
||||
* 1. A tools.access array (indicating they use tools)
|
||||
* 2. An "operation" dropdown subblock with options
|
||||
*
|
||||
* For each operation option, it creates a ToolOperationItem that maps
|
||||
* the operation to its corresponding tool.
|
||||
*/
|
||||
export function buildToolOperationsIndex(): ToolOperationItem[] {
|
||||
const operations: ToolOperationItem[] = []
|
||||
const allBlocks = getAllBlocks()
|
||||
|
||||
for (const block of allBlocks) {
|
||||
if (!block.tools?.access?.length || block.hideFromToolbar) {
|
||||
continue
|
||||
}
|
||||
|
||||
if (block.category !== 'tools') {
|
||||
continue
|
||||
}
|
||||
|
||||
const operationDropdown = findOperationDropdown(block)
|
||||
if (!operationDropdown) {
|
||||
continue
|
||||
}
|
||||
|
||||
const options =
|
||||
typeof operationDropdown.options === 'function'
|
||||
? operationDropdown.options()
|
||||
: operationDropdown.options
|
||||
|
||||
if (!options) continue
|
||||
|
||||
for (const option of options) {
|
||||
if (!resolveToolId(block, option.id)) continue
|
||||
|
||||
const operationName = option.label
|
||||
const aliases = generateAliases(operationName)
|
||||
|
||||
operations.push({
|
||||
id: `${block.type}_${option.id}`,
|
||||
blockType: block.type,
|
||||
operationId: option.id,
|
||||
serviceName: block.name,
|
||||
operationName,
|
||||
icon: block.icon,
|
||||
bgColor: block.bgColor,
|
||||
aliases,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return operations
|
||||
}
|
||||
|
||||
/**
|
||||
* Cached operations index to avoid rebuilding on every search.
|
||||
* The index is built lazily on first access.
|
||||
*/
|
||||
let cachedOperations: ToolOperationItem[] | null = null
|
||||
|
||||
/**
|
||||
* Returns the tool operations index, building it if necessary.
|
||||
* The index is cached after first build since block registry is static.
|
||||
*/
|
||||
export function getToolOperationsIndex(): ToolOperationItem[] {
|
||||
if (!cachedOperations) {
|
||||
cachedOperations = buildToolOperationsIndex()
|
||||
}
|
||||
return cachedOperations
|
||||
}
|
||||
|
||||
/**
|
||||
* Clears the cached operations index.
|
||||
* Useful for testing or if blocks are dynamically modified.
|
||||
*/
|
||||
export function clearToolOperationsCache(): void {
|
||||
cachedOperations = null
|
||||
}
|
||||
Reference in New Issue
Block a user