github/sim
1
0
Fork 0
mirror of https://github.com/simstudioai/sim.git synced 2026-01-09 06:58:07 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix(vulns): fix various vulnerabilities and enhanced code security (#1611)

Browse Source 
* fix(vulns): fix SSRF vulnerabilities

* cleanup

* cleanup

* regen docs

* remove unused deps

* fix failing tests

* cleanup

* update deps

* regen bun lock
This commit is contained in:
Waleed
2025-10-11 22:14:31 -07:00
committed by GitHub
parent 1de6f09069
commit 8f06aec68b
100 changed files with 1865 additions and 1696 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
scripts/README.md
View File

@@ -54,7 +54,7 @@ The documentation generator runs automatically as part of the CI/CD pipeline whe
## Adding Support for New Block Properties
If you add new properties to block definitions that should be included in the documentation, update the `generateMarkdownForBlock` function in `scripts/generate-block-docs.ts`.
If you add new properties to block definitions that should be included in the documentation, update the `generateMarkdownForBlock` function in `scripts/generate-docs.ts`.
## Preserving Manual Content

59
scripts/generate-docs.sh
View File

@@ -1,59 +0,0 @@
#!/bin/bash
# Set error handling
set -e
# Enable debug mode if DEBUG env var is set
if [ ! -z "$DEBUG" ]; then
set -x
echo "Debug mode enabled"
fi
# Get script directories
SCRIPTS_DIR=$(dirname "$0")
ROOT_DIR=$(cd "$SCRIPTS_DIR/.." && pwd)
echo "Scripts directory: $SCRIPTS_DIR"
echo "Root directory: $ROOT_DIR"
# Check if dependencies are installed in scripts directory
if [ ! -d "$SCRIPTS_DIR/node_modules" ]; then
echo "Required dependencies not found. Installing now..."
bash "$SCRIPTS_DIR/setup-doc-generator.sh"
fi
# Generate documentation
echo "Generating block documentation..."
# Check if necessary files exist
if [ ! -f "$SCRIPTS_DIR/generate-block-docs.ts" ]; then
echo "Error: Could not find generate-block-docs.ts script"
ls -la "$SCRIPTS_DIR"
exit 1
fi
if [ ! -f "$SCRIPTS_DIR/tsconfig.json" ]; then
echo "Error: Could not find tsconfig.json in scripts directory"
exit 1
fi
# Check if npx is available
if ! command -v npx &> /dev/null; then
echo "Error: npx is not installed. Please install Node.js first."
exit 1
fi
# Change to scripts directory to use local dependencies
cd "$SCRIPTS_DIR"
echo "Executing: npx tsx ./generate-block-docs.ts"
# Run the generator with tsx using local dependencies
if ! npx tsx ./generate-block-docs.ts; then
echo ""
echo "Error running documentation generator."
echo ""
echo "For more detailed debugging, run with DEBUG=1:"
echo "DEBUG=1 ./scripts/generate-docs.sh"
exit 1
fi
echo "Documentation generation complete!"

246
scripts/generate-block-docs.ts → scripts/generate-docs.ts
View File

@@ -6,22 +6,18 @@ import { glob } from 'glob'
console.log('Starting documentation generator...')
// Define directory paths
const __filename = fileURLToPath(import.meta.url)
const __dirname = path.dirname(__filename)
const rootDir = path.resolve(__dirname, '..')
// Paths configuration
const BLOCKS_PATH = path.join(rootDir, 'apps/sim/blocks/blocks')
const DOCS_OUTPUT_PATH = path.join(rootDir, 'apps/docs/content/docs/en/tools')
const ICONS_PATH = path.join(rootDir, 'apps/sim/components/icons.tsx')
// Make sure the output directory exists
if (!fs.existsSync(DOCS_OUTPUT_PATH)) {
fs.mkdirSync(DOCS_OUTPUT_PATH, { recursive: true })
}
// Basic interface for BlockConfig to avoid import issues
interface BlockConfig {
type: string
name: string
@@ -36,39 +32,32 @@ interface BlockConfig {
[key: string]: any
}
// Function to extract SVG icons from icons.tsx file
function extractIcons(): Record<string, string> {
try {
const iconsContent = fs.readFileSync(ICONS_PATH, 'utf-8')
const icons: Record<string, string> = {}
// Match both function declaration and arrow function export patterns
const functionDeclarationRegex =
/export\s+function\s+(\w+Icon)\s*\([^)]*\)\s*{[\s\S]*?return\s*\(\s*<svg[\s\S]*?<\/svg>\s*\)/g
const arrowFunctionRegex =
/export\s+const\s+(\w+Icon)\s*=\s*\([^)]*\)\s*=>\s*(\(?\s*<svg[\s\S]*?<\/svg>\s*\)?)/g
// Extract function declaration style icons
const functionMatches = Array.from(iconsContent.matchAll(functionDeclarationRegex))
for (const match of functionMatches) {
const iconName = match[1]
const svgMatch = match[0].match(/<svg[\s\S]*?<\/svg>/)
if (iconName && svgMatch) {
// Clean the SVG to remove {...props} and standardize size
let svgContent = svgMatch[0]
svgContent = svgContent.replace(/{\.\.\.props}/g, '')
svgContent = svgContent.replace(/{\.\.\.(props|rest)}/g, '')
// Remove any existing width/height attributes to let CSS handle sizing
svgContent = svgContent.replace(/width=["'][^"']*["']/g, '')
svgContent = svgContent.replace(/height=["'][^"']*["']/g, '')
// Add className for styling
svgContent = svgContent.replace(/<svg/, '<svg className="block-icon"')
icons[iconName] = svgContent
}
}
// Extract arrow function style icons
const arrowMatches = Array.from(iconsContent.matchAll(arrowFunctionRegex))
for (const match of arrowMatches) {
const iconName = match[1]
@@ -76,14 +65,11 @@ function extractIcons(): Record<string, string> {
const svgMatch = svgContent.match(/<svg[\s\S]*?<\/svg>/)
if (iconName && svgMatch) {
// Clean the SVG to remove {...props} and standardize size
let cleanedSvg = svgMatch[0]
cleanedSvg = cleanedSvg.replace(/{\.\.\.props}/g, '')
cleanedSvg = cleanedSvg.replace(/{\.\.\.(props|rest)}/g, '')
// Remove any existing width/height attributes to let CSS handle sizing
cleanedSvg = cleanedSvg.replace(/width=["'][^"']*["']/g, '')
cleanedSvg = cleanedSvg.replace(/height=["'][^"']*["']/g, '')
// Add className for styling
cleanedSvg = cleanedSvg.replace(/<svg/, '<svg className="block-icon"')
icons[iconName] = cleanedSvg
}
@@ -95,10 +81,8 @@ function extractIcons(): Record<string, string> {
}
}
// Function to extract block configuration from file content
function extractBlockConfig(fileContent: string): BlockConfig | null {
try {
// Extract the block name from export statement
const exportMatch = fileContent.match(/export\s+const\s+(\w+)Block\s*:/)
if (!exportMatch) {
@@ -109,7 +93,6 @@ function extractBlockConfig(fileContent: string): BlockConfig | null {
const blockName = exportMatch[1]
const blockType = findBlockType(fileContent, blockName)
// Extract individual properties with more robust regex
const name = extractStringProperty(fileContent, 'name') || `${blockName} Block`
const description = extractStringProperty(fileContent, 'description') || ''
const longDescription = extractStringProperty(fileContent, 'longDescription') || ''
@@ -117,10 +100,8 @@ function extractBlockConfig(fileContent: string): BlockConfig | null {
const bgColor = extractStringProperty(fileContent, 'bgColor') || '#F5F5F5'
const iconName = extractIconName(fileContent) || ''
// Extract outputs object with better handling
const outputs = extractOutputs(fileContent)
// Extract tools access array
const toolsAccess = extractToolsAccess(fileContent)
return {
@@ -142,10 +123,7 @@ function extractBlockConfig(fileContent: string): BlockConfig | null {
}
}
// Helper function to find the block type
function findBlockType(content: string, blockName: string): string {
// Try to find the type within the main block export
// Look for the pattern: export const [BlockName]Block: BlockConfig = { ... type: 'value' ... }
const blockExportRegex = new RegExp(
`export\\s+const\\s+${blockName}Block\\s*:[^{]*{[\\s\\S]*?type\\s*:\\s*['"]([^'"]+)['"][\\s\\S]*?}`,
'i'
@@ -153,18 +131,14 @@ function findBlockType(content: string, blockName: string): string {
const blockExportMatch = content.match(blockExportRegex)
if (blockExportMatch) return blockExportMatch[1]
// Fallback: try to find type within a block config object that comes after the export
const exportMatch = content.match(new RegExp(`export\\s+const\\s+${blockName}Block\\s*:`))
if (exportMatch) {
// Find the content after the export statement
const afterExport = content.substring(exportMatch.index! + exportMatch[0].length)
// Look for the first opening brace and then find type within that block
const blockStartMatch = afterExport.match(/{/)
if (blockStartMatch) {
const blockStart = blockStartMatch.index!
// Find the matching closing brace by counting braces
let braceCount = 1
let blockEnd = blockStart + 1
@@ -174,47 +148,37 @@ function findBlockType(content: string, blockName: string): string {
blockEnd++
}
// Extract the block content and look for type
const blockContent = afterExport.substring(blockStart, blockEnd)
const typeMatch = blockContent.match(/type\s*:\s*['"]([^'"]+)['"]/)
if (typeMatch) return typeMatch[1]
}
}
// Convert CamelCase to snake_case as fallback
return blockName
.replace(/([A-Z])/g, '_$1')
.toLowerCase()
.replace(/^_/, '')
}
// Helper to extract a string property from content
function extractStringProperty(content: string, propName: string): string | null {
// Try single quotes first - more permissive approach
const singleQuoteMatch = content.match(new RegExp(`${propName}\\s*:\\s*'(.*?)'`, 'm'))
if (singleQuoteMatch) return singleQuoteMatch[1]
// Try double quotes
const doubleQuoteMatch = content.match(new RegExp(`${propName}\\s*:\\s*"(.*?)"`, 'm'))
if (doubleQuoteMatch) return doubleQuoteMatch[1]
// Try to match multi-line string with template literals
const templateMatch = content.match(new RegExp(`${propName}\\s*:\\s*\`([^\`]+)\``, 's'))
if (templateMatch) {
let templateContent = templateMatch[1]
// Handle template literals with expressions by replacing them with reasonable defaults
// This is a simple approach - we'll replace common variable references with sensible defaults
templateContent = templateContent.replace(
/\$\{[^}]*shouldEnableURLInput[^}]*\?[^:]*:[^}]*\}/g,
'Upload files directly. '
)
templateContent = templateContent.replace(/\$\{[^}]*shouldEnableURLInput[^}]*\}/g, 'false')
// Remove any remaining template expressions that we can't safely evaluate
templateContent = templateContent.replace(/\$\{[^}]+\}/g, '')
// Clean up any extra whitespace
templateContent = templateContent.replace(/\s+/g, ' ').trim()
return templateContent
@@ -223,23 +187,18 @@ function extractStringProperty(content: string, propName: string): string | null
return null
}
// Helper to extract icon name from content
function extractIconName(content: string): string | null {
const iconMatch = content.match(/icon\s*:\s*(\w+Icon)/)
return iconMatch ? iconMatch[1] : null
}
// Updated function to extract outputs with a simpler and more reliable approach
function extractOutputs(content: string): Record<string, any> {
// Look for the outputs section using balanced brace matching
const outputsStart = content.search(/outputs\s*:\s*{/)
if (outputsStart === -1) return {}
// Find the opening brace position
const openBracePos = content.indexOf('{', outputsStart)
if (openBracePos === -1) return {}
// Use balanced brace counting to find the complete outputs section
let braceCount = 1
let pos = openBracePos + 1
@@ -256,27 +215,22 @@ function extractOutputs(content: string): Record<string, any> {
const outputsContent = content.substring(openBracePos + 1, pos - 1).trim()
const outputs: Record<string, any> = {}
// First try to handle the new object format: fieldName: { type: 'type', description: 'desc' }
// Use a more robust approach to extract field definitions
const fieldRegex = /(\w+)\s*:\s*{/g
let match
const fieldPositions: Array<{ name: string; start: number }> = []
// Find all field starting positions
while ((match = fieldRegex.exec(outputsContent)) !== null) {
fieldPositions.push({
name: match[1],
start: match.index + match[0].length - 1, // Position of the opening brace
start: match.index + match[0].length - 1,
})
}
// Extract each field's content by finding balanced braces
fieldPositions.forEach((field) => {
const startPos = field.start
let braceCount = 1
let endPos = startPos + 1
// Find the matching closing brace
while (endPos < outputsContent.length && braceCount > 0) {
if (outputsContent[endPos] === '{') {
braceCount++
@@ -287,10 +241,8 @@ function extractOutputs(content: string): Record<string, any> {
}
if (braceCount === 0) {
// Extract the content between braces
const fieldContent = outputsContent.substring(startPos + 1, endPos - 1).trim()
// Extract type and description from the object
const typeMatch = fieldContent.match(/type\s*:\s*['"](.*?)['"]/)
const descriptionMatch = fieldContent.match(/description\s*:\s*['"](.*?)['"]/)
@@ -305,12 +257,10 @@ function extractOutputs(content: string): Record<string, any> {
}
})
// If we found object fields, return them
if (Object.keys(outputs).length > 0) {
return outputs
}
// Fallback: try to handle the old flat format: fieldName: 'type'
const flatFieldMatches = outputsContent.match(/(\w+)\s*:\s*['"](.*?)['"]/g)
if (flatFieldMatches && flatFieldMatches.length > 0) {
@@ -327,7 +277,6 @@ function extractOutputs(content: string): Record<string, any> {
}
})
// If we found flat fields, return them
if (Object.keys(outputs).length > 0) {
return outputs
}
@@ -337,9 +286,8 @@ function extractOutputs(content: string): Record<string, any> {
return {}
}
// Helper to extract tools access array
function extractToolsAccess(content: string): string[] {
const accessMatch = content.match(/access\s*:\s*\[\s*((?:['"][^'"]+['"](?:\s*,\s*)?)+)\s*\]/)
const accessMatch = content.match(/access\s*:\s*\[\s*([^\]]+)\s*\]/)
if (!accessMatch) return []
const accessContent = accessMatch[1]
@@ -358,7 +306,6 @@ function extractToolsAccess(content: string): string[] {
return tools
}
// Function to extract tool information from file content
function extractToolInfo(
toolName: string,
fileContent: string
@@ -368,33 +315,27 @@ function extractToolInfo(
outputs: Record<string, any>
} | null {
try {
// Extract tool config section - Match params until the next top-level property
const toolConfigRegex =
/params\s*:\s*{([\s\S]*?)},?\s*(?:outputs|oauth|request|directExecution|postProcess|transformResponse)/
const toolConfigMatch = fileContent.match(toolConfigRegex)
// Extract description
const descriptionRegex = /description\s*:\s*['"](.*?)['"].*/
const descriptionMatch = fileContent.match(descriptionRegex)
const description = descriptionMatch ? descriptionMatch[1] : 'No description available'
// Parse parameters
const params: Array<{ name: string; type: string; required: boolean; description: string }> = []
if (toolConfigMatch) {
const paramsContent = toolConfigMatch[1]
// More robust approach to extract parameters with balanced brace matching
// Extract each parameter block completely
const paramBlocksRegex = /(\w+)\s*:\s*{/g
let paramMatch
const paramPositions: Array<{ name: string; start: number; content: string }> = []
while ((paramMatch = paramBlocksRegex.exec(paramsContent)) !== null) {
const paramName = paramMatch[1]
const startPos = paramMatch.index + paramMatch[0].length - 1 // Position of opening brace
const startPos = paramMatch.index + paramMatch[0].length - 1
// Find matching closing brace using balanced counting
let braceCount = 1
let endPos = startPos + 1
@@ -417,27 +358,21 @@ function extractToolInfo(
const paramName = param.name
const paramBlock = param.content
// Skip the accessToken parameter as it's handled automatically by the OAuth flow
// Also skip any params parameter which isn't a real input
if (paramName === 'accessToken' || paramName === 'params' || paramName === 'tools') {
continue
}
// Extract param details with more robust patterns
const typeMatch = paramBlock.match(/type\s*:\s*['"]([^'"]+)['"]/)
const requiredMatch = paramBlock.match(/required\s*:\s*(true|false)/)
// More careful extraction of description with handling for multiline descriptions
let descriptionMatch = paramBlock.match(/description\s*:\s*'(.*?)'(?=\s*[,}])/s)
if (!descriptionMatch) {
descriptionMatch = paramBlock.match(/description\s*:\s*"(.*?)"(?=\s*[,}])/s)
}
if (!descriptionMatch) {
// Try for template literals if the description uses backticks
descriptionMatch = paramBlock.match(/description\s*:\s*`([^`]+)`/s)
}
if (!descriptionMatch) {
// Handle multi-line descriptions without ending quote on same line
descriptionMatch = paramBlock.match(
/description\s*:\s*['"]([^'"]*(?:\n[^'"]*)*?)['"](?=\s*[,}])/s
)
@@ -452,7 +387,6 @@ function extractToolInfo(
}
}
// First priority: Extract outputs from the new outputs field in ToolConfig
let outputs: Record<string, any> = {}
const outputsFieldRegex =
/outputs\s*:\s*{([\s\S]*?)}\s*,?\s*(?:oauth|params|request|directExecution|postProcess|transformResponse|$|\})/
@@ -475,7 +409,6 @@ function extractToolInfo(
}
}
// Helper function to recursively format output structure for documentation
function formatOutputStructure(outputs: Record<string, any>, indentLevel = 0): string {
let result = ''
@@ -493,7 +426,6 @@ function formatOutputStructure(outputs: Record<string, any>, indentLevel = 0): s
}
}
// Escape special characters in the description
const escapedDescription = description
.replace(/\|/g, '\\|')
.replace(/\{/g, '\\{')
@@ -505,28 +437,21 @@ function formatOutputStructure(outputs: Record<string, any>, indentLevel = 0): s
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
// Create prefix based on nesting level with visual hierarchy
let prefix = ''
if (indentLevel === 1) {
prefix = '↳ '
} else if (indentLevel >= 2) {
// For deeper nesting (like array items), use indented arrows
prefix = ' ↳ '
}
// For arrays, expand nested items
if (typeof output === 'object' && output !== null && output.type === 'array') {
result += `| ${prefix}\`${key}\` | ${type} | ${escapedDescription} |\n`
// Handle array items with properties (nested TWO more levels to show it's inside the array)
if (output.items?.properties) {
// Create a visual separator to show these are array item properties
const arrayItemsResult = formatOutputStructure(output.items.properties, indentLevel + 2)
result += arrayItemsResult
}
}
// For objects, expand properties
else if (
} else if (
typeof output === 'object' &&
output !== null &&
output.properties &&
@@ -536,9 +461,7 @@ function formatOutputStructure(outputs: Record<string, any>, indentLevel = 0): s
const nestedResult = formatOutputStructure(output.properties, indentLevel + 1)
result += nestedResult
}
// For simple types, show with prefix if nested
else {
} else {
result += `| ${prefix}\`${key}\` | ${type} | ${escapedDescription} |\n`
}
}
@@ -546,11 +469,9 @@ function formatOutputStructure(outputs: Record<string, any>, indentLevel = 0): s
return result
}
// New function to parse the structured outputs field from ToolConfig
function parseToolOutputsField(outputsContent: string): Record<string, any> {
const outputs: Record<string, any> = {}
// Calculate nesting levels for all braces first
const braces: Array<{ type: 'open' | 'close'; pos: number; level: number }> = []
for (let i = 0; i < outputsContent.length; i++) {
if (outputsContent[i] === '{') {
@@ -560,7 +481,6 @@ function parseToolOutputsField(outputsContent: string): Record<string, any> {
}
}
// Calculate actual nesting levels
let currentLevel = 0
for (const brace of braces) {
if (brace.type === 'open') {
@@ -572,7 +492,6 @@ function parseToolOutputsField(outputsContent: string): Record<string, any> {
}
}
// Find field definitions and their nesting levels
const fieldStartRegex = /(\w+)\s*:\s*{/g
let match
const fieldPositions: Array<{ name: string; start: number; end: number; level: number }> = []
@@ -581,10 +500,8 @@ function parseToolOutputsField(outputsContent: string): Record<string, any> {
const fieldName = match[1]
const bracePos = match.index + match[0].length - 1
// Find the corresponding opening brace to determine nesting level
const openBrace = braces.find((b) => b.type === 'open' && b.pos === bracePos)
if (openBrace) {
// Find the matching closing brace
let braceCount = 1
let endPos = bracePos + 1
@@ -606,13 +523,11 @@ function parseToolOutputsField(outputsContent: string): Record<string, any> {
}
}
// Only process level 0 fields (top-level outputs)
const topLevelFields = fieldPositions.filter((f) => f.level === 0)
topLevelFields.forEach((field) => {
const fieldContent = outputsContent.substring(field.start + 1, field.end - 1).trim()
// Parse the field content
const parsedField = parseFieldContent(fieldContent)
if (parsedField) {
outputs[field.name] = parsedField
@@ -622,9 +537,7 @@ function parseToolOutputsField(outputsContent: string): Record<string, any> {
return outputs
}
// Helper function to parse individual field content with support for nested structures
function parseFieldContent(fieldContent: string): any {
// Extract type and description
const typeMatch = fieldContent.match(/type\s*:\s*['"]([^'"]+)['"]/)
const descMatch = fieldContent.match(/description\s*:\s*['"`]([^'"`\n]+)['"`]/)
@@ -638,7 +551,6 @@ function parseFieldContent(fieldContent: string): any {
description: description,
}
// Check for properties (nested objects) - only for object types, not arrays
if (fieldType === 'object' || fieldType === 'json') {
const propertiesRegex = /properties\s*:\s*{/
const propertiesStart = fieldContent.search(propertiesRegex)
@@ -648,7 +560,6 @@ function parseFieldContent(fieldContent: string): any {
let braceCount = 1
let braceEnd = braceStart + 1
// Find matching closing brace
while (braceEnd < fieldContent.length && braceCount > 0) {
if (fieldContent[braceEnd] === '{') braceCount++
else if (fieldContent[braceEnd] === '}') braceCount--
@@ -662,7 +573,6 @@ function parseFieldContent(fieldContent: string): any {
}
}
// Check for items (array items) - ensure balanced brace matching
const itemsRegex = /items\s*:\s*{/
const itemsStart = fieldContent.search(itemsRegex)
@@ -671,7 +581,6 @@ function parseFieldContent(fieldContent: string): any {
let braceCount = 1
let braceEnd = braceStart + 1
// Find matching closing brace
while (braceEnd < fieldContent.length && braceCount > 0) {
if (fieldContent[braceEnd] === '{') braceCount++
else if (fieldContent[braceEnd] === '}') braceCount--
@@ -682,7 +591,6 @@ function parseFieldContent(fieldContent: string): any {
const itemsContent = fieldContent.substring(braceStart + 1, braceEnd - 1).trim()
const itemsType = itemsContent.match(/type\s*:\s*['"]([^'"]+)['"]/)
// Only look for description before any properties block to avoid picking up nested property descriptions
const propertiesStart = itemsContent.search(/properties\s*:\s*{/)
const searchContent =
propertiesStart >= 0 ? itemsContent.substring(0, propertiesStart) : itemsContent
@@ -693,7 +601,6 @@ function parseFieldContent(fieldContent: string): any {
description: itemsDesc ? itemsDesc[1] : '',
}
// Check if items have properties
const itemsPropertiesRegex = /properties\s*:\s*{/
const itemsPropsStart = itemsContent.search(itemsPropertiesRegex)
@@ -721,11 +628,9 @@ function parseFieldContent(fieldContent: string): any {
return result
}
// Helper function to parse properties content recursively
function parsePropertiesContent(propertiesContent: string): Record<string, any> {
const properties: Record<string, any> = {}
// Find property definitions using balanced brace matching, but exclude type-only definitions
const propStartRegex = /(\w+)\s*:\s*{/g
let match
const propPositions: Array<{ name: string; start: number; content: string }> = []
@@ -733,14 +638,12 @@ function parsePropertiesContent(propertiesContent: string): Record<string, any>
while ((match = propStartRegex.exec(propertiesContent)) !== null) {
const propName = match[1]
// Skip structural keywords that should never be treated as property names
if (propName === 'items' || propName === 'properties') {
continue
}
const startPos = match.index + match[0].length - 1 // Position of opening brace
const startPos = match.index + match[0].length - 1
// Find the matching closing brace
let braceCount = 1
let endPos = startPos + 1
@@ -756,9 +659,6 @@ function parsePropertiesContent(propertiesContent: string): Record<string, any>
if (braceCount === 0) {
const propContent = propertiesContent.substring(startPos + 1, endPos - 1).trim()
// Skip if this is just a type definition (contains only 'type' field) rather than a real property
// This happens with array items definitions like: items: { type: 'string' }
// More precise check: only skip if it ONLY has 'type' and nothing else meaningful
const hasDescription = /description\s*:\s*/.test(propContent)
const hasProperties = /properties\s*:\s*{/.test(propContent)
const hasItems = /items\s*:\s*{/.test(propContent)
@@ -778,7 +678,6 @@ function parsePropertiesContent(propertiesContent: string): Record<string, any>
}
}
// Process the actual property definitions
propPositions.forEach((prop) => {
const parsedProp = parseFieldContent(prop.content)
if (parsedProp) {
@@ -789,26 +688,21 @@ function parsePropertiesContent(propertiesContent: string): Record<string, any>
return properties
}
// Find and extract information about a tool
async function getToolInfo(toolName: string): Promise<{
description: string
params: Array<{ name: string; type: string; required: boolean; description: string }>
outputs: Record<string, any>
} | null> {
try {
// Split the tool name into parts
const parts = toolName.split('_')
// Try to find the correct split point by checking if directories exist
let toolPrefix = ''
let toolSuffix = ''
// Start from the longest possible prefix and work backwards
for (let i = parts.length - 1; i >= 1; i--) {
const possiblePrefix = parts.slice(0, i).join('_')
const possibleSuffix = parts.slice(i).join('_')
// Check if a directory exists for this prefix
const toolDirPath = path.join(rootDir, `apps/sim/tools/${possiblePrefix}`)
if (fs.existsSync(toolDirPath) && fs.statSync(toolDirPath).isDirectory()) {
@@ -818,29 +712,23 @@ async function getToolInfo(toolName: string): Promise<{
}
}
// If no directory was found, fall back to single-part prefix
if (!toolPrefix) {
toolPrefix = parts[0]
toolSuffix = parts.slice(1).join('_')
}
// Simplify the file search strategy
const possibleLocations = []
// Most common pattern: suffix.ts file in the prefix directory
possibleLocations.push(path.join(rootDir, `apps/sim/tools/${toolPrefix}/${toolSuffix}.ts`))
// Try camelCase version of suffix
const camelCaseSuffix = toolSuffix
.split('_')
.map((part, i) => (i === 0 ? part : part.charAt(0).toUpperCase() + part.slice(1)))
.join('')
possibleLocations.push(path.join(rootDir, `apps/sim/tools/${toolPrefix}/${camelCaseSuffix}.ts`))
// Also check the index.ts file in the tool directory
possibleLocations.push(path.join(rootDir, `apps/sim/tools/${toolPrefix}/index.ts`))
// Try to find the tool definition file
let toolFileContent = ''
for (const location of possibleLocations) {
@@ -855,7 +743,6 @@ async function getToolInfo(toolName: string): Promise<{
return null
}
// Extract tool information from the file
return extractToolInfo(toolName, toolFileContent)
} catch (error) {
console.error(`Error getting info for tool ${toolName}:`, error)
@@ -863,10 +750,8 @@ async function getToolInfo(toolName: string): Promise<{
}
}
// Function to extract content between manual content markers
function extractManualContent(existingContent: string): Record<string, string> {
const manualSections: Record<string, string> = {}
// Improved regex to better handle MDX comments
const manualContentRegex =
/\{\/\*\s*MANUAL-CONTENT-START:(\w+)\s*\*\/\}([\s\S]*?)\{\/\*\s*MANUAL-CONTENT-END\s*\*\/\}/g
@@ -881,7 +766,6 @@ function extractManualContent(existingContent: string): Record<string, string> {
return manualSections
}
// Function to merge generated markdown with manual content
function mergeWithManualContent(
generatedMarkdown: string,
existingContent: string | null,
@@ -893,18 +777,14 @@ function mergeWithManualContent(
console.log('Merging manual content with generated markdown')
// Log what we found for debugging
console.log(`Found ${Object.keys(manualSections).length} manual sections`)
Object.keys(manualSections).forEach((section) => {
console.log(` - ${section}: ${manualSections[section].substring(0, 20)}...`)
})
// Replace placeholders in generated markdown with manual content
let mergedContent = generatedMarkdown
// Add manual content for each section we found
Object.entries(manualSections).forEach(([sectionName, content]) => {
// Define insertion points for different section types with improved patterns
const insertionPoints: Record<string, { regex: RegExp }> = {
intro: {
regex: /<BlockInfoCard[\s\S]*?<\/svg>`}\s*\/>/,
@@ -920,15 +800,12 @@ function mergeWithManualContent(
},
}
// Find the appropriate insertion point
const insertionPoint = insertionPoints[sectionName]
if (insertionPoint) {
// Use regex to find the insertion point
const match = mergedContent.match(insertionPoint.regex)
if (match && match.index !== undefined) {
// Insert after the matched content
const insertPosition = match.index + match[0].length
console.log(`Inserting ${sectionName} content after position ${insertPosition}`)
mergedContent = `${mergedContent.slice(0, insertPosition)}\n\n{/* MANUAL-CONTENT-START:${sectionName} */}\n${content}\n{/* MANUAL-CONTENT-END */}\n${mergedContent.slice(insertPosition)}`
@@ -945,19 +822,15 @@ function mergeWithManualContent(
return mergedContent
}
// Function to generate documentation for a block
async function generateBlockDoc(blockPath: string, icons: Record<string, string>) {
try {
// Extract the block name from the file path
const blockFileName = path.basename(blockPath, '.ts')
if (blockFileName.endsWith('.test')) {
return // Skip test files
return
}
// Read the file content
const fileContent = fs.readFileSync(blockPath, 'utf-8')
// Extract block configuration from the file content
const blockConfig = extractBlockConfig(fileContent)
if (!blockConfig || !blockConfig.type) {
@@ -965,7 +838,11 @@ async function generateBlockDoc(blockPath: string, icons: Record<string, string>
return
}
// Skip blocks with category 'blocks' (except memory type), and skip specific blocks
if (blockConfig.type.includes('_trigger')) {
console.log(`Skipping ${blockConfig.type} - contains '_trigger'`)
return
}
if (
(blockConfig.category === 'blocks' &&
blockConfig.type !== 'memory' &&
@@ -976,23 +853,18 @@ async function generateBlockDoc(blockPath: string, icons: Record<string, string>
return
}
// Output file path
const outputFilePath = path.join(DOCS_OUTPUT_PATH, `${blockConfig.type}.mdx`)
// IMPORTANT: Check if file already exists and read its content FIRST
let existingContent: string | null = null
if (fs.existsSync(outputFilePath)) {
existingContent = fs.readFileSync(outputFilePath, 'utf-8')
console.log(`Existing file found for ${blockConfig.type}.mdx, checking for manual content...`)
}
// Extract manual content from existing file before generating new content
const manualSections = existingContent ? extractManualContent(existingContent) : {}
// Create the markdown content - now async
const markdown = await generateMarkdownForBlock(blockConfig, icons)
// Merge with manual content if we found any
let finalContent = markdown
if (Object.keys(manualSections).length > 0) {
console.log(`Found manual content in ${blockConfig.type}.mdx, merging...`)
@@ -1001,7 +873,6 @@ async function generateBlockDoc(blockPath: string, icons: Record<string, string>
console.log(`No manual content found in ${blockConfig.type}.mdx`)
}
// Write the markdown file
fs.writeFileSync(outputFilePath, finalContent)
console.log(`Generated documentation for ${blockConfig.type}`)
} catch (error) {
@@ -1009,7 +880,6 @@ async function generateBlockDoc(blockPath: string, icons: Record<string, string>
}
}
// Update generateMarkdownForBlock to remove placeholders
async function generateMarkdownForBlock(
blockConfig: BlockConfig,
icons: Record<string, string>
@@ -1026,48 +896,39 @@ async function generateMarkdownForBlock(
tools = { access: [] },
} = blockConfig
// Get SVG icon if available
const iconSvg = iconName && icons[iconName] ? icons[iconName] : null
// Generate the outputs section
let outputsSection = ''
if (outputs && Object.keys(outputs).length > 0) {
outputsSection = '## Outputs\n\n'
// Create the base outputs table
outputsSection += '| Output | Type | Description |\n'
outputsSection += '| ------ | ---- | ----------- |\n'
// Process each output field
for (const outputKey in outputs) {
const output = outputs[outputKey]
// Escape special characters in the description that could break markdown tables
const escapedDescription = output.description
? output.description
.replace(/\|/g, '\\|') // Escape pipe characters
.replace(/\{/g, '\\{') // Escape curly braces
.replace(/\}/g, '\\}') // Escape curly braces
.replace(/\(/g, '\\(') // Escape opening parentheses
.replace(/\)/g, '\\)') // Escape closing parentheses
.replace(/\[/g, '\\[') // Escape opening brackets
.replace(/\]/g, '\\]') // Escape closing brackets
.replace(/</g, '&lt;') // Convert less than to HTML entity
.replace(/>/g, '&gt;') // Convert greater than to HTML entity
.replace(/\|/g, '\\|')
.replace(/\{/g, '\\{')
.replace(/\}/g, '\\}')
.replace(/\(/g, '\\(')
.replace(/\)/g, '\\)')
.replace(/\[/g, '\\[')
.replace(/\]/g, '\\]')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
: `Output from ${outputKey}`
if (typeof output.type === 'string') {
// Simple output with explicit type
outputsSection += `| \`${outputKey}\` | ${output.type} | ${escapedDescription} |\n`
} else if (output.type && typeof output.type === 'object') {
// For cases where output.type is an object containing field types
outputsSection += `| \`${outputKey}\` | object | ${escapedDescription} |\n`
// Add properties directly to the main table with indentation
for (const propName in output.type) {
const propType = output.type[propName]
// Get description from comments if available
const commentMatch =
propName && output.type[propName]._comment
? output.type[propName]._comment
@@ -1076,24 +937,21 @@ async function generateMarkdownForBlock(
outputsSection += `| ↳ \`${propName}\` | ${propType} | ${commentMatch} |\n`
}
} else if (output.properties) {
// Complex output with properties
outputsSection += `| \`${outputKey}\` | object | ${escapedDescription} |\n`
// Add properties directly to the main table with indentation
for (const propName in output.properties) {
const prop = output.properties[propName]
// Escape special characters in the description
const escapedPropertyDescription = prop.description
? prop.description
.replace(/\|/g, '\\|') // Escape pipe characters
.replace(/\{/g, '\\{') // Escape curly braces
.replace(/\}/g, '\\}') // Escape curly braces
.replace(/\(/g, '\\(') // Escape opening parentheses
.replace(/\)/g, '\\)') // Escape closing parentheses
.replace(/\[/g, '\\[') // Escape opening brackets
.replace(/\]/g, '\\]') // Escape closing brackets
.replace(/</g, '&lt;') // Convert less than to HTML entity
.replace(/>/g, '&gt;') // Convert greater than to HTML entity
.replace(/\|/g, '\\|')
.replace(/\{/g, '\\{')
.replace(/\}/g, '\\}')
.replace(/\(/g, '\\(')
.replace(/\)/g, '\\)')
.replace(/\[/g, '\\[')
.replace(/\]/g, '\\]')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
: `The ${propName} of the ${outputKey}`
outputsSection += `| ↳ \`${propName}\` | ${prop.type} | ${escapedPropertyDescription} |\n`
@@ -1104,16 +962,14 @@ async function generateMarkdownForBlock(
outputsSection = 'This block does not produce any outputs.'
}
// Create tools section with more details
let toolsSection = ''
if (tools.access?.length) {
toolsSection = '## Tools\n\n'
// For each tool, try to find its definition and extract parameter information
for (const tool of tools.access) {
toolsSection += `### \`${tool}\`\n\n`
// Get dynamic tool information
console.log(`Getting info for tool: ${tool}`)
const toolInfo = await getToolInfo(tool)
if (toolInfo) {
@@ -1121,45 +977,37 @@ async function generateMarkdownForBlock(
toolsSection += `${toolInfo.description}\n\n`
}
// Add Input Parameters section for the tool
toolsSection += '#### Input\n\n'
toolsSection += '| Parameter | Type | Required | Description |\n'
toolsSection += '| --------- | ---- | -------- | ----------- |\n'
if (toolInfo.params.length > 0) {
// Use dynamically extracted parameters
for (const param of toolInfo.params) {
// Escape special characters in the description that could break markdown tables
const escapedDescription = param.description
? param.description
.replace(/\|/g, '\\|') // Escape pipe characters
.replace(/\{/g, '\\{') // Escape curly braces
.replace(/\}/g, '\\}') // Escape curly braces
.replace(/\(/g, '\\(') // Escape opening parentheses
.replace(/\)/g, '\\)') // Escape closing parentheses
.replace(/\[/g, '\\[') // Escape opening brackets
.replace(/\]/g, '\\]') // Escape closing brackets
.replace(/</g, '&lt;') // Convert less than to HTML entity
.replace(/>/g, '&gt;') // Convert greater than to HTML entity
.replace(/\|/g, '\\|')
.replace(/\{/g, '\\{')
.replace(/\}/g, '\\}')
.replace(/\(/g, '\\(')
.replace(/\)/g, '\\)')
.replace(/\[/g, '\\[')
.replace(/\]/g, '\\]')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
: 'No description'
toolsSection += `| \`${param.name}\` | ${param.type} | ${param.required ? 'Yes' : 'No'} | ${escapedDescription} |\n`
}
}
// Add Output Parameters section for the tool
toolsSection += '\n#### Output\n\n'
// Always prefer tool-specific outputs over block outputs for accuracy
if (Object.keys(toolInfo.outputs).length > 0) {
// Use tool-specific outputs (most accurate)
toolsSection += '| Parameter | Type | Description |\n'
toolsSection += '| --------- | ---- | ----------- |\n'
// Use the enhanced formatOutputStructure function to handle nested structures
toolsSection += formatOutputStructure(toolInfo.outputs)
} else if (Object.keys(outputs).length > 0) {
// Fallback to block outputs only if no tool outputs are available
toolsSection += '| Parameter | Type | Description |\n'
toolsSection += '| --------- | ---- | ----------- |\n'
@@ -1178,7 +1026,6 @@ async function generateMarkdownForBlock(
}
}
// Escape special characters in the description
const escapedDescription = description
.replace(/\|/g, '\\|')
.replace(/\{/g, '\\{')
@@ -1201,13 +1048,11 @@ async function generateMarkdownForBlock(
}
}
// Add usage instructions if available in block config
let usageInstructions = ''
if (longDescription) {
usageInstructions = `## Usage Instructions\n\n${longDescription}\n\n`
}
// Generate the markdown content without any placeholders
return `---
title: ${name}
description: ${description}
@@ -1233,21 +1078,16 @@ ${toolsSection}
`
}
// Main function to generate all block docs
async function generateAllBlockDocs() {
try {
// Extract icons first
const icons = extractIcons()
// Get all block files
const blockFiles = await glob(`${BLOCKS_PATH}/*.ts`)
// Generate docs for each block
for (const blockFile of blockFiles) {
await generateBlockDoc(blockFile, icons)
}
// Update the meta.json file
updateMetaJson()
return true
@@ -1257,18 +1097,14 @@ async function generateAllBlockDocs() {
}
}
// Function to update the meta.json file with all blocks
function updateMetaJson() {
const metaJsonPath = path.join(DOCS_OUTPUT_PATH, 'meta.json')
// Get all MDX files in the tools directory
const blockFiles = fs
.readdirSync(DOCS_OUTPUT_PATH)
.filter((file: string) => file.endsWith('.mdx'))
.map((file: string) => path.basename(file, '.mdx'))
// Create meta.json structure
// Keep "index" as the first item if it exists
const items = [
...(blockFiles.includes('index') ? ['index'] : []),
...blockFiles.filter((file: string) => file !== 'index').sort(),
@@ -1278,11 +1114,9 @@ function updateMetaJson() {
items,
}
// Write the meta.json file
fs.writeFileSync(metaJsonPath, JSON.stringify(metaJson, null, 2))
}
// Run the script
generateAllBlockDocs()
.then((success) => {
if (success) {