mirror of
https://github.com/simstudioai/sim.git
synced 2026-01-14 17:37:55 -05:00
Compare commits
65 Commits
feat/group
...
improvemen
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2e4945508c | ||
|
|
1feef4ce4b | ||
|
|
2404f8af14 | ||
|
|
ba72e35d43 | ||
|
|
ab52458191 | ||
|
|
ef5e2b699c | ||
|
|
ba45404423 | ||
|
|
4ce2fc760a | ||
|
|
71e06f2b31 | ||
|
|
ce04d56d79 | ||
|
|
626e9a37da | ||
|
|
100ae1d23e | ||
|
|
792df1a9f0 | ||
|
|
d313a0f171 | ||
|
|
29270d15ff | ||
|
|
879711d786 | ||
|
|
f98138a550 | ||
|
|
888609a93c | ||
|
|
a516325733 | ||
|
|
396c9db204 | ||
|
|
a1acbc9616 | ||
|
|
5a74ab28e2 | ||
|
|
cf5532c852 | ||
|
|
3e6d454de3 | ||
|
|
4c4b3351e6 | ||
|
|
0c1ee239fe | ||
|
|
9c065a1c2a | ||
|
|
dc92a79f33 | ||
|
|
efb0d22d3f | ||
|
|
9af445fa25 | ||
|
|
e09088bafc | ||
|
|
994c81ba3c | ||
|
|
4bba1eb8f6 | ||
|
|
de06e8c35c | ||
|
|
61534b05dd | ||
|
|
694538e1ee | ||
|
|
6df565e4c8 | ||
|
|
422df2be0f | ||
|
|
692b385ece | ||
|
|
728f5812ac | ||
|
|
1d51706057 | ||
|
|
c166c60d9b | ||
|
|
120b7ffd5c | ||
|
|
ecc2a55f9e | ||
|
|
f10b7c0493 | ||
|
|
5835df3496 | ||
|
|
c4924776b6 | ||
|
|
9444661d98 | ||
|
|
9137b2eab3 | ||
|
|
fa0ef07981 | ||
|
|
463ba208f4 | ||
|
|
2cedac5ffb | ||
|
|
9c3f559a91 | ||
|
|
0d7ab06bd1 | ||
|
|
7bda4468b8 | ||
|
|
bdb9b866ab | ||
|
|
460d515df2 | ||
|
|
7b49515798 | ||
|
|
049f188d2e | ||
|
|
6c8a7f0594 | ||
|
|
3727b5d395 | ||
|
|
75a3f4cce7 | ||
|
|
63616a1239 | ||
|
|
7cb6dfc211 | ||
|
|
b94d942204 |
2
.github/workflows/ci.yml
vendored
2
.github/workflows/ci.yml
vendored
@@ -26,7 +26,7 @@ jobs:
|
|||||||
node-version: latest
|
node-version: latest
|
||||||
|
|
||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
run: bun install
|
run: bun install --frozen-lockfile
|
||||||
|
|
||||||
- name: Run tests with coverage
|
- name: Run tests with coverage
|
||||||
env:
|
env:
|
||||||
|
|||||||
@@ -543,6 +543,8 @@ export async function executeWorkflowForChat(
|
|||||||
userId: deployment.userId,
|
userId: deployment.userId,
|
||||||
workspaceId: '', // TODO: Get from workflow
|
workspaceId: '', // TODO: Get from workflow
|
||||||
variables: workflowVariables,
|
variables: workflowVariables,
|
||||||
|
initialInput: { input, conversationId },
|
||||||
|
executionType: 'chat',
|
||||||
})
|
})
|
||||||
|
|
||||||
const stream = new ReadableStream({
|
const stream = new ReadableStream({
|
||||||
|
|||||||
@@ -12,9 +12,9 @@ import {
|
|||||||
import { getCopilotModel } from '@/lib/copilot/config'
|
import { getCopilotModel } from '@/lib/copilot/config'
|
||||||
import type { CopilotProviderConfig } from '@/lib/copilot/types'
|
import type { CopilotProviderConfig } from '@/lib/copilot/types'
|
||||||
import { env } from '@/lib/env'
|
import { env } from '@/lib/env'
|
||||||
import { generateChatTitle } from '@/lib/generate-chat-title'
|
|
||||||
import { createLogger } from '@/lib/logs/console/logger'
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
import { SIM_AGENT_API_URL_DEFAULT } from '@/lib/sim-agent'
|
import { SIM_AGENT_API_URL_DEFAULT } from '@/lib/sim-agent'
|
||||||
|
import { generateChatTitle } from '@/lib/sim-agent/utils'
|
||||||
import { createFileContent, isSupportedFileType } from '@/lib/uploads/file-utils'
|
import { createFileContent, isSupportedFileType } from '@/lib/uploads/file-utils'
|
||||||
import { S3_COPILOT_CONFIG } from '@/lib/uploads/setup'
|
import { S3_COPILOT_CONFIG } from '@/lib/uploads/setup'
|
||||||
import { downloadFile, getStorageProvider } from '@/lib/uploads/storage-client'
|
import { downloadFile, getStorageProvider } from '@/lib/uploads/storage-client'
|
||||||
|
|||||||
@@ -76,11 +76,9 @@ export async function POST(request: NextRequest) {
|
|||||||
|
|
||||||
logger.info('File parse request received:', { filePath, fileType })
|
logger.info('File parse request received:', { filePath, fileType })
|
||||||
|
|
||||||
// Handle multiple files
|
|
||||||
if (Array.isArray(filePath)) {
|
if (Array.isArray(filePath)) {
|
||||||
const results = []
|
const results = []
|
||||||
for (const path of filePath) {
|
for (const path of filePath) {
|
||||||
// Skip empty or invalid paths
|
|
||||||
if (!path || (typeof path === 'string' && path.trim() === '')) {
|
if (!path || (typeof path === 'string' && path.trim() === '')) {
|
||||||
results.push({
|
results.push({
|
||||||
success: false,
|
success: false,
|
||||||
@@ -91,12 +89,10 @@ export async function POST(request: NextRequest) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const result = await parseFileSingle(path, fileType)
|
const result = await parseFileSingle(path, fileType)
|
||||||
// Add processing time to metadata
|
|
||||||
if (result.metadata) {
|
if (result.metadata) {
|
||||||
result.metadata.processingTime = Date.now() - startTime
|
result.metadata.processingTime = Date.now() - startTime
|
||||||
}
|
}
|
||||||
|
|
||||||
// Transform each result to match expected frontend format
|
|
||||||
if (result.success) {
|
if (result.success) {
|
||||||
results.push({
|
results.push({
|
||||||
success: true,
|
success: true,
|
||||||
@@ -105,7 +101,7 @@ export async function POST(request: NextRequest) {
|
|||||||
name: result.filePath.split('/').pop() || 'unknown',
|
name: result.filePath.split('/').pop() || 'unknown',
|
||||||
fileType: result.metadata?.fileType || 'application/octet-stream',
|
fileType: result.metadata?.fileType || 'application/octet-stream',
|
||||||
size: result.metadata?.size || 0,
|
size: result.metadata?.size || 0,
|
||||||
binary: false, // We only return text content
|
binary: false,
|
||||||
},
|
},
|
||||||
filePath: result.filePath,
|
filePath: result.filePath,
|
||||||
})
|
})
|
||||||
@@ -120,15 +116,12 @@ export async function POST(request: NextRequest) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handle single file
|
|
||||||
const result = await parseFileSingle(filePath, fileType)
|
const result = await parseFileSingle(filePath, fileType)
|
||||||
|
|
||||||
// Add processing time to metadata
|
|
||||||
if (result.metadata) {
|
if (result.metadata) {
|
||||||
result.metadata.processingTime = Date.now() - startTime
|
result.metadata.processingTime = Date.now() - startTime
|
||||||
}
|
}
|
||||||
|
|
||||||
// Transform single file result to match expected frontend format
|
|
||||||
if (result.success) {
|
if (result.success) {
|
||||||
return NextResponse.json({
|
return NextResponse.json({
|
||||||
success: true,
|
success: true,
|
||||||
@@ -142,8 +135,6 @@ export async function POST(request: NextRequest) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// Only return 500 for actual server errors, not file processing failures
|
|
||||||
// File processing failures (like file not found, parsing errors) should return 200 with success:false
|
|
||||||
return NextResponse.json(result)
|
return NextResponse.json(result)
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error('Error in file parse API:', error)
|
logger.error('Error in file parse API:', error)
|
||||||
@@ -164,7 +155,6 @@ export async function POST(request: NextRequest) {
|
|||||||
async function parseFileSingle(filePath: string, fileType?: string): Promise<ParseResult> {
|
async function parseFileSingle(filePath: string, fileType?: string): Promise<ParseResult> {
|
||||||
logger.info('Parsing file:', filePath)
|
logger.info('Parsing file:', filePath)
|
||||||
|
|
||||||
// Validate that filePath is not empty
|
|
||||||
if (!filePath || filePath.trim() === '') {
|
if (!filePath || filePath.trim() === '') {
|
||||||
return {
|
return {
|
||||||
success: false,
|
success: false,
|
||||||
@@ -173,7 +163,6 @@ async function parseFileSingle(filePath: string, fileType?: string): Promise<Par
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate path for security before any processing
|
|
||||||
const pathValidation = validateFilePath(filePath)
|
const pathValidation = validateFilePath(filePath)
|
||||||
if (!pathValidation.isValid) {
|
if (!pathValidation.isValid) {
|
||||||
return {
|
return {
|
||||||
@@ -183,49 +172,40 @@ async function parseFileSingle(filePath: string, fileType?: string): Promise<Par
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if this is an external URL
|
|
||||||
if (filePath.startsWith('http://') || filePath.startsWith('https://')) {
|
if (filePath.startsWith('http://') || filePath.startsWith('https://')) {
|
||||||
return handleExternalUrl(filePath, fileType)
|
return handleExternalUrl(filePath, fileType)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if this is a cloud storage path (S3 or Blob)
|
|
||||||
const isS3Path = filePath.includes('/api/files/serve/s3/')
|
const isS3Path = filePath.includes('/api/files/serve/s3/')
|
||||||
const isBlobPath = filePath.includes('/api/files/serve/blob/')
|
const isBlobPath = filePath.includes('/api/files/serve/blob/')
|
||||||
|
|
||||||
// Use cloud handler if it's a cloud path or we're in cloud mode
|
|
||||||
if (isS3Path || isBlobPath || isUsingCloudStorage()) {
|
if (isS3Path || isBlobPath || isUsingCloudStorage()) {
|
||||||
return handleCloudFile(filePath, fileType)
|
return handleCloudFile(filePath, fileType)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Use local handler for local files
|
|
||||||
return handleLocalFile(filePath, fileType)
|
return handleLocalFile(filePath, fileType)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Validate file path for security
|
* Validate file path for security - prevents null byte injection and path traversal attacks
|
||||||
*/
|
*/
|
||||||
function validateFilePath(filePath: string): { isValid: boolean; error?: string } {
|
function validateFilePath(filePath: string): { isValid: boolean; error?: string } {
|
||||||
// Check for null bytes
|
|
||||||
if (filePath.includes('\0')) {
|
if (filePath.includes('\0')) {
|
||||||
return { isValid: false, error: 'Invalid path: null byte detected' }
|
return { isValid: false, error: 'Invalid path: null byte detected' }
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check for path traversal attempts
|
|
||||||
if (filePath.includes('..')) {
|
if (filePath.includes('..')) {
|
||||||
return { isValid: false, error: 'Access denied: path traversal detected' }
|
return { isValid: false, error: 'Access denied: path traversal detected' }
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check for tilde characters (home directory access)
|
|
||||||
if (filePath.includes('~')) {
|
if (filePath.includes('~')) {
|
||||||
return { isValid: false, error: 'Invalid path: tilde character not allowed' }
|
return { isValid: false, error: 'Invalid path: tilde character not allowed' }
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check for absolute paths outside allowed directories
|
|
||||||
if (filePath.startsWith('/') && !filePath.startsWith('/api/files/serve/')) {
|
if (filePath.startsWith('/') && !filePath.startsWith('/api/files/serve/')) {
|
||||||
return { isValid: false, error: 'Path outside allowed directory' }
|
return { isValid: false, error: 'Path outside allowed directory' }
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check for Windows absolute paths
|
|
||||||
if (/^[A-Za-z]:\\/.test(filePath)) {
|
if (/^[A-Za-z]:\\/.test(filePath)) {
|
||||||
return { isValid: false, error: 'Path outside allowed directory' }
|
return { isValid: false, error: 'Path outside allowed directory' }
|
||||||
}
|
}
|
||||||
@@ -260,12 +240,10 @@ async function handleExternalUrl(url: string, fileType?: string): Promise<ParseR
|
|||||||
|
|
||||||
logger.info(`Downloaded file from URL: ${url}, size: ${buffer.length} bytes`)
|
logger.info(`Downloaded file from URL: ${url}, size: ${buffer.length} bytes`)
|
||||||
|
|
||||||
// Extract filename from URL
|
|
||||||
const urlPath = new URL(url).pathname
|
const urlPath = new URL(url).pathname
|
||||||
const filename = urlPath.split('/').pop() || 'download'
|
const filename = urlPath.split('/').pop() || 'download'
|
||||||
const extension = path.extname(filename).toLowerCase().substring(1)
|
const extension = path.extname(filename).toLowerCase().substring(1)
|
||||||
|
|
||||||
// Process the file based on its content type
|
|
||||||
if (extension === 'pdf') {
|
if (extension === 'pdf') {
|
||||||
return await handlePdfBuffer(buffer, filename, fileType, url)
|
return await handlePdfBuffer(buffer, filename, fileType, url)
|
||||||
}
|
}
|
||||||
@@ -276,7 +254,6 @@ async function handleExternalUrl(url: string, fileType?: string): Promise<ParseR
|
|||||||
return await handleGenericTextBuffer(buffer, filename, extension, fileType, url)
|
return await handleGenericTextBuffer(buffer, filename, extension, fileType, url)
|
||||||
}
|
}
|
||||||
|
|
||||||
// For binary or unknown files
|
|
||||||
return handleGenericBuffer(buffer, filename, extension, fileType)
|
return handleGenericBuffer(buffer, filename, extension, fileType)
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error(`Error handling external URL ${url}:`, error)
|
logger.error(`Error handling external URL ${url}:`, error)
|
||||||
@@ -289,35 +266,29 @@ async function handleExternalUrl(url: string, fileType?: string): Promise<ParseR
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Handle file stored in cloud storage (S3 or Azure Blob)
|
* Handle file stored in cloud storage
|
||||||
*/
|
*/
|
||||||
async function handleCloudFile(filePath: string, fileType?: string): Promise<ParseResult> {
|
async function handleCloudFile(filePath: string, fileType?: string): Promise<ParseResult> {
|
||||||
try {
|
try {
|
||||||
// Extract the cloud key from the path
|
|
||||||
let cloudKey: string
|
let cloudKey: string
|
||||||
if (filePath.includes('/api/files/serve/s3/')) {
|
if (filePath.includes('/api/files/serve/s3/')) {
|
||||||
cloudKey = decodeURIComponent(filePath.split('/api/files/serve/s3/')[1])
|
cloudKey = decodeURIComponent(filePath.split('/api/files/serve/s3/')[1])
|
||||||
} else if (filePath.includes('/api/files/serve/blob/')) {
|
} else if (filePath.includes('/api/files/serve/blob/')) {
|
||||||
cloudKey = decodeURIComponent(filePath.split('/api/files/serve/blob/')[1])
|
cloudKey = decodeURIComponent(filePath.split('/api/files/serve/blob/')[1])
|
||||||
} else if (filePath.startsWith('/api/files/serve/')) {
|
} else if (filePath.startsWith('/api/files/serve/')) {
|
||||||
// Backwards-compatibility: path like "/api/files/serve/<key>"
|
|
||||||
cloudKey = decodeURIComponent(filePath.substring('/api/files/serve/'.length))
|
cloudKey = decodeURIComponent(filePath.substring('/api/files/serve/'.length))
|
||||||
} else {
|
} else {
|
||||||
// Assume raw key provided
|
|
||||||
cloudKey = filePath
|
cloudKey = filePath
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.info('Extracted cloud key:', cloudKey)
|
logger.info('Extracted cloud key:', cloudKey)
|
||||||
|
|
||||||
// Download the file from cloud storage - this can throw for access errors
|
|
||||||
const fileBuffer = await downloadFile(cloudKey)
|
const fileBuffer = await downloadFile(cloudKey)
|
||||||
logger.info(`Downloaded file from cloud storage: ${cloudKey}, size: ${fileBuffer.length} bytes`)
|
logger.info(`Downloaded file from cloud storage: ${cloudKey}, size: ${fileBuffer.length} bytes`)
|
||||||
|
|
||||||
// Extract the filename from the cloud key
|
|
||||||
const filename = cloudKey.split('/').pop() || cloudKey
|
const filename = cloudKey.split('/').pop() || cloudKey
|
||||||
const extension = path.extname(filename).toLowerCase().substring(1)
|
const extension = path.extname(filename).toLowerCase().substring(1)
|
||||||
|
|
||||||
// Process the file based on its content type
|
|
||||||
if (extension === 'pdf') {
|
if (extension === 'pdf') {
|
||||||
return await handlePdfBuffer(fileBuffer, filename, fileType, filePath)
|
return await handlePdfBuffer(fileBuffer, filename, fileType, filePath)
|
||||||
}
|
}
|
||||||
@@ -325,22 +296,19 @@ async function handleCloudFile(filePath: string, fileType?: string): Promise<Par
|
|||||||
return await handleCsvBuffer(fileBuffer, filename, fileType, filePath)
|
return await handleCsvBuffer(fileBuffer, filename, fileType, filePath)
|
||||||
}
|
}
|
||||||
if (isSupportedFileType(extension)) {
|
if (isSupportedFileType(extension)) {
|
||||||
// For other supported types that we have parsers for
|
|
||||||
return await handleGenericTextBuffer(fileBuffer, filename, extension, fileType, filePath)
|
return await handleGenericTextBuffer(fileBuffer, filename, extension, fileType, filePath)
|
||||||
}
|
}
|
||||||
// For binary or unknown files
|
|
||||||
return handleGenericBuffer(fileBuffer, filename, extension, fileType)
|
return handleGenericBuffer(fileBuffer, filename, extension, fileType)
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error(`Error handling cloud file ${filePath}:`, error)
|
logger.error(`Error handling cloud file ${filePath}:`, error)
|
||||||
|
|
||||||
// Check if this is a download/access error that should trigger a 500 response
|
// For download/access errors, throw to trigger 500 response
|
||||||
const errorMessage = (error as Error).message
|
const errorMessage = (error as Error).message
|
||||||
if (errorMessage.includes('Access denied') || errorMessage.includes('Forbidden')) {
|
if (errorMessage.includes('Access denied') || errorMessage.includes('Forbidden')) {
|
||||||
// For access errors, throw to trigger 500 response
|
|
||||||
throw new Error(`Error accessing file from cloud storage: ${errorMessage}`)
|
throw new Error(`Error accessing file from cloud storage: ${errorMessage}`)
|
||||||
}
|
}
|
||||||
|
|
||||||
// For other errors (parsing, processing), return success:false
|
// For other errors (parsing, processing), return success:false and an error message
|
||||||
return {
|
return {
|
||||||
success: false,
|
success: false,
|
||||||
error: `Error accessing file from cloud storage: ${errorMessage}`,
|
error: `Error accessing file from cloud storage: ${errorMessage}`,
|
||||||
@@ -354,28 +322,23 @@ async function handleCloudFile(filePath: string, fileType?: string): Promise<Par
|
|||||||
*/
|
*/
|
||||||
async function handleLocalFile(filePath: string, fileType?: string): Promise<ParseResult> {
|
async function handleLocalFile(filePath: string, fileType?: string): Promise<ParseResult> {
|
||||||
try {
|
try {
|
||||||
// Extract filename from path
|
|
||||||
const filename = filePath.split('/').pop() || filePath
|
const filename = filePath.split('/').pop() || filePath
|
||||||
const fullPath = path.join(UPLOAD_DIR_SERVER, filename)
|
const fullPath = path.join(UPLOAD_DIR_SERVER, filename)
|
||||||
|
|
||||||
logger.info('Processing local file:', fullPath)
|
logger.info('Processing local file:', fullPath)
|
||||||
|
|
||||||
// Check if file exists
|
|
||||||
try {
|
try {
|
||||||
await fsPromises.access(fullPath)
|
await fsPromises.access(fullPath)
|
||||||
} catch {
|
} catch {
|
||||||
throw new Error(`File not found: ${filename}`)
|
throw new Error(`File not found: ${filename}`)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse the file directly
|
|
||||||
const result = await parseFile(fullPath)
|
const result = await parseFile(fullPath)
|
||||||
|
|
||||||
// Get file stats for metadata
|
|
||||||
const stats = await fsPromises.stat(fullPath)
|
const stats = await fsPromises.stat(fullPath)
|
||||||
const fileBuffer = await readFile(fullPath)
|
const fileBuffer = await readFile(fullPath)
|
||||||
const hash = createHash('md5').update(fileBuffer).digest('hex')
|
const hash = createHash('md5').update(fileBuffer).digest('hex')
|
||||||
|
|
||||||
// Extract file extension for type detection
|
|
||||||
const extension = path.extname(filename).toLowerCase().substring(1)
|
const extension = path.extname(filename).toLowerCase().substring(1)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@@ -386,7 +349,7 @@ async function handleLocalFile(filePath: string, fileType?: string): Promise<Par
|
|||||||
fileType: fileType || getMimeType(extension),
|
fileType: fileType || getMimeType(extension),
|
||||||
size: stats.size,
|
size: stats.size,
|
||||||
hash,
|
hash,
|
||||||
processingTime: 0, // Will be set by caller
|
processingTime: 0,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -425,15 +388,14 @@ async function handlePdfBuffer(
|
|||||||
fileType: fileType || 'application/pdf',
|
fileType: fileType || 'application/pdf',
|
||||||
size: fileBuffer.length,
|
size: fileBuffer.length,
|
||||||
hash: createHash('md5').update(fileBuffer).digest('hex'),
|
hash: createHash('md5').update(fileBuffer).digest('hex'),
|
||||||
processingTime: 0, // Will be set by caller
|
processingTime: 0,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error('Failed to parse PDF in memory:', error)
|
logger.error('Failed to parse PDF in memory:', error)
|
||||||
|
|
||||||
// Create fallback message for PDF parsing failure
|
|
||||||
const content = createPdfFailureMessage(
|
const content = createPdfFailureMessage(
|
||||||
0, // We can't determine page count without parsing
|
0,
|
||||||
fileBuffer.length,
|
fileBuffer.length,
|
||||||
originalPath || filename,
|
originalPath || filename,
|
||||||
(error as Error).message
|
(error as Error).message
|
||||||
@@ -447,7 +409,7 @@ async function handlePdfBuffer(
|
|||||||
fileType: fileType || 'application/pdf',
|
fileType: fileType || 'application/pdf',
|
||||||
size: fileBuffer.length,
|
size: fileBuffer.length,
|
||||||
hash: createHash('md5').update(fileBuffer).digest('hex'),
|
hash: createHash('md5').update(fileBuffer).digest('hex'),
|
||||||
processingTime: 0, // Will be set by caller
|
processingTime: 0,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -465,7 +427,6 @@ async function handleCsvBuffer(
|
|||||||
try {
|
try {
|
||||||
logger.info(`Parsing CSV in memory: ${filename}`)
|
logger.info(`Parsing CSV in memory: ${filename}`)
|
||||||
|
|
||||||
// Use the parseBuffer function from our library
|
|
||||||
const { parseBuffer } = await import('@/lib/file-parsers')
|
const { parseBuffer } = await import('@/lib/file-parsers')
|
||||||
const result = await parseBuffer(fileBuffer, 'csv')
|
const result = await parseBuffer(fileBuffer, 'csv')
|
||||||
|
|
||||||
@@ -477,7 +438,7 @@ async function handleCsvBuffer(
|
|||||||
fileType: fileType || 'text/csv',
|
fileType: fileType || 'text/csv',
|
||||||
size: fileBuffer.length,
|
size: fileBuffer.length,
|
||||||
hash: createHash('md5').update(fileBuffer).digest('hex'),
|
hash: createHash('md5').update(fileBuffer).digest('hex'),
|
||||||
processingTime: 0, // Will be set by caller
|
processingTime: 0,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -490,7 +451,7 @@ async function handleCsvBuffer(
|
|||||||
fileType: 'text/csv',
|
fileType: 'text/csv',
|
||||||
size: 0,
|
size: 0,
|
||||||
hash: '',
|
hash: '',
|
||||||
processingTime: 0, // Will be set by caller
|
processingTime: 0,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -509,7 +470,6 @@ async function handleGenericTextBuffer(
|
|||||||
try {
|
try {
|
||||||
logger.info(`Parsing text file in memory: ${filename}`)
|
logger.info(`Parsing text file in memory: ${filename}`)
|
||||||
|
|
||||||
// Try to use a specialized parser if available
|
|
||||||
try {
|
try {
|
||||||
const { parseBuffer, isSupportedFileType } = await import('@/lib/file-parsers')
|
const { parseBuffer, isSupportedFileType } = await import('@/lib/file-parsers')
|
||||||
|
|
||||||
@@ -524,7 +484,7 @@ async function handleGenericTextBuffer(
|
|||||||
fileType: fileType || getMimeType(extension),
|
fileType: fileType || getMimeType(extension),
|
||||||
size: fileBuffer.length,
|
size: fileBuffer.length,
|
||||||
hash: createHash('md5').update(fileBuffer).digest('hex'),
|
hash: createHash('md5').update(fileBuffer).digest('hex'),
|
||||||
processingTime: 0, // Will be set by caller
|
processingTime: 0,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -532,7 +492,6 @@ async function handleGenericTextBuffer(
|
|||||||
logger.warn('Specialized parser failed, falling back to generic parsing:', parserError)
|
logger.warn('Specialized parser failed, falling back to generic parsing:', parserError)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fallback to generic text parsing
|
|
||||||
const content = fileBuffer.toString('utf-8')
|
const content = fileBuffer.toString('utf-8')
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@@ -543,7 +502,7 @@ async function handleGenericTextBuffer(
|
|||||||
fileType: fileType || getMimeType(extension),
|
fileType: fileType || getMimeType(extension),
|
||||||
size: fileBuffer.length,
|
size: fileBuffer.length,
|
||||||
hash: createHash('md5').update(fileBuffer).digest('hex'),
|
hash: createHash('md5').update(fileBuffer).digest('hex'),
|
||||||
processingTime: 0, // Will be set by caller
|
processingTime: 0,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -556,7 +515,7 @@ async function handleGenericTextBuffer(
|
|||||||
fileType: 'text/plain',
|
fileType: 'text/plain',
|
||||||
size: 0,
|
size: 0,
|
||||||
hash: '',
|
hash: '',
|
||||||
processingTime: 0, // Will be set by caller
|
processingTime: 0,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -584,7 +543,7 @@ function handleGenericBuffer(
|
|||||||
fileType: fileType || getMimeType(extension),
|
fileType: fileType || getMimeType(extension),
|
||||||
size: fileBuffer.length,
|
size: fileBuffer.length,
|
||||||
hash: createHash('md5').update(fileBuffer).digest('hex'),
|
hash: createHash('md5').update(fileBuffer).digest('hex'),
|
||||||
processingTime: 0, // Will be set by caller
|
processingTime: 0,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -594,8 +553,6 @@ function handleGenericBuffer(
|
|||||||
*/
|
*/
|
||||||
async function parseBufferAsPdf(buffer: Buffer) {
|
async function parseBufferAsPdf(buffer: Buffer) {
|
||||||
try {
|
try {
|
||||||
// Import parsers dynamically to avoid initialization issues in tests
|
|
||||||
// First try to use the main PDF parser
|
|
||||||
try {
|
try {
|
||||||
const { PdfParser } = await import('@/lib/file-parsers/pdf-parser')
|
const { PdfParser } = await import('@/lib/file-parsers/pdf-parser')
|
||||||
const parser = new PdfParser()
|
const parser = new PdfParser()
|
||||||
@@ -606,7 +563,6 @@ async function parseBufferAsPdf(buffer: Buffer) {
|
|||||||
}
|
}
|
||||||
throw new Error('PDF parser does not support buffer parsing')
|
throw new Error('PDF parser does not support buffer parsing')
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
// Fallback to raw PDF parser
|
|
||||||
logger.warn('Main PDF parser failed, using raw parser for buffer:', error)
|
logger.warn('Main PDF parser failed, using raw parser for buffer:', error)
|
||||||
const { RawPdfParser } = await import('@/lib/file-parsers/raw-pdf-parser')
|
const { RawPdfParser } = await import('@/lib/file-parsers/raw-pdf-parser')
|
||||||
const rawParser = new RawPdfParser()
|
const rawParser = new RawPdfParser()
|
||||||
@@ -655,7 +611,7 @@ Please use a PDF viewer for best results.`
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create error message for PDF parsing failure
|
* Create error message for PDF parsing failure and make it more readable
|
||||||
*/
|
*/
|
||||||
function createPdfFailureMessage(
|
function createPdfFailureMessage(
|
||||||
pageCount: number,
|
pageCount: number,
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import { randomUUID } from 'crypto'
|
|||||||
import { type NextRequest, NextResponse } from 'next/server'
|
import { type NextRequest, NextResponse } from 'next/server'
|
||||||
import { z } from 'zod'
|
import { z } from 'zod'
|
||||||
import { getSession } from '@/lib/auth'
|
import { getSession } from '@/lib/auth'
|
||||||
import { SUPPORTED_FIELD_TYPES } from '@/lib/constants/knowledge'
|
import { SUPPORTED_FIELD_TYPES } from '@/lib/knowledge/consts'
|
||||||
import {
|
import {
|
||||||
cleanupUnusedTagDefinitions,
|
cleanupUnusedTagDefinitions,
|
||||||
createOrUpdateTagDefinitionsBulk,
|
createOrUpdateTagDefinitionsBulk,
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import { randomUUID } from 'crypto'
|
|||||||
import { type NextRequest, NextResponse } from 'next/server'
|
import { type NextRequest, NextResponse } from 'next/server'
|
||||||
import { z } from 'zod'
|
import { z } from 'zod'
|
||||||
import { getSession } from '@/lib/auth'
|
import { getSession } from '@/lib/auth'
|
||||||
import { SUPPORTED_FIELD_TYPES } from '@/lib/constants/knowledge'
|
import { SUPPORTED_FIELD_TYPES } from '@/lib/knowledge/consts'
|
||||||
import { createTagDefinition, getTagDefinitions } from '@/lib/knowledge/tags/service'
|
import { createTagDefinition, getTagDefinitions } from '@/lib/knowledge/tags/service'
|
||||||
import { createLogger } from '@/lib/logs/console/logger'
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
import { checkKnowledgeBaseAccess } from '@/app/api/knowledge/utils'
|
import { checkKnowledgeBaseAccess } from '@/app/api/knowledge/utils'
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { type NextRequest, NextResponse } from 'next/server'
|
import { type NextRequest, NextResponse } from 'next/server'
|
||||||
import { z } from 'zod'
|
import { z } from 'zod'
|
||||||
import { TAG_SLOTS } from '@/lib/constants/knowledge'
|
import { TAG_SLOTS } from '@/lib/knowledge/consts'
|
||||||
import { getDocumentTagDefinitions } from '@/lib/knowledge/tags/service'
|
import { getDocumentTagDefinitions } from '@/lib/knowledge/tags/service'
|
||||||
import { createLogger } from '@/lib/logs/console/logger'
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
import { estimateTokenCount } from '@/lib/tokenization/estimators'
|
import { estimateTokenCount } from '@/lib/tokenization/estimators'
|
||||||
|
|||||||
@@ -41,6 +41,7 @@ export async function GET(
|
|||||||
executionId,
|
executionId,
|
||||||
workflowId: workflowLog.workflowId,
|
workflowId: workflowLog.workflowId,
|
||||||
workflowState: snapshot.stateData,
|
workflowState: snapshot.stateData,
|
||||||
|
executionData: workflowLog.executionData || {},
|
||||||
executionMetadata: {
|
executionMetadata: {
|
||||||
trigger: workflowLog.trigger,
|
trigger: workflowLog.trigger,
|
||||||
startedAt: workflowLog.startedAt.toISOString(),
|
startedAt: workflowLog.startedAt.toISOString(),
|
||||||
|
|||||||
@@ -399,6 +399,9 @@ export async function GET() {
|
|||||||
userId: workflowRecord.userId,
|
userId: workflowRecord.userId,
|
||||||
workspaceId: workflowRecord.workspaceId || '',
|
workspaceId: workflowRecord.workspaceId || '',
|
||||||
variables: variables || {},
|
variables: variables || {},
|
||||||
|
initialInput: input,
|
||||||
|
startBlockId: schedule.blockId || undefined,
|
||||||
|
executionType: 'schedule',
|
||||||
})
|
})
|
||||||
|
|
||||||
const executor = new Executor({
|
const executor = new Executor({
|
||||||
@@ -467,10 +470,19 @@ export async function GET() {
|
|||||||
|
|
||||||
// Create a minimal log entry for early failures
|
// Create a minimal log entry for early failures
|
||||||
try {
|
try {
|
||||||
|
const input = {
|
||||||
|
workflowId: schedule.workflowId,
|
||||||
|
_context: {
|
||||||
|
workflowId: schedule.workflowId,
|
||||||
|
},
|
||||||
|
}
|
||||||
await loggingSession.safeStart({
|
await loggingSession.safeStart({
|
||||||
userId: workflowRecord.userId,
|
userId: workflowRecord.userId,
|
||||||
workspaceId: workflowRecord.workspaceId || '',
|
workspaceId: workflowRecord.workspaceId || '',
|
||||||
variables: {},
|
variables: {},
|
||||||
|
initialInput: input,
|
||||||
|
startBlockId: schedule.blockId || undefined,
|
||||||
|
executionType: 'schedule',
|
||||||
})
|
})
|
||||||
|
|
||||||
await loggingSession.safeCompleteWithError({
|
await loggingSession.safeCompleteWithError({
|
||||||
@@ -586,10 +598,17 @@ export async function GET() {
|
|||||||
requestId
|
requestId
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const input = {
|
||||||
|
workflowId: schedule.workflowId,
|
||||||
|
_context: {
|
||||||
|
workflowId: schedule.workflowId,
|
||||||
|
},
|
||||||
|
}
|
||||||
await failureLoggingSession.safeStart({
|
await failureLoggingSession.safeStart({
|
||||||
userId: workflowRecord.userId,
|
userId: workflowRecord.userId,
|
||||||
workspaceId: workflowRecord.workspaceId || '',
|
workspaceId: workflowRecord.workspaceId || '',
|
||||||
variables: {},
|
variables: {},
|
||||||
|
initialInput: input,
|
||||||
})
|
})
|
||||||
|
|
||||||
await failureLoggingSession.safeCompleteWithError({
|
await failureLoggingSession.safeCompleteWithError({
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { randomUUID } from 'crypto'
|
||||||
import { type NextRequest, NextResponse } from 'next/server'
|
import { type NextRequest, NextResponse } from 'next/server'
|
||||||
import { z } from 'zod'
|
import { z } from 'zod'
|
||||||
import { createLogger } from '@/lib/logs/console/logger'
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
@@ -11,13 +12,13 @@ const DeleteSchema = z.object({
|
|||||||
database: z.string().min(1, 'Database name is required'),
|
database: z.string().min(1, 'Database name is required'),
|
||||||
username: z.string().min(1, 'Username is required'),
|
username: z.string().min(1, 'Username is required'),
|
||||||
password: z.string().min(1, 'Password is required'),
|
password: z.string().min(1, 'Password is required'),
|
||||||
ssl: z.enum(['disabled', 'required', 'preferred']).default('required'),
|
ssl: z.enum(['disabled', 'required', 'preferred']).default('preferred'),
|
||||||
table: z.string().min(1, 'Table name is required'),
|
table: z.string().min(1, 'Table name is required'),
|
||||||
where: z.string().min(1, 'WHERE clause is required'),
|
where: z.string().min(1, 'WHERE clause is required'),
|
||||||
})
|
})
|
||||||
|
|
||||||
export async function POST(request: NextRequest) {
|
export async function POST(request: NextRequest) {
|
||||||
const requestId = crypto.randomUUID().slice(0, 8)
|
const requestId = randomUUID().slice(0, 8)
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const body = await request.json()
|
const body = await request.json()
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { randomUUID } from 'crypto'
|
||||||
import { type NextRequest, NextResponse } from 'next/server'
|
import { type NextRequest, NextResponse } from 'next/server'
|
||||||
import { z } from 'zod'
|
import { z } from 'zod'
|
||||||
import { createLogger } from '@/lib/logs/console/logger'
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
@@ -11,12 +12,12 @@ const ExecuteSchema = z.object({
|
|||||||
database: z.string().min(1, 'Database name is required'),
|
database: z.string().min(1, 'Database name is required'),
|
||||||
username: z.string().min(1, 'Username is required'),
|
username: z.string().min(1, 'Username is required'),
|
||||||
password: z.string().min(1, 'Password is required'),
|
password: z.string().min(1, 'Password is required'),
|
||||||
ssl: z.enum(['disabled', 'required', 'preferred']).default('required'),
|
ssl: z.enum(['disabled', 'required', 'preferred']).default('preferred'),
|
||||||
query: z.string().min(1, 'Query is required'),
|
query: z.string().min(1, 'Query is required'),
|
||||||
})
|
})
|
||||||
|
|
||||||
export async function POST(request: NextRequest) {
|
export async function POST(request: NextRequest) {
|
||||||
const requestId = crypto.randomUUID().slice(0, 8)
|
const requestId = randomUUID().slice(0, 8)
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const body = await request.json()
|
const body = await request.json()
|
||||||
@@ -26,7 +27,6 @@ export async function POST(request: NextRequest) {
|
|||||||
`[${requestId}] Executing raw SQL on ${params.host}:${params.port}/${params.database}`
|
`[${requestId}] Executing raw SQL on ${params.host}:${params.port}/${params.database}`
|
||||||
)
|
)
|
||||||
|
|
||||||
// Validate query before execution
|
|
||||||
const validation = validateQuery(params.query)
|
const validation = validateQuery(params.query)
|
||||||
if (!validation.isValid) {
|
if (!validation.isValid) {
|
||||||
logger.warn(`[${requestId}] Query validation failed: ${validation.error}`)
|
logger.warn(`[${requestId}] Query validation failed: ${validation.error}`)
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { randomUUID } from 'crypto'
|
||||||
import { type NextRequest, NextResponse } from 'next/server'
|
import { type NextRequest, NextResponse } from 'next/server'
|
||||||
import { z } from 'zod'
|
import { z } from 'zod'
|
||||||
import { createLogger } from '@/lib/logs/console/logger'
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
@@ -11,7 +12,7 @@ const InsertSchema = z.object({
|
|||||||
database: z.string().min(1, 'Database name is required'),
|
database: z.string().min(1, 'Database name is required'),
|
||||||
username: z.string().min(1, 'Username is required'),
|
username: z.string().min(1, 'Username is required'),
|
||||||
password: z.string().min(1, 'Password is required'),
|
password: z.string().min(1, 'Password is required'),
|
||||||
ssl: z.enum(['disabled', 'required', 'preferred']).default('required'),
|
ssl: z.enum(['disabled', 'required', 'preferred']).default('preferred'),
|
||||||
table: z.string().min(1, 'Table name is required'),
|
table: z.string().min(1, 'Table name is required'),
|
||||||
data: z.union([
|
data: z.union([
|
||||||
z
|
z
|
||||||
@@ -38,13 +39,10 @@ const InsertSchema = z.object({
|
|||||||
})
|
})
|
||||||
|
|
||||||
export async function POST(request: NextRequest) {
|
export async function POST(request: NextRequest) {
|
||||||
const requestId = crypto.randomUUID().slice(0, 8)
|
const requestId = randomUUID().slice(0, 8)
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const body = await request.json()
|
const body = await request.json()
|
||||||
|
|
||||||
logger.info(`[${requestId}] Received data field type: ${typeof body.data}, value:`, body.data)
|
|
||||||
|
|
||||||
const params = InsertSchema.parse(body)
|
const params = InsertSchema.parse(body)
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { randomUUID } from 'crypto'
|
||||||
import { type NextRequest, NextResponse } from 'next/server'
|
import { type NextRequest, NextResponse } from 'next/server'
|
||||||
import { z } from 'zod'
|
import { z } from 'zod'
|
||||||
import { createLogger } from '@/lib/logs/console/logger'
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
@@ -11,12 +12,12 @@ const QuerySchema = z.object({
|
|||||||
database: z.string().min(1, 'Database name is required'),
|
database: z.string().min(1, 'Database name is required'),
|
||||||
username: z.string().min(1, 'Username is required'),
|
username: z.string().min(1, 'Username is required'),
|
||||||
password: z.string().min(1, 'Password is required'),
|
password: z.string().min(1, 'Password is required'),
|
||||||
ssl: z.enum(['disabled', 'required', 'preferred']).default('required'),
|
ssl: z.enum(['disabled', 'required', 'preferred']).default('preferred'),
|
||||||
query: z.string().min(1, 'Query is required'),
|
query: z.string().min(1, 'Query is required'),
|
||||||
})
|
})
|
||||||
|
|
||||||
export async function POST(request: NextRequest) {
|
export async function POST(request: NextRequest) {
|
||||||
const requestId = crypto.randomUUID().slice(0, 8)
|
const requestId = randomUUID().slice(0, 8)
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const body = await request.json()
|
const body = await request.json()
|
||||||
@@ -26,7 +27,6 @@ export async function POST(request: NextRequest) {
|
|||||||
`[${requestId}] Executing MySQL query on ${params.host}:${params.port}/${params.database}`
|
`[${requestId}] Executing MySQL query on ${params.host}:${params.port}/${params.database}`
|
||||||
)
|
)
|
||||||
|
|
||||||
// Validate query before execution
|
|
||||||
const validation = validateQuery(params.query)
|
const validation = validateQuery(params.query)
|
||||||
if (!validation.isValid) {
|
if (!validation.isValid) {
|
||||||
logger.warn(`[${requestId}] Query validation failed: ${validation.error}`)
|
logger.warn(`[${requestId}] Query validation failed: ${validation.error}`)
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { randomUUID } from 'crypto'
|
||||||
import { type NextRequest, NextResponse } from 'next/server'
|
import { type NextRequest, NextResponse } from 'next/server'
|
||||||
import { z } from 'zod'
|
import { z } from 'zod'
|
||||||
import { createLogger } from '@/lib/logs/console/logger'
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
@@ -11,7 +12,7 @@ const UpdateSchema = z.object({
|
|||||||
database: z.string().min(1, 'Database name is required'),
|
database: z.string().min(1, 'Database name is required'),
|
||||||
username: z.string().min(1, 'Username is required'),
|
username: z.string().min(1, 'Username is required'),
|
||||||
password: z.string().min(1, 'Password is required'),
|
password: z.string().min(1, 'Password is required'),
|
||||||
ssl: z.enum(['disabled', 'required', 'preferred']).default('required'),
|
ssl: z.enum(['disabled', 'required', 'preferred']).default('preferred'),
|
||||||
table: z.string().min(1, 'Table name is required'),
|
table: z.string().min(1, 'Table name is required'),
|
||||||
data: z.union([
|
data: z.union([
|
||||||
z
|
z
|
||||||
@@ -36,7 +37,7 @@ const UpdateSchema = z.object({
|
|||||||
})
|
})
|
||||||
|
|
||||||
export async function POST(request: NextRequest) {
|
export async function POST(request: NextRequest) {
|
||||||
const requestId = crypto.randomUUID().slice(0, 8)
|
const requestId = randomUUID().slice(0, 8)
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const body = await request.json()
|
const body = await request.json()
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ export interface MySQLConnectionConfig {
|
|||||||
database: string
|
database: string
|
||||||
username: string
|
username: string
|
||||||
password: string
|
password: string
|
||||||
ssl?: string
|
ssl?: 'disabled' | 'required' | 'preferred'
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function createMySQLConnection(config: MySQLConnectionConfig) {
|
export async function createMySQLConnection(config: MySQLConnectionConfig) {
|
||||||
@@ -18,13 +18,13 @@ export async function createMySQLConnection(config: MySQLConnectionConfig) {
|
|||||||
password: config.password,
|
password: config.password,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handle SSL configuration
|
if (config.ssl === 'disabled') {
|
||||||
if (config.ssl === 'required') {
|
// Don't set ssl property at all to disable SSL
|
||||||
|
} else if (config.ssl === 'required') {
|
||||||
connectionConfig.ssl = { rejectUnauthorized: true }
|
connectionConfig.ssl = { rejectUnauthorized: true }
|
||||||
} else if (config.ssl === 'preferred') {
|
} else if (config.ssl === 'preferred') {
|
||||||
connectionConfig.ssl = { rejectUnauthorized: false }
|
connectionConfig.ssl = { rejectUnauthorized: false }
|
||||||
}
|
}
|
||||||
// For 'disabled', we don't set the ssl property at all
|
|
||||||
|
|
||||||
return mysql.createConnection(connectionConfig)
|
return mysql.createConnection(connectionConfig)
|
||||||
}
|
}
|
||||||
@@ -54,7 +54,6 @@ export async function executeQuery(
|
|||||||
export function validateQuery(query: string): { isValid: boolean; error?: string } {
|
export function validateQuery(query: string): { isValid: boolean; error?: string } {
|
||||||
const trimmedQuery = query.trim().toLowerCase()
|
const trimmedQuery = query.trim().toLowerCase()
|
||||||
|
|
||||||
// Block dangerous SQL operations
|
|
||||||
const dangerousPatterns = [
|
const dangerousPatterns = [
|
||||||
/drop\s+database/i,
|
/drop\s+database/i,
|
||||||
/drop\s+schema/i,
|
/drop\s+schema/i,
|
||||||
@@ -91,7 +90,6 @@ export function validateQuery(query: string): { isValid: boolean; error?: string
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Only allow specific statement types for execute endpoint
|
|
||||||
const allowedStatements = /^(select|insert|update|delete|with|show|describe|explain)\s+/i
|
const allowedStatements = /^(select|insert|update|delete|with|show|describe|explain)\s+/i
|
||||||
if (!allowedStatements.test(trimmedQuery)) {
|
if (!allowedStatements.test(trimmedQuery)) {
|
||||||
return {
|
return {
|
||||||
@@ -116,6 +114,8 @@ export function buildInsertQuery(table: string, data: Record<string, unknown>) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function buildUpdateQuery(table: string, data: Record<string, unknown>, where: string) {
|
export function buildUpdateQuery(table: string, data: Record<string, unknown>, where: string) {
|
||||||
|
validateWhereClause(where)
|
||||||
|
|
||||||
const sanitizedTable = sanitizeIdentifier(table)
|
const sanitizedTable = sanitizeIdentifier(table)
|
||||||
const columns = Object.keys(data)
|
const columns = Object.keys(data)
|
||||||
const values = Object.values(data)
|
const values = Object.values(data)
|
||||||
@@ -127,14 +127,33 @@ export function buildUpdateQuery(table: string, data: Record<string, unknown>, w
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function buildDeleteQuery(table: string, where: string) {
|
export function buildDeleteQuery(table: string, where: string) {
|
||||||
|
validateWhereClause(where)
|
||||||
|
|
||||||
const sanitizedTable = sanitizeIdentifier(table)
|
const sanitizedTable = sanitizeIdentifier(table)
|
||||||
const query = `DELETE FROM ${sanitizedTable} WHERE ${where}`
|
const query = `DELETE FROM ${sanitizedTable} WHERE ${where}`
|
||||||
|
|
||||||
return { query, values: [] }
|
return { query, values: [] }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function validateWhereClause(where: string): void {
|
||||||
|
const dangerousPatterns = [
|
||||||
|
/;\s*(drop|delete|insert|update|create|alter|grant|revoke)/i,
|
||||||
|
/union\s+select/i,
|
||||||
|
/into\s+outfile/i,
|
||||||
|
/load_file/i,
|
||||||
|
/--/,
|
||||||
|
/\/\*/,
|
||||||
|
/\*\//,
|
||||||
|
]
|
||||||
|
|
||||||
|
for (const pattern of dangerousPatterns) {
|
||||||
|
if (pattern.test(where)) {
|
||||||
|
throw new Error('WHERE clause contains potentially dangerous operation')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export function sanitizeIdentifier(identifier: string): string {
|
export function sanitizeIdentifier(identifier: string): string {
|
||||||
// Handle schema.table format
|
|
||||||
if (identifier.includes('.')) {
|
if (identifier.includes('.')) {
|
||||||
const parts = identifier.split('.')
|
const parts = identifier.split('.')
|
||||||
return parts.map((part) => sanitizeSingleIdentifier(part)).join('.')
|
return parts.map((part) => sanitizeSingleIdentifier(part)).join('.')
|
||||||
@@ -144,16 +163,13 @@ export function sanitizeIdentifier(identifier: string): string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function sanitizeSingleIdentifier(identifier: string): string {
|
function sanitizeSingleIdentifier(identifier: string): string {
|
||||||
// Remove any existing backticks to prevent double-escaping
|
|
||||||
const cleaned = identifier.replace(/`/g, '')
|
const cleaned = identifier.replace(/`/g, '')
|
||||||
|
|
||||||
// Validate identifier contains only safe characters
|
|
||||||
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(cleaned)) {
|
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(cleaned)) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`Invalid identifier: ${identifier}. Identifiers must start with a letter or underscore and contain only letters, numbers, and underscores.`
|
`Invalid identifier: ${identifier}. Identifiers must start with a letter or underscore and contain only letters, numbers, and underscores.`
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Wrap in backticks for MySQL
|
|
||||||
return `\`${cleaned}\``
|
return `\`${cleaned}\``
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,11 +1,8 @@
|
|||||||
|
import { randomUUID } from 'crypto'
|
||||||
import { type NextRequest, NextResponse } from 'next/server'
|
import { type NextRequest, NextResponse } from 'next/server'
|
||||||
import { z } from 'zod'
|
import { z } from 'zod'
|
||||||
import { createLogger } from '@/lib/logs/console/logger'
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
import {
|
import { createPostgresConnection, executeDelete } from '@/app/api/tools/postgresql/utils'
|
||||||
buildDeleteQuery,
|
|
||||||
createPostgresConnection,
|
|
||||||
executeQuery,
|
|
||||||
} from '@/app/api/tools/postgresql/utils'
|
|
||||||
|
|
||||||
const logger = createLogger('PostgreSQLDeleteAPI')
|
const logger = createLogger('PostgreSQLDeleteAPI')
|
||||||
|
|
||||||
@@ -15,13 +12,13 @@ const DeleteSchema = z.object({
|
|||||||
database: z.string().min(1, 'Database name is required'),
|
database: z.string().min(1, 'Database name is required'),
|
||||||
username: z.string().min(1, 'Username is required'),
|
username: z.string().min(1, 'Username is required'),
|
||||||
password: z.string().min(1, 'Password is required'),
|
password: z.string().min(1, 'Password is required'),
|
||||||
ssl: z.enum(['disabled', 'required', 'preferred']).default('required'),
|
ssl: z.enum(['disabled', 'required', 'preferred']).default('preferred'),
|
||||||
table: z.string().min(1, 'Table name is required'),
|
table: z.string().min(1, 'Table name is required'),
|
||||||
where: z.string().min(1, 'WHERE clause is required'),
|
where: z.string().min(1, 'WHERE clause is required'),
|
||||||
})
|
})
|
||||||
|
|
||||||
export async function POST(request: NextRequest) {
|
export async function POST(request: NextRequest) {
|
||||||
const requestId = crypto.randomUUID().slice(0, 8)
|
const requestId = randomUUID().slice(0, 8)
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const body = await request.json()
|
const body = await request.json()
|
||||||
@@ -31,7 +28,7 @@ export async function POST(request: NextRequest) {
|
|||||||
`[${requestId}] Deleting data from ${params.table} on ${params.host}:${params.port}/${params.database}`
|
`[${requestId}] Deleting data from ${params.table} on ${params.host}:${params.port}/${params.database}`
|
||||||
)
|
)
|
||||||
|
|
||||||
const client = await createPostgresConnection({
|
const sql = createPostgresConnection({
|
||||||
host: params.host,
|
host: params.host,
|
||||||
port: params.port,
|
port: params.port,
|
||||||
database: params.database,
|
database: params.database,
|
||||||
@@ -41,8 +38,7 @@ export async function POST(request: NextRequest) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const { query, values } = buildDeleteQuery(params.table, params.where)
|
const result = await executeDelete(sql, params.table, params.where)
|
||||||
const result = await executeQuery(client, query, values)
|
|
||||||
|
|
||||||
logger.info(`[${requestId}] Delete executed successfully, ${result.rowCount} row(s) deleted`)
|
logger.info(`[${requestId}] Delete executed successfully, ${result.rowCount} row(s) deleted`)
|
||||||
|
|
||||||
@@ -52,7 +48,7 @@ export async function POST(request: NextRequest) {
|
|||||||
rowCount: result.rowCount,
|
rowCount: result.rowCount,
|
||||||
})
|
})
|
||||||
} finally {
|
} finally {
|
||||||
await client.end()
|
await sql.end()
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (error instanceof z.ZodError) {
|
if (error instanceof z.ZodError) {
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { randomUUID } from 'crypto'
|
||||||
import { type NextRequest, NextResponse } from 'next/server'
|
import { type NextRequest, NextResponse } from 'next/server'
|
||||||
import { z } from 'zod'
|
import { z } from 'zod'
|
||||||
import { createLogger } from '@/lib/logs/console/logger'
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
@@ -15,12 +16,12 @@ const ExecuteSchema = z.object({
|
|||||||
database: z.string().min(1, 'Database name is required'),
|
database: z.string().min(1, 'Database name is required'),
|
||||||
username: z.string().min(1, 'Username is required'),
|
username: z.string().min(1, 'Username is required'),
|
||||||
password: z.string().min(1, 'Password is required'),
|
password: z.string().min(1, 'Password is required'),
|
||||||
ssl: z.enum(['disabled', 'required', 'preferred']).default('required'),
|
ssl: z.enum(['disabled', 'required', 'preferred']).default('preferred'),
|
||||||
query: z.string().min(1, 'Query is required'),
|
query: z.string().min(1, 'Query is required'),
|
||||||
})
|
})
|
||||||
|
|
||||||
export async function POST(request: NextRequest) {
|
export async function POST(request: NextRequest) {
|
||||||
const requestId = crypto.randomUUID().slice(0, 8)
|
const requestId = randomUUID().slice(0, 8)
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const body = await request.json()
|
const body = await request.json()
|
||||||
@@ -30,7 +31,6 @@ export async function POST(request: NextRequest) {
|
|||||||
`[${requestId}] Executing raw SQL on ${params.host}:${params.port}/${params.database}`
|
`[${requestId}] Executing raw SQL on ${params.host}:${params.port}/${params.database}`
|
||||||
)
|
)
|
||||||
|
|
||||||
// Validate query before execution
|
|
||||||
const validation = validateQuery(params.query)
|
const validation = validateQuery(params.query)
|
||||||
if (!validation.isValid) {
|
if (!validation.isValid) {
|
||||||
logger.warn(`[${requestId}] Query validation failed: ${validation.error}`)
|
logger.warn(`[${requestId}] Query validation failed: ${validation.error}`)
|
||||||
@@ -40,7 +40,7 @@ export async function POST(request: NextRequest) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
const client = await createPostgresConnection({
|
const sql = createPostgresConnection({
|
||||||
host: params.host,
|
host: params.host,
|
||||||
port: params.port,
|
port: params.port,
|
||||||
database: params.database,
|
database: params.database,
|
||||||
@@ -50,7 +50,7 @@ export async function POST(request: NextRequest) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const result = await executeQuery(client, params.query)
|
const result = await executeQuery(sql, params.query)
|
||||||
|
|
||||||
logger.info(`[${requestId}] SQL executed successfully, ${result.rowCount} row(s) affected`)
|
logger.info(`[${requestId}] SQL executed successfully, ${result.rowCount} row(s) affected`)
|
||||||
|
|
||||||
@@ -60,7 +60,7 @@ export async function POST(request: NextRequest) {
|
|||||||
rowCount: result.rowCount,
|
rowCount: result.rowCount,
|
||||||
})
|
})
|
||||||
} finally {
|
} finally {
|
||||||
await client.end()
|
await sql.end()
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (error instanceof z.ZodError) {
|
if (error instanceof z.ZodError) {
|
||||||
|
|||||||
@@ -1,11 +1,8 @@
|
|||||||
|
import { randomUUID } from 'crypto'
|
||||||
import { type NextRequest, NextResponse } from 'next/server'
|
import { type NextRequest, NextResponse } from 'next/server'
|
||||||
import { z } from 'zod'
|
import { z } from 'zod'
|
||||||
import { createLogger } from '@/lib/logs/console/logger'
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
import {
|
import { createPostgresConnection, executeInsert } from '@/app/api/tools/postgresql/utils'
|
||||||
buildInsertQuery,
|
|
||||||
createPostgresConnection,
|
|
||||||
executeQuery,
|
|
||||||
} from '@/app/api/tools/postgresql/utils'
|
|
||||||
|
|
||||||
const logger = createLogger('PostgreSQLInsertAPI')
|
const logger = createLogger('PostgreSQLInsertAPI')
|
||||||
|
|
||||||
@@ -15,7 +12,7 @@ const InsertSchema = z.object({
|
|||||||
database: z.string().min(1, 'Database name is required'),
|
database: z.string().min(1, 'Database name is required'),
|
||||||
username: z.string().min(1, 'Username is required'),
|
username: z.string().min(1, 'Username is required'),
|
||||||
password: z.string().min(1, 'Password is required'),
|
password: z.string().min(1, 'Password is required'),
|
||||||
ssl: z.enum(['disabled', 'required', 'preferred']).default('required'),
|
ssl: z.enum(['disabled', 'required', 'preferred']).default('preferred'),
|
||||||
table: z.string().min(1, 'Table name is required'),
|
table: z.string().min(1, 'Table name is required'),
|
||||||
data: z.union([
|
data: z.union([
|
||||||
z
|
z
|
||||||
@@ -42,21 +39,18 @@ const InsertSchema = z.object({
|
|||||||
})
|
})
|
||||||
|
|
||||||
export async function POST(request: NextRequest) {
|
export async function POST(request: NextRequest) {
|
||||||
const requestId = crypto.randomUUID().slice(0, 8)
|
const requestId = randomUUID().slice(0, 8)
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const body = await request.json()
|
const body = await request.json()
|
||||||
|
|
||||||
// Debug: Log the data field to see what we're getting
|
|
||||||
logger.info(`[${requestId}] Received data field type: ${typeof body.data}, value:`, body.data)
|
|
||||||
|
|
||||||
const params = InsertSchema.parse(body)
|
const params = InsertSchema.parse(body)
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
`[${requestId}] Inserting data into ${params.table} on ${params.host}:${params.port}/${params.database}`
|
`[${requestId}] Inserting data into ${params.table} on ${params.host}:${params.port}/${params.database}`
|
||||||
)
|
)
|
||||||
|
|
||||||
const client = await createPostgresConnection({
|
const sql = createPostgresConnection({
|
||||||
host: params.host,
|
host: params.host,
|
||||||
port: params.port,
|
port: params.port,
|
||||||
database: params.database,
|
database: params.database,
|
||||||
@@ -66,8 +60,7 @@ export async function POST(request: NextRequest) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const { query, values } = buildInsertQuery(params.table, params.data)
|
const result = await executeInsert(sql, params.table, params.data)
|
||||||
const result = await executeQuery(client, query, values)
|
|
||||||
|
|
||||||
logger.info(`[${requestId}] Insert executed successfully, ${result.rowCount} row(s) inserted`)
|
logger.info(`[${requestId}] Insert executed successfully, ${result.rowCount} row(s) inserted`)
|
||||||
|
|
||||||
@@ -77,7 +70,7 @@ export async function POST(request: NextRequest) {
|
|||||||
rowCount: result.rowCount,
|
rowCount: result.rowCount,
|
||||||
})
|
})
|
||||||
} finally {
|
} finally {
|
||||||
await client.end()
|
await sql.end()
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (error instanceof z.ZodError) {
|
if (error instanceof z.ZodError) {
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { randomUUID } from 'crypto'
|
||||||
import { type NextRequest, NextResponse } from 'next/server'
|
import { type NextRequest, NextResponse } from 'next/server'
|
||||||
import { z } from 'zod'
|
import { z } from 'zod'
|
||||||
import { createLogger } from '@/lib/logs/console/logger'
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
@@ -11,12 +12,12 @@ const QuerySchema = z.object({
|
|||||||
database: z.string().min(1, 'Database name is required'),
|
database: z.string().min(1, 'Database name is required'),
|
||||||
username: z.string().min(1, 'Username is required'),
|
username: z.string().min(1, 'Username is required'),
|
||||||
password: z.string().min(1, 'Password is required'),
|
password: z.string().min(1, 'Password is required'),
|
||||||
ssl: z.enum(['disabled', 'required', 'preferred']).default('required'),
|
ssl: z.enum(['disabled', 'required', 'preferred']).default('preferred'),
|
||||||
query: z.string().min(1, 'Query is required'),
|
query: z.string().min(1, 'Query is required'),
|
||||||
})
|
})
|
||||||
|
|
||||||
export async function POST(request: NextRequest) {
|
export async function POST(request: NextRequest) {
|
||||||
const requestId = crypto.randomUUID().slice(0, 8)
|
const requestId = randomUUID().slice(0, 8)
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const body = await request.json()
|
const body = await request.json()
|
||||||
@@ -26,7 +27,7 @@ export async function POST(request: NextRequest) {
|
|||||||
`[${requestId}] Executing PostgreSQL query on ${params.host}:${params.port}/${params.database}`
|
`[${requestId}] Executing PostgreSQL query on ${params.host}:${params.port}/${params.database}`
|
||||||
)
|
)
|
||||||
|
|
||||||
const client = await createPostgresConnection({
|
const sql = createPostgresConnection({
|
||||||
host: params.host,
|
host: params.host,
|
||||||
port: params.port,
|
port: params.port,
|
||||||
database: params.database,
|
database: params.database,
|
||||||
@@ -36,7 +37,7 @@ export async function POST(request: NextRequest) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const result = await executeQuery(client, params.query)
|
const result = await executeQuery(sql, params.query)
|
||||||
|
|
||||||
logger.info(`[${requestId}] Query executed successfully, returned ${result.rowCount} rows`)
|
logger.info(`[${requestId}] Query executed successfully, returned ${result.rowCount} rows`)
|
||||||
|
|
||||||
@@ -46,7 +47,7 @@ export async function POST(request: NextRequest) {
|
|||||||
rowCount: result.rowCount,
|
rowCount: result.rowCount,
|
||||||
})
|
})
|
||||||
} finally {
|
} finally {
|
||||||
await client.end()
|
await sql.end()
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (error instanceof z.ZodError) {
|
if (error instanceof z.ZodError) {
|
||||||
|
|||||||
@@ -1,11 +1,8 @@
|
|||||||
|
import { randomUUID } from 'crypto'
|
||||||
import { type NextRequest, NextResponse } from 'next/server'
|
import { type NextRequest, NextResponse } from 'next/server'
|
||||||
import { z } from 'zod'
|
import { z } from 'zod'
|
||||||
import { createLogger } from '@/lib/logs/console/logger'
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
import {
|
import { createPostgresConnection, executeUpdate } from '@/app/api/tools/postgresql/utils'
|
||||||
buildUpdateQuery,
|
|
||||||
createPostgresConnection,
|
|
||||||
executeQuery,
|
|
||||||
} from '@/app/api/tools/postgresql/utils'
|
|
||||||
|
|
||||||
const logger = createLogger('PostgreSQLUpdateAPI')
|
const logger = createLogger('PostgreSQLUpdateAPI')
|
||||||
|
|
||||||
@@ -15,7 +12,7 @@ const UpdateSchema = z.object({
|
|||||||
database: z.string().min(1, 'Database name is required'),
|
database: z.string().min(1, 'Database name is required'),
|
||||||
username: z.string().min(1, 'Username is required'),
|
username: z.string().min(1, 'Username is required'),
|
||||||
password: z.string().min(1, 'Password is required'),
|
password: z.string().min(1, 'Password is required'),
|
||||||
ssl: z.enum(['disabled', 'required', 'preferred']).default('required'),
|
ssl: z.enum(['disabled', 'required', 'preferred']).default('preferred'),
|
||||||
table: z.string().min(1, 'Table name is required'),
|
table: z.string().min(1, 'Table name is required'),
|
||||||
data: z.union([
|
data: z.union([
|
||||||
z
|
z
|
||||||
@@ -40,7 +37,7 @@ const UpdateSchema = z.object({
|
|||||||
})
|
})
|
||||||
|
|
||||||
export async function POST(request: NextRequest) {
|
export async function POST(request: NextRequest) {
|
||||||
const requestId = crypto.randomUUID().slice(0, 8)
|
const requestId = randomUUID().slice(0, 8)
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const body = await request.json()
|
const body = await request.json()
|
||||||
@@ -50,7 +47,7 @@ export async function POST(request: NextRequest) {
|
|||||||
`[${requestId}] Updating data in ${params.table} on ${params.host}:${params.port}/${params.database}`
|
`[${requestId}] Updating data in ${params.table} on ${params.host}:${params.port}/${params.database}`
|
||||||
)
|
)
|
||||||
|
|
||||||
const client = await createPostgresConnection({
|
const sql = createPostgresConnection({
|
||||||
host: params.host,
|
host: params.host,
|
||||||
port: params.port,
|
port: params.port,
|
||||||
database: params.database,
|
database: params.database,
|
||||||
@@ -60,8 +57,7 @@ export async function POST(request: NextRequest) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const { query, values } = buildUpdateQuery(params.table, params.data, params.where)
|
const result = await executeUpdate(sql, params.table, params.data, params.where)
|
||||||
const result = await executeQuery(client, query, values)
|
|
||||||
|
|
||||||
logger.info(`[${requestId}] Update executed successfully, ${result.rowCount} row(s) updated`)
|
logger.info(`[${requestId}] Update executed successfully, ${result.rowCount} row(s) updated`)
|
||||||
|
|
||||||
@@ -71,7 +67,7 @@ export async function POST(request: NextRequest) {
|
|||||||
rowCount: result.rowCount,
|
rowCount: result.rowCount,
|
||||||
})
|
})
|
||||||
} finally {
|
} finally {
|
||||||
await client.end()
|
await sql.end()
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (error instanceof z.ZodError) {
|
if (error instanceof z.ZodError) {
|
||||||
|
|||||||
@@ -1,43 +1,41 @@
|
|||||||
import { Client } from 'pg'
|
import postgres from 'postgres'
|
||||||
import type { PostgresConnectionConfig } from '@/tools/postgresql/types'
|
import type { PostgresConnectionConfig } from '@/tools/postgresql/types'
|
||||||
|
|
||||||
export async function createPostgresConnection(config: PostgresConnectionConfig): Promise<Client> {
|
export function createPostgresConnection(config: PostgresConnectionConfig) {
|
||||||
const client = new Client({
|
const sslConfig =
|
||||||
|
config.ssl === 'disabled'
|
||||||
|
? false
|
||||||
|
: config.ssl === 'required'
|
||||||
|
? 'require'
|
||||||
|
: config.ssl === 'preferred'
|
||||||
|
? 'prefer'
|
||||||
|
: 'require'
|
||||||
|
|
||||||
|
const sql = postgres({
|
||||||
host: config.host,
|
host: config.host,
|
||||||
port: config.port,
|
port: config.port,
|
||||||
database: config.database,
|
database: config.database,
|
||||||
user: config.username,
|
username: config.username,
|
||||||
password: config.password,
|
password: config.password,
|
||||||
ssl:
|
ssl: sslConfig,
|
||||||
config.ssl === 'disabled'
|
connect_timeout: 10, // 10 seconds
|
||||||
? false
|
idle_timeout: 20, // 20 seconds
|
||||||
: config.ssl === 'required'
|
max_lifetime: 60 * 30, // 30 minutes
|
||||||
? true
|
max: 1, // Single connection for tool usage
|
||||||
: config.ssl === 'preferred'
|
|
||||||
? { rejectUnauthorized: false }
|
|
||||||
: false,
|
|
||||||
connectionTimeoutMillis: 10000, // 10 seconds
|
|
||||||
query_timeout: 30000, // 30 seconds
|
|
||||||
})
|
})
|
||||||
|
|
||||||
try {
|
return sql
|
||||||
await client.connect()
|
|
||||||
return client
|
|
||||||
} catch (error) {
|
|
||||||
await client.end()
|
|
||||||
throw error
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function executeQuery(
|
export async function executeQuery(
|
||||||
client: Client,
|
sql: any,
|
||||||
query: string,
|
query: string,
|
||||||
params: unknown[] = []
|
params: unknown[] = []
|
||||||
): Promise<{ rows: unknown[]; rowCount: number }> {
|
): Promise<{ rows: unknown[]; rowCount: number }> {
|
||||||
const result = await client.query(query, params)
|
const result = await sql.unsafe(query, params)
|
||||||
return {
|
return {
|
||||||
rows: result.rows || [],
|
rows: Array.isArray(result) ? result : [result],
|
||||||
rowCount: result.rowCount || 0,
|
rowCount: Array.isArray(result) ? result.length : result ? 1 : 0,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -84,7 +82,6 @@ export function validateQuery(query: string): { isValid: boolean; error?: string
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Only allow specific statement types for execute endpoint
|
|
||||||
const allowedStatements = /^(select|insert|update|delete|with|explain|analyze|show)\s+/i
|
const allowedStatements = /^(select|insert|update|delete|with|explain|analyze|show)\s+/i
|
||||||
if (!allowedStatements.test(trimmedQuery)) {
|
if (!allowedStatements.test(trimmedQuery)) {
|
||||||
return {
|
return {
|
||||||
@@ -98,7 +95,6 @@ export function validateQuery(query: string): { isValid: boolean; error?: string
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function sanitizeIdentifier(identifier: string): string {
|
export function sanitizeIdentifier(identifier: string): string {
|
||||||
// Handle schema.table format
|
|
||||||
if (identifier.includes('.')) {
|
if (identifier.includes('.')) {
|
||||||
const parts = identifier.split('.')
|
const parts = identifier.split('.')
|
||||||
return parts.map((part) => sanitizeSingleIdentifier(part)).join('.')
|
return parts.map((part) => sanitizeSingleIdentifier(part)).join('.')
|
||||||
@@ -107,28 +103,41 @@ export function sanitizeIdentifier(identifier: string): string {
|
|||||||
return sanitizeSingleIdentifier(identifier)
|
return sanitizeSingleIdentifier(identifier)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function validateWhereClause(where: string): void {
|
||||||
|
const dangerousPatterns = [
|
||||||
|
/;\s*(drop|delete|insert|update|create|alter|grant|revoke)/i,
|
||||||
|
/union\s+select/i,
|
||||||
|
/into\s+outfile/i,
|
||||||
|
/load_file/i,
|
||||||
|
/--/,
|
||||||
|
/\/\*/,
|
||||||
|
/\*\//,
|
||||||
|
]
|
||||||
|
|
||||||
|
for (const pattern of dangerousPatterns) {
|
||||||
|
if (pattern.test(where)) {
|
||||||
|
throw new Error('WHERE clause contains potentially dangerous operation')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function sanitizeSingleIdentifier(identifier: string): string {
|
function sanitizeSingleIdentifier(identifier: string): string {
|
||||||
// Remove any existing double quotes to prevent double-escaping
|
|
||||||
const cleaned = identifier.replace(/"/g, '')
|
const cleaned = identifier.replace(/"/g, '')
|
||||||
|
|
||||||
// Validate identifier contains only safe characters
|
|
||||||
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(cleaned)) {
|
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(cleaned)) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`Invalid identifier: ${identifier}. Identifiers must start with a letter or underscore and contain only letters, numbers, and underscores.`
|
`Invalid identifier: ${identifier}. Identifiers must start with a letter or underscore and contain only letters, numbers, and underscores.`
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Wrap in double quotes for PostgreSQL
|
|
||||||
return `"${cleaned}"`
|
return `"${cleaned}"`
|
||||||
}
|
}
|
||||||
|
|
||||||
export function buildInsertQuery(
|
export async function executeInsert(
|
||||||
|
sql: any,
|
||||||
table: string,
|
table: string,
|
||||||
data: Record<string, unknown>
|
data: Record<string, unknown>
|
||||||
): {
|
): Promise<{ rows: unknown[]; rowCount: number }> {
|
||||||
query: string
|
|
||||||
values: unknown[]
|
|
||||||
} {
|
|
||||||
const sanitizedTable = sanitizeIdentifier(table)
|
const sanitizedTable = sanitizeIdentifier(table)
|
||||||
const columns = Object.keys(data)
|
const columns = Object.keys(data)
|
||||||
const sanitizedColumns = columns.map((col) => sanitizeIdentifier(col))
|
const sanitizedColumns = columns.map((col) => sanitizeIdentifier(col))
|
||||||
@@ -136,18 +145,22 @@ export function buildInsertQuery(
|
|||||||
const values = columns.map((col) => data[col])
|
const values = columns.map((col) => data[col])
|
||||||
|
|
||||||
const query = `INSERT INTO ${sanitizedTable} (${sanitizedColumns.join(', ')}) VALUES (${placeholders.join(', ')}) RETURNING *`
|
const query = `INSERT INTO ${sanitizedTable} (${sanitizedColumns.join(', ')}) VALUES (${placeholders.join(', ')}) RETURNING *`
|
||||||
|
const result = await sql.unsafe(query, values)
|
||||||
|
|
||||||
return { query, values }
|
return {
|
||||||
|
rows: Array.isArray(result) ? result : [result],
|
||||||
|
rowCount: Array.isArray(result) ? result.length : result ? 1 : 0,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export function buildUpdateQuery(
|
export async function executeUpdate(
|
||||||
|
sql: any,
|
||||||
table: string,
|
table: string,
|
||||||
data: Record<string, unknown>,
|
data: Record<string, unknown>,
|
||||||
where: string
|
where: string
|
||||||
): {
|
): Promise<{ rows: unknown[]; rowCount: number }> {
|
||||||
query: string
|
validateWhereClause(where)
|
||||||
values: unknown[]
|
|
||||||
} {
|
|
||||||
const sanitizedTable = sanitizeIdentifier(table)
|
const sanitizedTable = sanitizeIdentifier(table)
|
||||||
const columns = Object.keys(data)
|
const columns = Object.keys(data)
|
||||||
const sanitizedColumns = columns.map((col) => sanitizeIdentifier(col))
|
const sanitizedColumns = columns.map((col) => sanitizeIdentifier(col))
|
||||||
@@ -155,19 +168,27 @@ export function buildUpdateQuery(
|
|||||||
const values = columns.map((col) => data[col])
|
const values = columns.map((col) => data[col])
|
||||||
|
|
||||||
const query = `UPDATE ${sanitizedTable} SET ${setClause} WHERE ${where} RETURNING *`
|
const query = `UPDATE ${sanitizedTable} SET ${setClause} WHERE ${where} RETURNING *`
|
||||||
|
const result = await sql.unsafe(query, values)
|
||||||
|
|
||||||
return { query, values }
|
return {
|
||||||
|
rows: Array.isArray(result) ? result : [result],
|
||||||
|
rowCount: Array.isArray(result) ? result.length : result ? 1 : 0,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export function buildDeleteQuery(
|
export async function executeDelete(
|
||||||
|
sql: any,
|
||||||
table: string,
|
table: string,
|
||||||
where: string
|
where: string
|
||||||
): {
|
): Promise<{ rows: unknown[]; rowCount: number }> {
|
||||||
query: string
|
validateWhereClause(where)
|
||||||
values: unknown[]
|
|
||||||
} {
|
|
||||||
const sanitizedTable = sanitizeIdentifier(table)
|
const sanitizedTable = sanitizeIdentifier(table)
|
||||||
const query = `DELETE FROM ${sanitizedTable} WHERE ${where} RETURNING *`
|
const query = `DELETE FROM ${sanitizedTable} WHERE ${where} RETURNING *`
|
||||||
|
const result = await sql.unsafe(query, [])
|
||||||
|
|
||||||
return { query, values: [] }
|
return {
|
||||||
|
rows: Array.isArray(result) ? result : [result],
|
||||||
|
rowCount: Array.isArray(result) ? result.length : result ? 1 : 0,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import { env } from '@/lib/env'
|
|||||||
import { createLogger } from '@/lib/logs/console/logger'
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
|
|
||||||
export const dynamic = 'force-dynamic'
|
export const dynamic = 'force-dynamic'
|
||||||
export const runtime = 'edge'
|
export const runtime = 'nodejs'
|
||||||
export const maxDuration = 60
|
export const maxDuration = 60
|
||||||
|
|
||||||
const logger = createLogger('WandGenerateAPI')
|
const logger = createLogger('WandGenerateAPI')
|
||||||
@@ -49,6 +49,15 @@ interface RequestBody {
|
|||||||
history?: ChatMessage[]
|
history?: ChatMessage[]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Helper: safe stringify for error payloads that may include circular structures
|
||||||
|
function safeStringify(value: unknown): string {
|
||||||
|
try {
|
||||||
|
return JSON.stringify(value)
|
||||||
|
} catch {
|
||||||
|
return '[unserializable]'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export async function POST(req: NextRequest) {
|
export async function POST(req: NextRequest) {
|
||||||
const requestId = crypto.randomUUID().slice(0, 8)
|
const requestId = crypto.randomUUID().slice(0, 8)
|
||||||
logger.info(`[${requestId}] Received wand generation request`)
|
logger.info(`[${requestId}] Received wand generation request`)
|
||||||
@@ -110,124 +119,172 @@ export async function POST(req: NextRequest) {
|
|||||||
`[${requestId}] About to create stream with model: ${useWandAzure ? wandModelName : 'gpt-4o'}`
|
`[${requestId}] About to create stream with model: ${useWandAzure ? wandModelName : 'gpt-4o'}`
|
||||||
)
|
)
|
||||||
|
|
||||||
// Add AbortController with timeout
|
// Use native fetch for streaming to avoid OpenAI SDK issues with Node.js runtime
|
||||||
const abortController = new AbortController()
|
const apiUrl = useWandAzure
|
||||||
const timeoutId = setTimeout(() => {
|
? `${azureEndpoint}/openai/deployments/${wandModelName}/chat/completions?api-version=${azureApiVersion}`
|
||||||
abortController.abort('Stream timeout after 30 seconds')
|
: 'https://api.openai.com/v1/chat/completions'
|
||||||
}, 30000)
|
|
||||||
|
|
||||||
// Forward request abort signal if available
|
const headers: Record<string, string> = {
|
||||||
req.signal?.addEventListener('abort', () => {
|
'Content-Type': 'application/json',
|
||||||
abortController.abort('Request cancelled by client')
|
}
|
||||||
})
|
|
||||||
|
|
||||||
const streamCompletion = await client.chat.completions.create(
|
if (useWandAzure) {
|
||||||
{
|
headers['api-key'] = azureApiKey!
|
||||||
|
} else {
|
||||||
|
headers.Authorization = `Bearer ${openaiApiKey}`
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.debug(`[${requestId}] Making streaming request to: ${apiUrl}`)
|
||||||
|
|
||||||
|
const response = await fetch(apiUrl, {
|
||||||
|
method: 'POST',
|
||||||
|
headers,
|
||||||
|
body: JSON.stringify({
|
||||||
model: useWandAzure ? wandModelName : 'gpt-4o',
|
model: useWandAzure ? wandModelName : 'gpt-4o',
|
||||||
messages: messages,
|
messages: messages,
|
||||||
temperature: 0.3,
|
temperature: 0.3,
|
||||||
max_tokens: 10000,
|
max_tokens: 10000,
|
||||||
stream: true,
|
stream: true,
|
||||||
stream_options: { include_usage: true },
|
stream_options: { include_usage: true },
|
||||||
},
|
|
||||||
{
|
|
||||||
signal: abortController.signal, // Add AbortSignal
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
clearTimeout(timeoutId) // Clear timeout after successful creation
|
|
||||||
logger.info(`[${requestId}] Stream created successfully, starting reader pattern`)
|
|
||||||
|
|
||||||
logger.debug(`[${requestId}] Stream connection established successfully`)
|
|
||||||
|
|
||||||
return new Response(
|
|
||||||
new ReadableStream({
|
|
||||||
async start(controller) {
|
|
||||||
const encoder = new TextEncoder()
|
|
||||||
|
|
||||||
try {
|
|
||||||
logger.info(`[${requestId}] Starting streaming with timeout protection`)
|
|
||||||
let chunkCount = 0
|
|
||||||
let hasUsageData = false
|
|
||||||
|
|
||||||
// Use for await with AbortController timeout protection
|
|
||||||
for await (const chunk of streamCompletion) {
|
|
||||||
chunkCount++
|
|
||||||
|
|
||||||
if (chunkCount === 1) {
|
|
||||||
logger.info(`[${requestId}] Received first chunk via for await`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Process the chunk
|
|
||||||
const content = chunk.choices?.[0]?.delta?.content || ''
|
|
||||||
if (content) {
|
|
||||||
// Use SSE format identical to chat streaming
|
|
||||||
controller.enqueue(
|
|
||||||
encoder.encode(`data: ${JSON.stringify({ chunk: content })}\n\n`)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check for usage data
|
|
||||||
if (chunk.usage) {
|
|
||||||
hasUsageData = true
|
|
||||||
logger.info(
|
|
||||||
`[${requestId}] Received usage data: ${JSON.stringify(chunk.usage)}`
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Log every 5th chunk to avoid spam
|
|
||||||
if (chunkCount % 5 === 0) {
|
|
||||||
logger.debug(`[${requestId}] Processed ${chunkCount} chunks so far`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
`[${requestId}] Reader pattern completed. Total chunks: ${chunkCount}, Usage data received: ${hasUsageData}`
|
|
||||||
)
|
|
||||||
|
|
||||||
// Send completion signal in SSE format
|
|
||||||
logger.info(`[${requestId}] Sending completion signal`)
|
|
||||||
controller.enqueue(encoder.encode(`data: ${JSON.stringify({ done: true })}\n\n`))
|
|
||||||
|
|
||||||
logger.info(`[${requestId}] Closing controller`)
|
|
||||||
controller.close()
|
|
||||||
|
|
||||||
logger.info(`[${requestId}] Wand generation streaming completed successfully`)
|
|
||||||
} catch (streamError: any) {
|
|
||||||
if (streamError.name === 'AbortError') {
|
|
||||||
logger.info(
|
|
||||||
`[${requestId}] Stream was aborted (timeout or cancel): ${streamError.message}`
|
|
||||||
)
|
|
||||||
controller.enqueue(
|
|
||||||
encoder.encode(
|
|
||||||
`data: ${JSON.stringify({ error: 'Stream cancelled', done: true })}\n\n`
|
|
||||||
)
|
|
||||||
)
|
|
||||||
} else {
|
|
||||||
logger.error(`[${requestId}] Streaming error`, { error: streamError.message })
|
|
||||||
controller.enqueue(
|
|
||||||
encoder.encode(
|
|
||||||
`data: ${JSON.stringify({ error: 'Streaming failed', done: true })}\n\n`
|
|
||||||
)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
controller.close()
|
|
||||||
}
|
|
||||||
},
|
|
||||||
}),
|
}),
|
||||||
{
|
})
|
||||||
headers: {
|
|
||||||
'Content-Type': 'text/event-stream',
|
if (!response.ok) {
|
||||||
'Cache-Control': 'no-cache',
|
const errorText = await response.text()
|
||||||
Connection: 'keep-alive',
|
logger.error(`[${requestId}] API request failed`, {
|
||||||
'X-Accel-Buffering': 'no',
|
status: response.status,
|
||||||
},
|
statusText: response.statusText,
|
||||||
}
|
error: errorText,
|
||||||
)
|
})
|
||||||
|
throw new Error(`API request failed: ${response.status} ${response.statusText}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(`[${requestId}] Stream response received, starting processing`)
|
||||||
|
|
||||||
|
// Create a TransformStream to process the SSE data
|
||||||
|
const encoder = new TextEncoder()
|
||||||
|
const decoder = new TextDecoder()
|
||||||
|
|
||||||
|
const readable = new ReadableStream({
|
||||||
|
async start(controller) {
|
||||||
|
const reader = response.body?.getReader()
|
||||||
|
if (!reader) {
|
||||||
|
controller.close()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
let buffer = ''
|
||||||
|
let chunkCount = 0
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
const { done, value } = await reader.read()
|
||||||
|
|
||||||
|
if (done) {
|
||||||
|
logger.info(`[${requestId}] Stream completed. Total chunks: ${chunkCount}`)
|
||||||
|
controller.enqueue(encoder.encode(`data: ${JSON.stringify({ done: true })}\n\n`))
|
||||||
|
controller.close()
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
// Decode the chunk
|
||||||
|
buffer += decoder.decode(value, { stream: true })
|
||||||
|
|
||||||
|
// Process complete SSE messages
|
||||||
|
const lines = buffer.split('\n')
|
||||||
|
buffer = lines.pop() || '' // Keep incomplete line in buffer
|
||||||
|
|
||||||
|
for (const line of lines) {
|
||||||
|
if (line.startsWith('data: ')) {
|
||||||
|
const data = line.slice(6).trim()
|
||||||
|
|
||||||
|
if (data === '[DONE]') {
|
||||||
|
logger.info(`[${requestId}] Received [DONE] signal`)
|
||||||
|
controller.enqueue(
|
||||||
|
encoder.encode(`data: ${JSON.stringify({ done: true })}\n\n`)
|
||||||
|
)
|
||||||
|
controller.close()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(data)
|
||||||
|
const content = parsed.choices?.[0]?.delta?.content
|
||||||
|
|
||||||
|
if (content) {
|
||||||
|
chunkCount++
|
||||||
|
if (chunkCount === 1) {
|
||||||
|
logger.info(`[${requestId}] Received first content chunk`)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Forward the content
|
||||||
|
controller.enqueue(
|
||||||
|
encoder.encode(`data: ${JSON.stringify({ chunk: content })}\n\n`)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Log usage if present
|
||||||
|
if (parsed.usage) {
|
||||||
|
logger.info(
|
||||||
|
`[${requestId}] Received usage data: ${JSON.stringify(parsed.usage)}`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Log progress periodically
|
||||||
|
if (chunkCount % 10 === 0) {
|
||||||
|
logger.debug(`[${requestId}] Processed ${chunkCount} chunks`)
|
||||||
|
}
|
||||||
|
} catch (parseError) {
|
||||||
|
// Skip invalid JSON lines
|
||||||
|
logger.debug(
|
||||||
|
`[${requestId}] Skipped non-JSON line: ${data.substring(0, 100)}`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(`[${requestId}] Wand generation streaming completed successfully`)
|
||||||
|
} catch (streamError: any) {
|
||||||
|
logger.error(`[${requestId}] Streaming error`, {
|
||||||
|
name: streamError?.name,
|
||||||
|
message: streamError?.message || 'Unknown error',
|
||||||
|
stack: streamError?.stack,
|
||||||
|
})
|
||||||
|
|
||||||
|
// Send error to client
|
||||||
|
const errorData = `data: ${JSON.stringify({ error: 'Streaming failed', done: true })}\n\n`
|
||||||
|
controller.enqueue(encoder.encode(errorData))
|
||||||
|
controller.close()
|
||||||
|
} finally {
|
||||||
|
reader.releaseLock()
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
// Return Response with proper headers for Node.js runtime
|
||||||
|
return new Response(readable, {
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'text/event-stream',
|
||||||
|
'Cache-Control': 'no-cache, no-transform',
|
||||||
|
Connection: 'keep-alive',
|
||||||
|
'X-Accel-Buffering': 'no', // Disable Nginx buffering
|
||||||
|
'Transfer-Encoding': 'chunked', // Important for Node.js runtime
|
||||||
|
},
|
||||||
|
})
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
logger.error(`[${requestId}] Streaming error`, {
|
logger.error(`[${requestId}] Failed to create stream`, {
|
||||||
error: error.message || 'Unknown error',
|
name: error?.name,
|
||||||
stack: error.stack,
|
message: error?.message || 'Unknown error',
|
||||||
|
code: error?.code,
|
||||||
|
status: error?.status,
|
||||||
|
responseStatus: error?.response?.status,
|
||||||
|
responseData: error?.response?.data ? safeStringify(error.response.data) : undefined,
|
||||||
|
stack: error?.stack,
|
||||||
|
useWandAzure,
|
||||||
|
model: useWandAzure ? wandModelName : 'gpt-4o',
|
||||||
|
endpoint: useWandAzure ? azureEndpoint : 'api.openai.com',
|
||||||
|
apiVersion: useWandAzure ? azureApiVersion : 'N/A',
|
||||||
})
|
})
|
||||||
|
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
@@ -261,8 +318,19 @@ export async function POST(req: NextRequest) {
|
|||||||
return NextResponse.json({ success: true, content: generatedContent })
|
return NextResponse.json({ success: true, content: generatedContent })
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
logger.error(`[${requestId}] Wand generation failed`, {
|
logger.error(`[${requestId}] Wand generation failed`, {
|
||||||
error: error.message || 'Unknown error',
|
name: error?.name,
|
||||||
stack: error.stack,
|
message: error?.message || 'Unknown error',
|
||||||
|
code: error?.code,
|
||||||
|
status: error?.status,
|
||||||
|
responseStatus: error instanceof OpenAI.APIError ? error.status : error?.response?.status,
|
||||||
|
responseData: (error as any)?.response?.data
|
||||||
|
? safeStringify((error as any).response.data)
|
||||||
|
: undefined,
|
||||||
|
stack: error?.stack,
|
||||||
|
useWandAzure,
|
||||||
|
model: useWandAzure ? wandModelName : 'gpt-4o',
|
||||||
|
endpoint: useWandAzure ? azureEndpoint : 'api.openai.com',
|
||||||
|
apiVersion: useWandAzure ? azureApiVersion : 'N/A',
|
||||||
})
|
})
|
||||||
|
|
||||||
let clientErrorMessage = 'Wand generation failed. Please try again later.'
|
let clientErrorMessage = 'Wand generation failed. Please try again later.'
|
||||||
|
|||||||
@@ -146,6 +146,8 @@ async function executeWorkflow(workflow: any, requestId: string, input?: any): P
|
|||||||
userId: workflow.userId,
|
userId: workflow.userId,
|
||||||
workspaceId: workflow.workspaceId,
|
workspaceId: workflow.workspaceId,
|
||||||
variables,
|
variables,
|
||||||
|
initialInput: processedInput || {},
|
||||||
|
executionType: 'api',
|
||||||
})
|
})
|
||||||
|
|
||||||
// Replace environment variables in the block states
|
// Replace environment variables in the block states
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import { LoggingSession } from '@/lib/logs/execution/logging-session'
|
|||||||
import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans'
|
import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans'
|
||||||
import { validateWorkflowAccess } from '@/app/api/workflows/middleware'
|
import { validateWorkflowAccess } from '@/app/api/workflows/middleware'
|
||||||
import { createErrorResponse, createSuccessResponse } from '@/app/api/workflows/utils'
|
import { createErrorResponse, createSuccessResponse } from '@/app/api/workflows/utils'
|
||||||
|
import { loadWorkflowStateForExecution } from '@/lib/logs/execution/logging-factory'
|
||||||
|
|
||||||
const logger = createLogger('WorkflowLogAPI')
|
const logger = createLogger('WorkflowLogAPI')
|
||||||
|
|
||||||
@@ -30,6 +31,22 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{
|
|||||||
success: result.success,
|
success: result.success,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Log current normalized state before starting logging session (what snapshot will save)
|
||||||
|
try {
|
||||||
|
const normalizedState = await loadWorkflowStateForExecution(id)
|
||||||
|
logger.info(`[${requestId}] 🔍 Normalized workflow state at persistence time:`, {
|
||||||
|
blocks: Object.entries(normalizedState.blocks || {}).map(([bid, b]: [string, any]) => ({
|
||||||
|
id: bid,
|
||||||
|
type: (b as any).type,
|
||||||
|
triggerMode: (b as any).triggerMode,
|
||||||
|
enabled: (b as any).enabled,
|
||||||
|
})),
|
||||||
|
edgesCount: (normalizedState.edges || []).length,
|
||||||
|
})
|
||||||
|
} catch (e) {
|
||||||
|
logger.warn(`[${requestId}] Failed to load normalized state for logging snapshot context`)
|
||||||
|
}
|
||||||
|
|
||||||
// Check if this execution is from chat using only the explicit source flag
|
// Check if this execution is from chat using only the explicit source flag
|
||||||
const isChatExecution = result.metadata?.source === 'chat'
|
const isChatExecution = result.metadata?.source === 'chat'
|
||||||
|
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import { createLogger } from '@/lib/logs/console/logger'
|
|||||||
import { getAssetUrl } from '@/lib/utils'
|
import { getAssetUrl } from '@/lib/utils'
|
||||||
import '@/app/globals.css'
|
import '@/app/globals.css'
|
||||||
|
|
||||||
import { SessionProvider } from '@/lib/session-context'
|
import { SessionProvider } from '@/lib/session/session-context'
|
||||||
import { ThemeProvider } from '@/app/theme-provider'
|
import { ThemeProvider } from '@/app/theme-provider'
|
||||||
import { ZoomPrevention } from '@/app/zoom-prevention'
|
import { ZoomPrevention } from '@/app/zoom-prevention'
|
||||||
|
|
||||||
|
|||||||
@@ -21,6 +21,10 @@ const ACCEPTED_FILE_TYPES = [
|
|||||||
'text/csv',
|
'text/csv',
|
||||||
'application/vnd.ms-excel',
|
'application/vnd.ms-excel',
|
||||||
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
||||||
|
'text/markdown',
|
||||||
|
'application/vnd.ms-powerpoint',
|
||||||
|
'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
||||||
|
'text/html',
|
||||||
]
|
]
|
||||||
|
|
||||||
interface FileWithPreview extends File {
|
interface FileWithPreview extends File {
|
||||||
@@ -74,7 +78,7 @@ export function UploadModal({
|
|||||||
return `File "${file.name}" is too large. Maximum size is 100MB.`
|
return `File "${file.name}" is too large. Maximum size is 100MB.`
|
||||||
}
|
}
|
||||||
if (!ACCEPTED_FILE_TYPES.includes(file.type)) {
|
if (!ACCEPTED_FILE_TYPES.includes(file.type)) {
|
||||||
return `File "${file.name}" has an unsupported format. Please use PDF, DOC, DOCX, TXT, CSV, XLS, or XLSX files.`
|
return `File "${file.name}" has an unsupported format. Please use PDF, DOC, DOCX, TXT, CSV, XLS, XLSX, MD, PPT, PPTX, or HTML files.`
|
||||||
}
|
}
|
||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
@@ -203,7 +207,8 @@ export function UploadModal({
|
|||||||
{isDragging ? 'Drop files here!' : 'Drop files here or click to browse'}
|
{isDragging ? 'Drop files here!' : 'Drop files here or click to browse'}
|
||||||
</p>
|
</p>
|
||||||
<p className='text-muted-foreground text-xs'>
|
<p className='text-muted-foreground text-xs'>
|
||||||
Supports PDF, DOC, DOCX, TXT, CSV, XLS, XLSX (max 100MB each)
|
Supports PDF, DOC, DOCX, TXT, CSV, XLS, XLSX, MD, PPT, PPTX, HTML (max 100MB
|
||||||
|
each)
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -29,6 +29,10 @@ const ACCEPTED_FILE_TYPES = [
|
|||||||
'text/csv',
|
'text/csv',
|
||||||
'application/vnd.ms-excel',
|
'application/vnd.ms-excel',
|
||||||
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
||||||
|
'text/markdown',
|
||||||
|
'application/vnd.ms-powerpoint',
|
||||||
|
'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
||||||
|
'text/html',
|
||||||
]
|
]
|
||||||
|
|
||||||
interface FileWithPreview extends File {
|
interface FileWithPreview extends File {
|
||||||
@@ -168,7 +172,7 @@ export function CreateModal({ open, onOpenChange, onKnowledgeBaseCreated }: Crea
|
|||||||
// Check file type
|
// Check file type
|
||||||
if (!ACCEPTED_FILE_TYPES.includes(file.type)) {
|
if (!ACCEPTED_FILE_TYPES.includes(file.type)) {
|
||||||
setFileError(
|
setFileError(
|
||||||
`File ${file.name} has an unsupported format. Please use PDF, DOC, DOCX, TXT, CSV, XLS, or XLSX.`
|
`File ${file.name} has an unsupported format. Please use PDF, DOC, DOCX, TXT, CSV, XLS, XLSX, MD, PPT, PPTX, or HTML.`
|
||||||
)
|
)
|
||||||
hasError = true
|
hasError = true
|
||||||
continue
|
continue
|
||||||
@@ -511,7 +515,8 @@ export function CreateModal({ open, onOpenChange, onKnowledgeBaseCreated }: Crea
|
|||||||
: 'Drop files here or click to browse'}
|
: 'Drop files here or click to browse'}
|
||||||
</p>
|
</p>
|
||||||
<p className='text-muted-foreground text-xs'>
|
<p className='text-muted-foreground text-xs'>
|
||||||
Supports PDF, DOC, DOCX, TXT, CSV, XLS, XLSX (max 100MB each)
|
Supports PDF, DOC, DOCX, TXT, CSV, XLS, XLSX, MD, PPT, PPTX, HTML (max
|
||||||
|
100MB each)
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -552,7 +557,8 @@ export function CreateModal({ open, onOpenChange, onKnowledgeBaseCreated }: Crea
|
|||||||
: 'Drop more files or click to browse'}
|
: 'Drop more files or click to browse'}
|
||||||
</p>
|
</p>
|
||||||
<p className='text-muted-foreground text-xs'>
|
<p className='text-muted-foreground text-xs'>
|
||||||
PDF, DOC, DOCX, TXT, CSV, XLS, XLSX (max 100MB each)
|
PDF, DOC, DOCX, TXT, CSV, XLS, XLSX, MD, PPT, PPTX, HTML (max 100MB
|
||||||
|
each)
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ import {
|
|||||||
TooltipProvider,
|
TooltipProvider,
|
||||||
TooltipTrigger,
|
TooltipTrigger,
|
||||||
} from '@/components/ui'
|
} from '@/components/ui'
|
||||||
import { MAX_TAG_SLOTS, type TagSlot } from '@/lib/constants/knowledge'
|
import { MAX_TAG_SLOTS, type TagSlot } from '@/lib/knowledge/consts'
|
||||||
import { createLogger } from '@/lib/logs/console/logger'
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
import { useKnowledgeBaseTagDefinitions } from '@/hooks/use-knowledge-base-tag-definitions'
|
import { useKnowledgeBaseTagDefinitions } from '@/hooks/use-knowledge-base-tag-definitions'
|
||||||
import { useNextAvailableSlot } from '@/hooks/use-next-available-slot'
|
import { useNextAvailableSlot } from '@/hooks/use-next-available-slot'
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import { Button } from '@/components/ui/button'
|
|||||||
import { Collapsible, CollapsibleContent, CollapsibleTrigger } from '@/components/ui/collapsible'
|
import { Collapsible, CollapsibleContent, CollapsibleTrigger } from '@/components/ui/collapsible'
|
||||||
import { Input } from '@/components/ui/input'
|
import { Input } from '@/components/ui/input'
|
||||||
import { Label } from '@/components/ui/label'
|
import { Label } from '@/components/ui/label'
|
||||||
import { TAG_SLOTS, type TagSlot } from '@/lib/constants/knowledge'
|
import { TAG_SLOTS, type TagSlot } from '@/lib/knowledge/consts'
|
||||||
import { useKnowledgeBaseTagDefinitions } from '@/hooks/use-knowledge-base-tag-definitions'
|
import { useKnowledgeBaseTagDefinitions } from '@/hooks/use-knowledge-base-tag-definitions'
|
||||||
|
|
||||||
export type TagData = {
|
export type TagData = {
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import { TraceSpansDisplay } from '@/app/workspace/[workspaceId]/logs/components
|
|||||||
import { formatDate } from '@/app/workspace/[workspaceId]/logs/utils/format-date'
|
import { formatDate } from '@/app/workspace/[workspaceId]/logs/utils/format-date'
|
||||||
import { formatCost } from '@/providers/utils'
|
import { formatCost } from '@/providers/utils'
|
||||||
import type { WorkflowLog } from '@/stores/logs/filters/types'
|
import type { WorkflowLog } from '@/stores/logs/filters/types'
|
||||||
|
import { useParams, useRouter } from 'next/navigation'
|
||||||
|
|
||||||
interface LogSidebarProps {
|
interface LogSidebarProps {
|
||||||
log: WorkflowLog | null
|
log: WorkflowLog | null
|
||||||
@@ -199,6 +200,8 @@ export function Sidebar({
|
|||||||
const [isModelsExpanded, setIsModelsExpanded] = useState(false)
|
const [isModelsExpanded, setIsModelsExpanded] = useState(false)
|
||||||
const [isFrozenCanvasOpen, setIsFrozenCanvasOpen] = useState(false)
|
const [isFrozenCanvasOpen, setIsFrozenCanvasOpen] = useState(false)
|
||||||
const scrollAreaRef = useRef<HTMLDivElement>(null)
|
const scrollAreaRef = useRef<HTMLDivElement>(null)
|
||||||
|
const router = useRouter()
|
||||||
|
const params = useParams() as { workspaceId?: string }
|
||||||
|
|
||||||
// Update currentLogId when log changes
|
// Update currentLogId when log changes
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
@@ -529,15 +532,31 @@ export function Sidebar({
|
|||||||
<h3 className='mb-1 font-medium text-muted-foreground text-xs'>
|
<h3 className='mb-1 font-medium text-muted-foreground text-xs'>
|
||||||
Workflow State
|
Workflow State
|
||||||
</h3>
|
</h3>
|
||||||
<Button
|
<div className='flex w-full gap-2'>
|
||||||
variant='outline'
|
<Button
|
||||||
size='sm'
|
variant='outline'
|
||||||
onClick={() => setIsFrozenCanvasOpen(true)}
|
size='sm'
|
||||||
className='w-full justify-start gap-2'
|
onClick={() => setIsFrozenCanvasOpen(true)}
|
||||||
>
|
className='flex-1 justify-start gap-2'
|
||||||
<Eye className='h-4 w-4' />
|
>
|
||||||
View Snapshot
|
<Eye className='h-4 w-4' />
|
||||||
</Button>
|
View Snapshot
|
||||||
|
</Button>
|
||||||
|
<Button
|
||||||
|
variant='secondary'
|
||||||
|
size='sm'
|
||||||
|
onClick={() => {
|
||||||
|
try {
|
||||||
|
const href = `/workspace/${encodeURIComponent(String(params?.workspaceId || ''))}/w/${encodeURIComponent(String(log.workflowId || ''))}`
|
||||||
|
router.push(href)
|
||||||
|
} catch {}
|
||||||
|
}}
|
||||||
|
className='flex-1 justify-start gap-2'
|
||||||
|
>
|
||||||
|
<Eye className='h-4 w-4' />
|
||||||
|
Open Live Debug
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
<p className='mt-1 text-muted-foreground text-xs'>
|
<p className='mt-1 text-muted-foreground text-xs'>
|
||||||
See the exact workflow state and block inputs/outputs at execution time
|
See the exact workflow state and block inputs/outputs at execution time
|
||||||
</p>
|
</p>
|
||||||
|
|||||||
@@ -8,8 +8,6 @@ import {
|
|||||||
Layers,
|
Layers,
|
||||||
Play,
|
Play,
|
||||||
RefreshCw,
|
RefreshCw,
|
||||||
SkipForward,
|
|
||||||
StepForward,
|
|
||||||
Store,
|
Store,
|
||||||
Trash2,
|
Trash2,
|
||||||
WifiOff,
|
WifiOff,
|
||||||
@@ -44,6 +42,8 @@ import {
|
|||||||
getKeyboardShortcutText,
|
getKeyboardShortcutText,
|
||||||
useKeyboardShortcuts,
|
useKeyboardShortcuts,
|
||||||
} from '@/app/workspace/[workspaceId]/w/hooks/use-keyboard-shortcuts'
|
} from '@/app/workspace/[workspaceId]/w/hooks/use-keyboard-shortcuts'
|
||||||
|
import { useExecutionStore } from '@/stores/execution/store'
|
||||||
|
import { useDebugCanvasStore } from '@/stores/execution/debug-canvas/store'
|
||||||
import { useFolderStore } from '@/stores/folders/store'
|
import { useFolderStore } from '@/stores/folders/store'
|
||||||
import { usePanelStore } from '@/stores/panel/store'
|
import { usePanelStore } from '@/stores/panel/store'
|
||||||
import { useGeneralStore } from '@/stores/settings/general/store'
|
import { useGeneralStore } from '@/stores/settings/general/store'
|
||||||
@@ -111,6 +111,9 @@ export function ControlBar({ hasValidationErrors = false }: ControlBarProps) {
|
|||||||
const [isExpanded, setIsExpanded] = useState(false)
|
const [isExpanded, setIsExpanded] = useState(false)
|
||||||
const [isTemplateModalOpen, setIsTemplateModalOpen] = useState(false)
|
const [isTemplateModalOpen, setIsTemplateModalOpen] = useState(false)
|
||||||
const [isAutoLayouting, setIsAutoLayouting] = useState(false)
|
const [isAutoLayouting, setIsAutoLayouting] = useState(false)
|
||||||
|
// Remove chat modal state
|
||||||
|
// const [isChatPromptOpen, setIsChatPromptOpen] = useState(false)
|
||||||
|
// const [chatPrompt, setChatPrompt] = useState('')
|
||||||
|
|
||||||
// Delete workflow state - grouped for better organization
|
// Delete workflow state - grouped for better organization
|
||||||
const [deleteState, setDeleteState] = useState({
|
const [deleteState, setDeleteState] = useState({
|
||||||
@@ -146,6 +149,13 @@ export function ControlBar({ hasValidationErrors = false }: ControlBarProps) {
|
|||||||
}
|
}
|
||||||
}, [setActiveTab, isOpen, togglePanel])
|
}, [setActiveTab, isOpen, togglePanel])
|
||||||
|
|
||||||
|
const openDebugPanel = useCallback(() => {
|
||||||
|
setActiveTab('debug')
|
||||||
|
if (!isOpen) {
|
||||||
|
togglePanel()
|
||||||
|
}
|
||||||
|
}, [setActiveTab, isOpen, togglePanel])
|
||||||
|
|
||||||
// Shared condition for keyboard shortcut and button disabled state
|
// Shared condition for keyboard shortcut and button disabled state
|
||||||
const isWorkflowBlocked = isExecuting || hasValidationErrors
|
const isWorkflowBlocked = isExecuting || hasValidationErrors
|
||||||
|
|
||||||
@@ -808,6 +818,7 @@ export function ControlBar({ hasValidationErrors = false }: ControlBarProps) {
|
|||||||
|
|
||||||
if (isDebugging) {
|
if (isDebugging) {
|
||||||
// Stop debugging
|
// Stop debugging
|
||||||
|
try { useDebugCanvasStore.getState().clear() } catch {}
|
||||||
handleCancelDebug()
|
handleCancelDebug()
|
||||||
} else {
|
} else {
|
||||||
// Check if there are executable blocks before starting debug mode
|
// Check if there are executable blocks before starting debug mode
|
||||||
@@ -819,15 +830,31 @@ export function ControlBar({ hasValidationErrors = false }: ControlBarProps) {
|
|||||||
return // Do nothing if no executable blocks
|
return // Do nothing if no executable blocks
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start debugging
|
// Determine starter id for focus
|
||||||
|
const starter = Object.values(blocks).find((b) => b.type === 'starter') as any
|
||||||
|
const starterId = starter?.id as string | undefined
|
||||||
|
|
||||||
|
// Enable debug UI but do NOT start execution
|
||||||
if (!isDebugModeEnabled) {
|
if (!isDebugModeEnabled) {
|
||||||
toggleDebugMode()
|
toggleDebugMode()
|
||||||
}
|
}
|
||||||
if (usageExceeded) {
|
if (usageExceeded) {
|
||||||
openSubscriptionSettings()
|
openSubscriptionSettings()
|
||||||
} else {
|
} else {
|
||||||
openConsolePanel()
|
// Activate debug session state so the panel is active
|
||||||
handleRunWorkflow(undefined, true) // Start in debug mode
|
const execStore = useExecutionStore.getState()
|
||||||
|
execStore.setIsExecuting(false)
|
||||||
|
execStore.setIsDebugging(true)
|
||||||
|
// Set the Start block as pending - it will execute on first Step
|
||||||
|
execStore.setPendingBlocks(starterId ? [starterId] : [])
|
||||||
|
|
||||||
|
// Show Debug tab and mark starter as the current block to execute
|
||||||
|
openDebugPanel()
|
||||||
|
if (starterId) {
|
||||||
|
execStore.setActiveBlocks(new Set([starterId]))
|
||||||
|
}
|
||||||
|
// Ensure debug canvas starts in a clean state
|
||||||
|
try { useDebugCanvasStore.getState().clear() } catch {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}, [
|
}, [
|
||||||
@@ -838,8 +865,7 @@ export function ControlBar({ hasValidationErrors = false }: ControlBarProps) {
|
|||||||
blocks,
|
blocks,
|
||||||
handleCancelDebug,
|
handleCancelDebug,
|
||||||
toggleDebugMode,
|
toggleDebugMode,
|
||||||
handleRunWorkflow,
|
openDebugPanel,
|
||||||
openConsolePanel,
|
|
||||||
])
|
])
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -859,40 +885,7 @@ export function ControlBar({ hasValidationErrors = false }: ControlBarProps) {
|
|||||||
|
|
||||||
return (
|
return (
|
||||||
<div className='flex items-center gap-1'>
|
<div className='flex items-center gap-1'>
|
||||||
<Tooltip>
|
{/* Keep only cancel (X) here; step/resume moved to panel */}
|
||||||
<TooltipTrigger asChild>
|
|
||||||
<Button
|
|
||||||
onClick={() => {
|
|
||||||
openConsolePanel()
|
|
||||||
handleStepDebug()
|
|
||||||
}}
|
|
||||||
className={debugButtonClass}
|
|
||||||
disabled={isControlDisabled}
|
|
||||||
>
|
|
||||||
<StepForward className='h-5 w-5' />
|
|
||||||
<span className='sr-only'>Step Forward</span>
|
|
||||||
</Button>
|
|
||||||
</TooltipTrigger>
|
|
||||||
<TooltipContent>Step Forward</TooltipContent>
|
|
||||||
</Tooltip>
|
|
||||||
|
|
||||||
<Tooltip>
|
|
||||||
<TooltipTrigger asChild>
|
|
||||||
<Button
|
|
||||||
onClick={() => {
|
|
||||||
openConsolePanel()
|
|
||||||
handleResumeDebug()
|
|
||||||
}}
|
|
||||||
className={debugButtonClass}
|
|
||||||
disabled={isControlDisabled}
|
|
||||||
>
|
|
||||||
<SkipForward className='h-5 w-5' />
|
|
||||||
<span className='sr-only'>Resume Until End</span>
|
|
||||||
</Button>
|
|
||||||
</TooltipTrigger>
|
|
||||||
<TooltipContent>Resume Until End</TooltipContent>
|
|
||||||
</Tooltip>
|
|
||||||
|
|
||||||
<Tooltip>
|
<Tooltip>
|
||||||
<TooltipTrigger asChild>
|
<TooltipTrigger asChild>
|
||||||
<Button
|
<Button
|
||||||
@@ -1214,7 +1207,7 @@ export function ControlBar({ hasValidationErrors = false }: ControlBarProps) {
|
|||||||
{isExpanded && renderPublishButton()}
|
{isExpanded && renderPublishButton()}
|
||||||
{renderDeleteButton()}
|
{renderDeleteButton()}
|
||||||
{renderDuplicateButton()}
|
{renderDuplicateButton()}
|
||||||
{!isDebugging && renderDebugModeToggle()}
|
{renderDebugModeToggle()}
|
||||||
{renderDeployButton()}
|
{renderDeployButton()}
|
||||||
{isDebugging ? renderDebugControlsBar() : renderRunButton()}
|
{isDebugging ? renderDebugControlsBar() : renderRunButton()}
|
||||||
|
|
||||||
@@ -1226,6 +1219,8 @@ export function ControlBar({ hasValidationErrors = false }: ControlBarProps) {
|
|||||||
workflowId={activeWorkflowId}
|
workflowId={activeWorkflowId}
|
||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
{/* Removed chat prompt dialog; chat input now lives in DebugPanel */}
|
||||||
</div>
|
</div>
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -10,6 +10,7 @@ import {
|
|||||||
import { ScrollArea } from '@/components/ui/scroll-area'
|
import { ScrollArea } from '@/components/ui/scroll-area'
|
||||||
import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip'
|
import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip'
|
||||||
import { useCopilotStore } from '@/stores/copilot/store'
|
import { useCopilotStore } from '@/stores/copilot/store'
|
||||||
|
import { useExecutionStore } from '@/stores/execution/store'
|
||||||
import { useChatStore } from '@/stores/panel/chat/store'
|
import { useChatStore } from '@/stores/panel/chat/store'
|
||||||
import { useConsoleStore } from '@/stores/panel/console/store'
|
import { useConsoleStore } from '@/stores/panel/console/store'
|
||||||
import { usePanelStore } from '@/stores/panel/store'
|
import { usePanelStore } from '@/stores/panel/store'
|
||||||
@@ -17,6 +18,7 @@ import { useWorkflowRegistry } from '@/stores/workflows/registry/store'
|
|||||||
import { Chat } from './components/chat/chat'
|
import { Chat } from './components/chat/chat'
|
||||||
import { Console } from './components/console/console'
|
import { Console } from './components/console/console'
|
||||||
import { Copilot } from './components/copilot/copilot'
|
import { Copilot } from './components/copilot/copilot'
|
||||||
|
import { DebugPanel } from './components/debug/debug'
|
||||||
import { Variables } from './components/variables/variables'
|
import { Variables } from './components/variables/variables'
|
||||||
|
|
||||||
export function Panel() {
|
export function Panel() {
|
||||||
@@ -44,6 +46,9 @@ export function Panel() {
|
|||||||
const exportChatCSV = useChatStore((state) => state.exportChatCSV)
|
const exportChatCSV = useChatStore((state) => state.exportChatCSV)
|
||||||
const { activeWorkflowId } = useWorkflowRegistry()
|
const { activeWorkflowId } = useWorkflowRegistry()
|
||||||
|
|
||||||
|
// Get debug state
|
||||||
|
const isDebugging = useExecutionStore((state) => state.isDebugging)
|
||||||
|
|
||||||
// Copilot store for chat management
|
// Copilot store for chat management
|
||||||
const {
|
const {
|
||||||
chats,
|
chats,
|
||||||
@@ -216,7 +221,11 @@ export function Panel() {
|
|||||||
)
|
)
|
||||||
|
|
||||||
// Handle tab clicks - no loading, just switch tabs
|
// Handle tab clicks - no loading, just switch tabs
|
||||||
const handleTabClick = async (tab: 'chat' | 'console' | 'variables' | 'copilot') => {
|
const handleTabClick = async (tab: 'chat' | 'console' | 'variables' | 'copilot' | 'debug') => {
|
||||||
|
// Don't allow clicking debug tab if not debugging
|
||||||
|
if (tab === 'debug' && !isDebugging) {
|
||||||
|
return
|
||||||
|
}
|
||||||
setActiveTab(tab)
|
setActiveTab(tab)
|
||||||
if (!isOpen) {
|
if (!isOpen) {
|
||||||
togglePanel()
|
togglePanel()
|
||||||
@@ -284,10 +293,30 @@ export function Panel() {
|
|||||||
}
|
}
|
||||||
}, [activeWorkflowId, copilotWorkflowId, ensureCopilotDataLoaded])
|
}, [activeWorkflowId, copilotWorkflowId, ensureCopilotDataLoaded])
|
||||||
|
|
||||||
|
// When debug mode ends, switch to a different tab if debug was active
|
||||||
|
useEffect(() => {
|
||||||
|
if (!isDebugging && activeTab === 'debug') {
|
||||||
|
setActiveTab('console')
|
||||||
|
}
|
||||||
|
}, [isDebugging, activeTab, setActiveTab])
|
||||||
|
|
||||||
|
// When debug mode starts, automatically open the debug panel
|
||||||
|
useEffect(() => {
|
||||||
|
if (isDebugging) {
|
||||||
|
setActiveTab('debug')
|
||||||
|
if (!isOpen) {
|
||||||
|
togglePanel()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, [isDebugging, setActiveTab, isOpen, togglePanel])
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<>
|
<>
|
||||||
{/* Tab Selector - Always visible */}
|
{/* Tab Selector - Always visible */}
|
||||||
<div className='fixed top-[76px] right-4 z-20 flex h-9 w-[308px] items-center gap-1 rounded-[14px] border bg-card px-[2.5px] py-1 shadow-xs'>
|
<div
|
||||||
|
className='fixed top-[76px] right-4 z-20 flex h-9 items-center gap-1 rounded-[14px] border bg-card px-[2.5px] py-1 shadow-xs'
|
||||||
|
style={{ width: isDebugging ? '380px' : '308px' }}
|
||||||
|
>
|
||||||
<button
|
<button
|
||||||
onClick={() => handleTabClick('chat')}
|
onClick={() => handleTabClick('chat')}
|
||||||
className={`panel-tab-base inline-flex flex-1 cursor-pointer items-center justify-center rounded-[10px] border border-transparent py-1 font-[450] text-sm outline-none transition-colors duration-200 ${
|
className={`panel-tab-base inline-flex flex-1 cursor-pointer items-center justify-center rounded-[10px] border border-transparent py-1 font-[450] text-sm outline-none transition-colors duration-200 ${
|
||||||
@@ -320,6 +349,16 @@ export function Panel() {
|
|||||||
>
|
>
|
||||||
Variables
|
Variables
|
||||||
</button>
|
</button>
|
||||||
|
{isDebugging && (
|
||||||
|
<button
|
||||||
|
onClick={() => handleTabClick('debug')}
|
||||||
|
className={`panel-tab-base inline-flex flex-1 cursor-pointer items-center justify-center rounded-[10px] border border-transparent py-1 font-[450] text-sm outline-none transition-colors duration-200 ${
|
||||||
|
isOpen && activeTab === 'debug' ? 'panel-tab-active' : 'panel-tab-inactive'
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
Debug
|
||||||
|
</button>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* Panel Content - Only visible when isOpen is true */}
|
{/* Panel Content - Only visible when isOpen is true */}
|
||||||
@@ -512,6 +551,9 @@ export function Panel() {
|
|||||||
<div style={{ display: activeTab === 'variables' ? 'block' : 'none', height: '100%' }}>
|
<div style={{ display: activeTab === 'variables' ? 'block' : 'none', height: '100%' }}>
|
||||||
<Variables />
|
<Variables />
|
||||||
</div>
|
</div>
|
||||||
|
<div style={{ display: activeTab === 'debug' ? 'block' : 'none', height: '100%' }}>
|
||||||
|
<DebugPanel />
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|||||||
@@ -385,16 +385,16 @@ export function Code({
|
|||||||
|
|
||||||
<div
|
<div
|
||||||
className={cn(
|
className={cn(
|
||||||
'group relative min-h-[100px] rounded-md border bg-background font-mono text-sm transition-colors',
|
'group relative min-h-[100px] rounded-md border border-input bg-background font-mono text-sm transition-colors',
|
||||||
isConnecting && 'ring-2 ring-blue-500 ring-offset-2',
|
isConnecting && 'ring-2 ring-blue-500 ring-offset-2',
|
||||||
!isValidJson && 'border-2 border-destructive bg-destructive/10'
|
!isValidJson && 'border-destructive bg-destructive/10'
|
||||||
)}
|
)}
|
||||||
title={!isValidJson ? 'Invalid JSON' : undefined}
|
title={!isValidJson ? 'Invalid JSON' : undefined}
|
||||||
onDragOver={(e) => e.preventDefault()}
|
onDragOver={(e) => e.preventDefault()}
|
||||||
onDrop={handleDrop}
|
onDrop={handleDrop}
|
||||||
>
|
>
|
||||||
<div className='absolute top-2 right-3 z-10 flex items-center gap-1 opacity-0 transition-opacity group-hover:opacity-100'>
|
<div className='absolute top-2 right-3 z-10 flex items-center gap-1 opacity-0 transition-opacity group-hover:opacity-100'>
|
||||||
{!isCollapsed && !isAiStreaming && !isPreview && (
|
{wandConfig?.enabled && !isCollapsed && !isAiStreaming && !isPreview && (
|
||||||
<Button
|
<Button
|
||||||
variant='ghost'
|
variant='ghost'
|
||||||
size='icon'
|
size='icon'
|
||||||
@@ -486,7 +486,7 @@ export function Code({
|
|||||||
outline: 'none',
|
outline: 'none',
|
||||||
}}
|
}}
|
||||||
className={cn(
|
className={cn(
|
||||||
'code-editor-area caret-primary',
|
'code-editor-area caret-primary dark:caret-white',
|
||||||
'bg-transparent focus:outline-none',
|
'bg-transparent focus:outline-none',
|
||||||
(isCollapsed || isAiStreaming) && 'cursor-not-allowed opacity-50'
|
(isCollapsed || isAiStreaming) && 'cursor-not-allowed opacity-50'
|
||||||
)}
|
)}
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import { Button } from '@/components/ui/button'
|
|||||||
import { formatDisplayText } from '@/components/ui/formatted-text'
|
import { formatDisplayText } from '@/components/ui/formatted-text'
|
||||||
import { Input } from '@/components/ui/input'
|
import { Input } from '@/components/ui/input'
|
||||||
import { checkTagTrigger, TagDropdown } from '@/components/ui/tag-dropdown'
|
import { checkTagTrigger, TagDropdown } from '@/components/ui/tag-dropdown'
|
||||||
import { MAX_TAG_SLOTS } from '@/lib/constants/knowledge'
|
import { MAX_TAG_SLOTS } from '@/lib/knowledge/consts'
|
||||||
import { cn } from '@/lib/utils'
|
import { cn } from '@/lib/utils'
|
||||||
import { useSubBlockValue } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/hooks/use-sub-block-value'
|
import { useSubBlockValue } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/hooks/use-sub-block-value'
|
||||||
import type { SubBlockConfig } from '@/blocks/types'
|
import type { SubBlockConfig } from '@/blocks/types'
|
||||||
|
|||||||
@@ -1,12 +1,16 @@
|
|||||||
import { useEffect, useMemo, useRef, useState } from 'react'
|
import { useEffect, useMemo, useRef, useState } from 'react'
|
||||||
|
import { Wand2 } from 'lucide-react'
|
||||||
import { useReactFlow } from 'reactflow'
|
import { useReactFlow } from 'reactflow'
|
||||||
|
import { Button } from '@/components/ui/button'
|
||||||
import { checkEnvVarTrigger, EnvVarDropdown } from '@/components/ui/env-var-dropdown'
|
import { checkEnvVarTrigger, EnvVarDropdown } from '@/components/ui/env-var-dropdown'
|
||||||
import { formatDisplayText } from '@/components/ui/formatted-text'
|
import { formatDisplayText } from '@/components/ui/formatted-text'
|
||||||
import { Input } from '@/components/ui/input'
|
import { Input } from '@/components/ui/input'
|
||||||
import { checkTagTrigger, TagDropdown } from '@/components/ui/tag-dropdown'
|
import { checkTagTrigger, TagDropdown } from '@/components/ui/tag-dropdown'
|
||||||
import { createLogger } from '@/lib/logs/console/logger'
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
import { cn } from '@/lib/utils'
|
import { cn } from '@/lib/utils'
|
||||||
|
import { WandPromptBar } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/wand-prompt-bar/wand-prompt-bar'
|
||||||
import { useSubBlockValue } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/hooks/use-sub-block-value'
|
import { useSubBlockValue } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/components/sub-block/hooks/use-sub-block-value'
|
||||||
|
import { useWand } from '@/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-wand'
|
||||||
import type { SubBlockConfig } from '@/blocks/types'
|
import type { SubBlockConfig } from '@/blocks/types'
|
||||||
import { useTagSelection } from '@/hooks/use-tag-selection'
|
import { useTagSelection } from '@/hooks/use-tag-selection'
|
||||||
import { useOperationQueueStore } from '@/stores/operation-queue/store'
|
import { useOperationQueueStore } from '@/stores/operation-queue/store'
|
||||||
@@ -40,19 +44,39 @@ export function ShortInput({
|
|||||||
previewValue,
|
previewValue,
|
||||||
disabled = false,
|
disabled = false,
|
||||||
}: ShortInputProps) {
|
}: ShortInputProps) {
|
||||||
|
// Local state for immediate UI updates during streaming
|
||||||
|
const [localContent, setLocalContent] = useState<string>('')
|
||||||
const [isFocused, setIsFocused] = useState(false)
|
const [isFocused, setIsFocused] = useState(false)
|
||||||
const [showEnvVars, setShowEnvVars] = useState(false)
|
const [showEnvVars, setShowEnvVars] = useState(false)
|
||||||
const [showTags, setShowTags] = useState(false)
|
const [showTags, setShowTags] = useState(false)
|
||||||
const validatePropValue = (value: any): string => {
|
|
||||||
if (value === undefined || value === null) return ''
|
// Wand functionality (only if wandConfig is enabled)
|
||||||
if (typeof value === 'string') return value
|
const wandHook = config.wandConfig?.enabled
|
||||||
try {
|
? useWand({
|
||||||
return String(value)
|
wandConfig: config.wandConfig,
|
||||||
} catch {
|
currentValue: localContent,
|
||||||
return ''
|
onStreamStart: () => {
|
||||||
}
|
// Clear the content when streaming starts
|
||||||
}
|
setLocalContent('')
|
||||||
const [storeValue, setStoreValue] = useSubBlockValue(blockId, subBlockId)
|
},
|
||||||
|
onStreamChunk: (chunk) => {
|
||||||
|
// Update local content with each chunk as it arrives
|
||||||
|
setLocalContent((current) => current + chunk)
|
||||||
|
},
|
||||||
|
onGeneratedContent: (content) => {
|
||||||
|
// Final content update
|
||||||
|
setLocalContent(content)
|
||||||
|
},
|
||||||
|
})
|
||||||
|
: null
|
||||||
|
// State management - useSubBlockValue with explicit streaming control
|
||||||
|
const [storeValue, setStoreValue] = useSubBlockValue(blockId, subBlockId, false, {
|
||||||
|
isStreaming: wandHook?.isStreaming || false,
|
||||||
|
onStreamingEnd: () => {
|
||||||
|
logger.debug('Wand streaming ended, value persisted', { blockId, subBlockId })
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
const [searchTerm, setSearchTerm] = useState('')
|
const [searchTerm, setSearchTerm] = useState('')
|
||||||
const [cursorPosition, setCursorPosition] = useState(0)
|
const [cursorPosition, setCursorPosition] = useState(0)
|
||||||
const inputRef = useRef<HTMLInputElement>(null)
|
const inputRef = useRef<HTMLInputElement>(null)
|
||||||
@@ -65,7 +89,29 @@ export function ShortInput({
|
|||||||
const reactFlowInstance = useReactFlow()
|
const reactFlowInstance = useReactFlow()
|
||||||
|
|
||||||
// Use preview value when in preview mode, otherwise use store value or prop value
|
// Use preview value when in preview mode, otherwise use store value or prop value
|
||||||
const value = isPreview ? previewValue : propValue !== undefined ? propValue : storeValue
|
const baseValue = isPreview ? previewValue : propValue !== undefined ? propValue : storeValue
|
||||||
|
|
||||||
|
// During streaming, use local content; otherwise use base value
|
||||||
|
const value = wandHook?.isStreaming ? localContent : baseValue
|
||||||
|
|
||||||
|
// Sync local content with base value when not streaming
|
||||||
|
useEffect(() => {
|
||||||
|
if (!wandHook?.isStreaming) {
|
||||||
|
const baseValueString = baseValue?.toString() ?? ''
|
||||||
|
if (baseValueString !== localContent) {
|
||||||
|
setLocalContent(baseValueString)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, [baseValue, wandHook?.isStreaming])
|
||||||
|
|
||||||
|
// Update store value during streaming (but won't persist until streaming ends)
|
||||||
|
useEffect(() => {
|
||||||
|
if (wandHook?.isStreaming && localContent !== '') {
|
||||||
|
if (!isPreview && !disabled) {
|
||||||
|
setStoreValue(localContent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, [localContent, wandHook?.isStreaming, isPreview, disabled, setStoreValue])
|
||||||
|
|
||||||
// Check if this input is API key related
|
// Check if this input is API key related
|
||||||
const isApiKeyField = useMemo(() => {
|
const isApiKeyField = useMemo(() => {
|
||||||
@@ -297,91 +343,130 @@ export function ShortInput({
|
|||||||
}
|
}
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className='relative w-full'>
|
<>
|
||||||
<Input
|
<WandPromptBar
|
||||||
ref={inputRef}
|
isVisible={wandHook?.isPromptVisible || false}
|
||||||
className={cn(
|
isLoading={wandHook?.isLoading || false}
|
||||||
'allow-scroll w-full overflow-auto text-transparent caret-foreground placeholder:text-muted-foreground/50',
|
isStreaming={wandHook?.isStreaming || false}
|
||||||
isConnecting &&
|
promptValue={wandHook?.promptInputValue || ''}
|
||||||
config?.connectionDroppable !== false &&
|
onSubmit={(prompt: string) => wandHook?.generateStream({ prompt }) || undefined}
|
||||||
'ring-2 ring-blue-500 ring-offset-2 focus-visible:ring-blue-500'
|
onCancel={
|
||||||
)}
|
wandHook?.isStreaming
|
||||||
placeholder={placeholder ?? ''}
|
? wandHook?.cancelGeneration
|
||||||
type='text'
|
: wandHook?.hidePromptInline || (() => {})
|
||||||
value={displayValue}
|
}
|
||||||
onChange={handleChange}
|
onChange={(value: string) => wandHook?.updatePromptValue?.(value)}
|
||||||
onFocus={() => {
|
placeholder={config.wandConfig?.placeholder || 'Describe what you want to generate...'}
|
||||||
setIsFocused(true)
|
/>
|
||||||
|
|
||||||
// If this is an API key field, automatically show env vars dropdown
|
<div className='group relative w-full'>
|
||||||
if (isApiKeyField) {
|
<Input
|
||||||
setShowEnvVars(true)
|
ref={inputRef}
|
||||||
setSearchTerm('')
|
className={cn(
|
||||||
|
'allow-scroll w-full overflow-auto text-transparent caret-foreground placeholder:text-muted-foreground/50',
|
||||||
|
isConnecting &&
|
||||||
|
config?.connectionDroppable !== false &&
|
||||||
|
'ring-2 ring-blue-500 ring-offset-2 focus-visible:ring-blue-500'
|
||||||
|
)}
|
||||||
|
placeholder={placeholder ?? ''}
|
||||||
|
type='text'
|
||||||
|
value={displayValue}
|
||||||
|
onChange={handleChange}
|
||||||
|
onFocus={() => {
|
||||||
|
setIsFocused(true)
|
||||||
|
|
||||||
// Set cursor position to the end of the input
|
// If this is an API key field, automatically show env vars dropdown
|
||||||
const inputLength = value?.toString().length ?? 0
|
if (isApiKeyField) {
|
||||||
setCursorPosition(inputLength)
|
setShowEnvVars(true)
|
||||||
} else {
|
setSearchTerm('')
|
||||||
|
|
||||||
|
// Set cursor position to the end of the input
|
||||||
|
const inputLength = value?.toString().length ?? 0
|
||||||
|
setCursorPosition(inputLength)
|
||||||
|
} else {
|
||||||
|
setShowEnvVars(false)
|
||||||
|
setShowTags(false)
|
||||||
|
setSearchTerm('')
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
onBlur={() => {
|
||||||
|
setIsFocused(false)
|
||||||
setShowEnvVars(false)
|
setShowEnvVars(false)
|
||||||
setShowTags(false)
|
try {
|
||||||
setSearchTerm('')
|
useOperationQueueStore.getState().flushDebouncedForBlock(blockId)
|
||||||
}
|
} catch {}
|
||||||
}}
|
}}
|
||||||
onBlur={() => {
|
onDrop={handleDrop}
|
||||||
setIsFocused(false)
|
onDragOver={handleDragOver}
|
||||||
setShowEnvVars(false)
|
onScroll={handleScroll}
|
||||||
try {
|
onPaste={handlePaste}
|
||||||
useOperationQueueStore.getState().flushDebouncedForBlock(blockId)
|
onWheel={handleWheel}
|
||||||
} catch {}
|
onKeyDown={handleKeyDown}
|
||||||
}}
|
autoComplete='off'
|
||||||
onDrop={handleDrop}
|
style={{ overflowX: 'auto' }}
|
||||||
onDragOver={handleDragOver}
|
disabled={disabled}
|
||||||
onScroll={handleScroll}
|
/>
|
||||||
onPaste={handlePaste}
|
|
||||||
onWheel={handleWheel}
|
|
||||||
onKeyDown={handleKeyDown}
|
|
||||||
autoComplete='off'
|
|
||||||
style={{ overflowX: 'auto' }}
|
|
||||||
disabled={disabled}
|
|
||||||
/>
|
|
||||||
<div
|
|
||||||
ref={overlayRef}
|
|
||||||
className='pointer-events-none absolute inset-0 flex items-center overflow-x-auto bg-transparent px-3 text-sm'
|
|
||||||
style={{ overflowX: 'auto' }}
|
|
||||||
>
|
|
||||||
<div
|
<div
|
||||||
className='w-full whitespace-pre'
|
ref={overlayRef}
|
||||||
style={{ scrollbarWidth: 'none', minWidth: 'fit-content' }}
|
className='pointer-events-none absolute inset-0 flex items-center overflow-x-auto bg-transparent px-3 text-sm'
|
||||||
|
style={{ overflowX: 'auto' }}
|
||||||
>
|
>
|
||||||
{password && !isFocused
|
<div
|
||||||
? '•'.repeat(value?.toString().length ?? 0)
|
className='w-full whitespace-pre'
|
||||||
: formatDisplayText(value?.toString() ?? '', true)}
|
style={{ scrollbarWidth: 'none', minWidth: 'fit-content' }}
|
||||||
|
>
|
||||||
|
{password && !isFocused
|
||||||
|
? '•'.repeat(value?.toString().length ?? 0)
|
||||||
|
: formatDisplayText(value?.toString() ?? '', true)}
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
|
||||||
|
|
||||||
<EnvVarDropdown
|
{/* Wand Button */}
|
||||||
visible={showEnvVars}
|
{wandHook && !isPreview && !wandHook.isStreaming && (
|
||||||
onSelect={handleEnvVarSelect}
|
<div className='-translate-y-1/2 absolute top-1/2 right-3 z-10 flex items-center gap-1 opacity-0 transition-opacity group-hover:opacity-100'>
|
||||||
searchTerm={searchTerm}
|
<Button
|
||||||
inputValue={value?.toString() ?? ''}
|
variant='ghost'
|
||||||
cursorPosition={cursorPosition}
|
size='icon'
|
||||||
onClose={() => {
|
onClick={
|
||||||
setShowEnvVars(false)
|
wandHook.isPromptVisible ? wandHook.hidePromptInline : wandHook.showPromptInline
|
||||||
setSearchTerm('')
|
}
|
||||||
}}
|
disabled={wandHook.isLoading || wandHook.isStreaming || disabled}
|
||||||
/>
|
aria-label='Generate content with AI'
|
||||||
<TagDropdown
|
className='h-8 w-8 rounded-full border border-transparent bg-muted/80 text-muted-foreground shadow-sm transition-all duration-200 hover:border-primary/20 hover:bg-muted hover:text-primary hover:shadow'
|
||||||
visible={showTags}
|
>
|
||||||
onSelect={handleEnvVarSelect}
|
<Wand2 className='h-4 w-4' />
|
||||||
blockId={blockId}
|
</Button>
|
||||||
activeSourceBlockId={activeSourceBlockId}
|
</div>
|
||||||
inputValue={value?.toString() ?? ''}
|
)}
|
||||||
cursorPosition={cursorPosition}
|
|
||||||
onClose={() => {
|
{!wandHook?.isStreaming && (
|
||||||
setShowTags(false)
|
<>
|
||||||
setActiveSourceBlockId(null)
|
<EnvVarDropdown
|
||||||
}}
|
visible={showEnvVars}
|
||||||
/>
|
onSelect={handleEnvVarSelect}
|
||||||
</div>
|
searchTerm={searchTerm}
|
||||||
|
inputValue={value?.toString() ?? ''}
|
||||||
|
cursorPosition={cursorPosition}
|
||||||
|
onClose={() => {
|
||||||
|
setShowEnvVars(false)
|
||||||
|
setSearchTerm('')
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
<TagDropdown
|
||||||
|
visible={showTags}
|
||||||
|
onSelect={handleEnvVarSelect}
|
||||||
|
blockId={blockId}
|
||||||
|
activeSourceBlockId={activeSourceBlockId}
|
||||||
|
inputValue={value?.toString() ?? ''}
|
||||||
|
cursorPosition={cursorPosition}
|
||||||
|
onClose={() => {
|
||||||
|
setShowTags(false)
|
||||||
|
setActiveSourceBlockId(null)
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</>
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -344,7 +344,13 @@ export function TriggerConfig({
|
|||||||
|
|
||||||
// Check if the trigger is connected
|
// Check if the trigger is connected
|
||||||
// Both webhook and credential-based triggers now have webhook database entries
|
// Both webhook and credential-based triggers now have webhook database entries
|
||||||
const isTriggerConnected = Boolean(triggerId && actualTriggerId)
|
// In preview, consider it configured if the snapshot contains any trigger fields
|
||||||
|
const isConfiguredInPreview = isPreview && Boolean(
|
||||||
|
(propValue?.triggerPath && propValue.triggerPath.length > 0) ||
|
||||||
|
(propValue?.triggerConfig && Object.keys(propValue.triggerConfig).length > 0) ||
|
||||||
|
propValue?.triggerId
|
||||||
|
)
|
||||||
|
const isTriggerConnected = isConfiguredInPreview || Boolean(triggerId && actualTriggerId)
|
||||||
|
|
||||||
// Debug logging to help with troubleshooting
|
// Debug logging to help with troubleshooting
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
|
|||||||
@@ -486,10 +486,15 @@ export function SubBlock({
|
|||||||
</TooltipContent>
|
</TooltipContent>
|
||||||
</Tooltip>
|
</Tooltip>
|
||||||
)}
|
)}
|
||||||
{config.id === 'responseFormat' && !isValidJson && (
|
{config.id === 'responseFormat' && (
|
||||||
<Tooltip>
|
<Tooltip>
|
||||||
<TooltipTrigger asChild>
|
<TooltipTrigger asChild>
|
||||||
<AlertTriangle className='h-4 w-4 cursor-pointer text-destructive' />
|
<AlertTriangle
|
||||||
|
className={cn(
|
||||||
|
'h-4 w-4 cursor-pointer text-destructive',
|
||||||
|
!isValidJson ? 'opacity-100' : 'opacity-0'
|
||||||
|
)}
|
||||||
|
/>
|
||||||
</TooltipTrigger>
|
</TooltipTrigger>
|
||||||
<TooltipContent side='top'>
|
<TooltipContent side='top'>
|
||||||
<p>Invalid JSON</p>
|
<p>Invalid JSON</p>
|
||||||
|
|||||||
@@ -13,6 +13,8 @@ import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/provide
|
|||||||
import type { BlockConfig, SubBlockConfig, SubBlockType } from '@/blocks/types'
|
import type { BlockConfig, SubBlockConfig, SubBlockType } from '@/blocks/types'
|
||||||
import { useCollaborativeWorkflow } from '@/hooks/use-collaborative-workflow'
|
import { useCollaborativeWorkflow } from '@/hooks/use-collaborative-workflow'
|
||||||
import { useExecutionStore } from '@/stores/execution/store'
|
import { useExecutionStore } from '@/stores/execution/store'
|
||||||
|
import { usePanelStore } from '@/stores/panel/store'
|
||||||
|
import { useGeneralStore } from '@/stores/settings/general/store'
|
||||||
import { useWorkflowDiffStore } from '@/stores/workflow-diff'
|
import { useWorkflowDiffStore } from '@/stores/workflow-diff'
|
||||||
import { useWorkflowRegistry } from '@/stores/workflows/registry/store'
|
import { useWorkflowRegistry } from '@/stores/workflows/registry/store'
|
||||||
import { useSubBlockStore } from '@/stores/workflows/subblock/store'
|
import { useSubBlockStore } from '@/stores/workflows/subblock/store'
|
||||||
@@ -435,8 +437,12 @@ export function WorkflowBlock({ id, data }: NodeProps<WorkflowBlockProps>) {
|
|||||||
stateToUse = mergedState?.subBlocks || {}
|
stateToUse = mergedState?.subBlocks || {}
|
||||||
}
|
}
|
||||||
|
|
||||||
const isAdvancedMode = useWorkflowStore.getState().blocks[blockId]?.advancedMode ?? false
|
const isAdvancedMode = data.isPreview
|
||||||
const isTriggerMode = useWorkflowStore.getState().blocks[blockId]?.triggerMode ?? false
|
? ((data.blockState as any)?.advancedMode ?? false)
|
||||||
|
: useWorkflowStore.getState().blocks[blockId]?.advancedMode ?? false
|
||||||
|
const isTriggerMode = data.isPreview
|
||||||
|
? ((data.blockState as any)?.triggerMode ?? false)
|
||||||
|
: useWorkflowStore.getState().blocks[blockId]?.triggerMode ?? false
|
||||||
const effectiveAdvanced = currentWorkflow.isDiffMode ? displayAdvancedMode : isAdvancedMode
|
const effectiveAdvanced = currentWorkflow.isDiffMode ? displayAdvancedMode : isAdvancedMode
|
||||||
const effectiveTrigger = currentWorkflow.isDiffMode ? displayTriggerMode : isTriggerMode
|
const effectiveTrigger = currentWorkflow.isDiffMode ? displayTriggerMode : isTriggerMode
|
||||||
|
|
||||||
@@ -580,6 +586,72 @@ export function WorkflowBlock({ id, data }: NodeProps<WorkflowBlockProps>) {
|
|||||||
type === 'schedule' && !isLoadingScheduleInfo && scheduleInfo !== null
|
type === 'schedule' && !isLoadingScheduleInfo && scheduleInfo !== null
|
||||||
const userPermissions = useUserPermissionsContext()
|
const userPermissions = useUserPermissionsContext()
|
||||||
|
|
||||||
|
// Debug mode and active selection
|
||||||
|
const isDebugModeEnabled = useGeneralStore((s) => s.isDebugModeEnabled)
|
||||||
|
const activeBlockIds = useExecutionStore((s) => s.activeBlockIds)
|
||||||
|
const panelFocusedBlockId = useExecutionStore((s) => s.panelFocusedBlockId)
|
||||||
|
const setPanelFocusedBlockId = useExecutionStore((s) => s.setPanelFocusedBlockId)
|
||||||
|
const executingBlockIds = useExecutionStore((s) => s.executingBlockIds)
|
||||||
|
const setActiveBlocks = useExecutionStore((s) => s.setActiveBlocks)
|
||||||
|
const setActiveTab = usePanelStore((s) => s.setActiveTab)
|
||||||
|
const breakpointId = useExecutionStore((s) => s.breakpointId)
|
||||||
|
const debugContext = useExecutionStore((s) => s.debugContext)
|
||||||
|
const startPositionIds = useExecutionStore((s) => s.startPositionIds)
|
||||||
|
|
||||||
|
const handleDebugOpen = (e: React.MouseEvent) => {
|
||||||
|
if (!isDebugModeEnabled) return
|
||||||
|
e.stopPropagation()
|
||||||
|
setActiveBlocks(new Set([id]))
|
||||||
|
setActiveTab('debug')
|
||||||
|
// Always select this block for the debug panel focus
|
||||||
|
setPanelFocusedBlockId(id)
|
||||||
|
}
|
||||||
|
|
||||||
|
// In debug mode, use executingBlockIds to detect actual executing blocks (not selection);
|
||||||
|
// outside debug, fall back to activeBlockIds driven by the executor
|
||||||
|
const isExecutingNow = isDebugModeEnabled ? executingBlockIds.has(id) : activeBlockIds.has(id)
|
||||||
|
const isCurrentBlock = isDebugModeEnabled && isPending
|
||||||
|
const isPanelFocused = isDebugModeEnabled && panelFocusedBlockId === id
|
||||||
|
|
||||||
|
// Check if block has errored during debug execution
|
||||||
|
const hasError =
|
||||||
|
isDebugModeEnabled && debugContext
|
||||||
|
? (() => {
|
||||||
|
// Check direct block state for error
|
||||||
|
const directState = debugContext.blockStates?.get(id)
|
||||||
|
if (
|
||||||
|
directState?.output &&
|
||||||
|
typeof directState.output === 'object' &&
|
||||||
|
'error' in directState.output
|
||||||
|
) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
// Check virtual executions for errors (for blocks inside parallels)
|
||||||
|
for (const [key, state] of debugContext.blockStates?.entries() || []) {
|
||||||
|
// Check if this is a virtual ID for our block
|
||||||
|
if (typeof key === 'string' && key.startsWith(`${id}_parallel_`)) {
|
||||||
|
if (state?.output && typeof state.output === 'object' && 'error' in state.output) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Also check block logs for this block
|
||||||
|
const hasErrorLog = debugContext.blockLogs?.some((log: any) => {
|
||||||
|
if (log.blockId === id && !log.success) return true
|
||||||
|
// Check if log is for a virtual version of this block
|
||||||
|
if (
|
||||||
|
typeof log.blockId === 'string' &&
|
||||||
|
log.blockId.startsWith(`${id}_parallel_`) &&
|
||||||
|
!log.success
|
||||||
|
) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
})
|
||||||
|
return hasErrorLog || false
|
||||||
|
})()
|
||||||
|
: false
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className='group relative'>
|
<div className='group relative'>
|
||||||
<Card
|
<Card
|
||||||
@@ -589,20 +661,54 @@ export function WorkflowBlock({ id, data }: NodeProps<WorkflowBlockProps>) {
|
|||||||
'transition-block-bg transition-ring',
|
'transition-block-bg transition-ring',
|
||||||
displayIsWide ? 'w-[480px]' : 'w-[320px]',
|
displayIsWide ? 'w-[480px]' : 'w-[320px]',
|
||||||
!isEnabled && 'shadow-sm',
|
!isEnabled && 'shadow-sm',
|
||||||
isActive && 'animate-pulse-ring ring-2 ring-blue-500',
|
// Error state - highest priority (only border, no background)
|
||||||
isPending && 'ring-2 ring-amber-500',
|
hasError && 'ring-2 ring-red-500',
|
||||||
// Diff highlighting
|
// Panel-focused block highlight (unless errored)
|
||||||
diffStatus === 'new' && 'bg-green-50/50 ring-2 ring-green-500 dark:bg-green-900/10',
|
!hasError && isPanelFocused && 'bg-blue-50/60 dark:bg-blue-900/5',
|
||||||
diffStatus === 'edited' && 'bg-orange-50/50 ring-2 ring-orange-500 dark:bg-orange-900/10',
|
// Executing blocks match staging: pulsing blue ring
|
||||||
|
!hasError && isExecutingNow && 'animate-pulse-ring ring-2 ring-blue-500',
|
||||||
|
// Pending blocks show blue border when not executing
|
||||||
|
!hasError && !isExecutingNow && isCurrentBlock && 'ring-2 ring-blue-500',
|
||||||
|
// Diff highlighting (only if not in debug error state)
|
||||||
|
!hasError &&
|
||||||
|
diffStatus === 'new' &&
|
||||||
|
'bg-green-50/50 ring-2 ring-green-500 dark:bg-green-900/10',
|
||||||
|
!hasError &&
|
||||||
|
diffStatus === 'edited' &&
|
||||||
|
'bg-orange-50/50 ring-2 ring-orange-500 dark:bg-orange-900/10',
|
||||||
// Deleted block highlighting (in original workflow)
|
// Deleted block highlighting (in original workflow)
|
||||||
isDeletedBlock && 'bg-red-50/50 ring-2 ring-red-500 dark:bg-red-900/10',
|
isDeletedBlock && 'bg-red-50/50 ring-2 ring-red-500 dark:bg-red-900/10',
|
||||||
'z-[20]'
|
'z-[20]'
|
||||||
)}
|
)}
|
||||||
|
onClick={handleDebugOpen}
|
||||||
>
|
>
|
||||||
{/* Show debug indicator for pending blocks */}
|
{/* Show error indicator for errored blocks */}
|
||||||
{isPending && (
|
{hasError && (
|
||||||
<div className='-top-6 -translate-x-1/2 absolute left-1/2 z-10 transform rounded-t-md bg-amber-500 px-2 py-0.5 text-white text-xs'>
|
<div className='-top-6 -translate-x-1/2 absolute left-1/2 z-10 transform rounded-t-md bg-red-500 px-2 py-0.5 text-white text-xs'>
|
||||||
Next Step
|
Error
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Show debug indicator for current blocks in debug mode (pending or executing) - but not if errored */}
|
||||||
|
{!hasError && isDebugModeEnabled && (isPending || executingBlockIds.has(id)) && (
|
||||||
|
<div className='-top-6 -translate-x-1/2 absolute left-1/2 z-10 transform rounded-t-md bg-blue-500 px-2 py-0.5 text-white text-xs'>
|
||||||
|
Current
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Bottom indicators: breakpoint and start position side by side */}
|
||||||
|
{isDebugModeEnabled && (breakpointId === id || startPositionIds.has(id)) && (
|
||||||
|
<div className='-bottom-6 -translate-x-1/2 absolute left-1/2 z-10 flex transform items-end gap-2'>
|
||||||
|
{breakpointId === id && (
|
||||||
|
<div className='rounded-b-md bg-orange-500 px-2 py-0.5 text-white text-xs'>
|
||||||
|
Breakpoint
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{startPositionIds.has(id) && (
|
||||||
|
<div className='rounded-b-md bg-purple-600 px-2 py-0.5 text-white text-xs'>
|
||||||
|
Start Position
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
|||||||
@@ -4,36 +4,38 @@ import { useWorkflowDiffStore } from '@/stores/workflow-diff/store'
|
|||||||
import type { DeploymentStatus } from '@/stores/workflows/registry/types'
|
import type { DeploymentStatus } from '@/stores/workflows/registry/types'
|
||||||
import { useWorkflowStore } from '@/stores/workflows/workflow/store'
|
import { useWorkflowStore } from '@/stores/workflows/workflow/store'
|
||||||
import type { BlockState, Loop, Parallel, WorkflowState } from '@/stores/workflows/workflow/types'
|
import type { BlockState, Loop, Parallel, WorkflowState } from '@/stores/workflows/workflow/types'
|
||||||
|
import { useDebugCanvasStore } from '@/stores/execution/debug-canvas/store'
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Interface for the current workflow abstraction
|
* Interface for the current workflow abstraction
|
||||||
*/
|
*/
|
||||||
export interface CurrentWorkflow {
|
export interface CurrentWorkflow {
|
||||||
// Current workflow state properties
|
// Current workflow state properties
|
||||||
blocks: Record<string, BlockState>
|
blocks: Record<string, BlockState>
|
||||||
edges: Edge[]
|
edges: Edge[]
|
||||||
loops: Record<string, Loop>
|
loops: Record<string, Loop>
|
||||||
parallels: Record<string, Parallel>
|
parallels: Record<string, Parallel>
|
||||||
lastSaved?: number
|
lastSaved?: number
|
||||||
isDeployed?: boolean
|
isDeployed?: boolean
|
||||||
deployedAt?: Date
|
deployedAt?: Date
|
||||||
deploymentStatuses?: Record<string, DeploymentStatus>
|
deploymentStatuses?: Record<string, DeploymentStatus>
|
||||||
needsRedeployment?: boolean
|
needsRedeployment?: boolean
|
||||||
hasActiveWebhook?: boolean
|
hasActiveWebhook?: boolean
|
||||||
|
|
||||||
// Mode information
|
// Mode information
|
||||||
isDiffMode: boolean
|
isDiffMode: boolean
|
||||||
isNormalMode: boolean
|
isNormalMode: boolean
|
||||||
|
isDebugCanvasMode?: boolean
|
||||||
|
|
||||||
// Full workflow state (for cases that need the complete object)
|
// Full workflow state (for cases that need the complete object)
|
||||||
workflowState: WorkflowState
|
workflowState: WorkflowState
|
||||||
|
|
||||||
// Helper methods
|
// Helper methods
|
||||||
getBlockById: (blockId: string) => BlockState | undefined
|
getBlockById: (blockId: string) => BlockState | undefined
|
||||||
getBlockCount: () => number
|
getBlockCount: () => number
|
||||||
getEdgeCount: () => number
|
getEdgeCount: () => number
|
||||||
hasBlocks: () => boolean
|
hasBlocks: () => boolean
|
||||||
hasEdges: () => boolean
|
hasEdges: () => boolean
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -41,48 +43,91 @@ export interface CurrentWorkflow {
|
|||||||
* Automatically handles diff vs normal mode without exposing the complexity to consumers.
|
* Automatically handles diff vs normal mode without exposing the complexity to consumers.
|
||||||
*/
|
*/
|
||||||
export function useCurrentWorkflow(): CurrentWorkflow {
|
export function useCurrentWorkflow(): CurrentWorkflow {
|
||||||
// Get normal workflow state
|
// Get normal workflow state
|
||||||
const normalWorkflow = useWorkflowStore((state) => state.getWorkflowState())
|
const normalWorkflow = useWorkflowStore((state) => state.getWorkflowState())
|
||||||
|
|
||||||
// Get diff state - now including isDiffReady
|
// Get diff state - now including isDiffReady
|
||||||
const { isShowingDiff, isDiffReady, diffWorkflow } = useWorkflowDiffStore()
|
const { isShowingDiff, isDiffReady, diffWorkflow } = useWorkflowDiffStore()
|
||||||
|
|
||||||
// Create the abstracted interface
|
// Get debug canvas override
|
||||||
const currentWorkflow = useMemo((): CurrentWorkflow => {
|
const debugCanvas = useDebugCanvasStore((s) => ({ isActive: s.isActive, workflowState: s.workflowState }))
|
||||||
// Determine which workflow to use - only use diff if it's ready
|
|
||||||
const hasDiffBlocks =
|
|
||||||
!!diffWorkflow && Object.keys((diffWorkflow as any).blocks || {}).length > 0
|
|
||||||
const shouldUseDiff = isShowingDiff && isDiffReady && hasDiffBlocks
|
|
||||||
const activeWorkflow = shouldUseDiff ? diffWorkflow : normalWorkflow
|
|
||||||
|
|
||||||
return {
|
// Create the abstracted interface
|
||||||
// Current workflow state
|
const currentWorkflow = useMemo((): CurrentWorkflow => {
|
||||||
blocks: activeWorkflow.blocks,
|
// Prefer debug canvas if active
|
||||||
edges: activeWorkflow.edges,
|
const hasDebugCanvas = !!debugCanvas.isActive && !!debugCanvas.workflowState
|
||||||
loops: activeWorkflow.loops || {},
|
if (hasDebugCanvas) {
|
||||||
parallels: activeWorkflow.parallels || {},
|
console.log('[useCurrentWorkflow] Using debug canvas state', {
|
||||||
lastSaved: activeWorkflow.lastSaved,
|
isActive: debugCanvas.isActive,
|
||||||
isDeployed: activeWorkflow.isDeployed,
|
hasWorkflowState: !!debugCanvas.workflowState,
|
||||||
deployedAt: activeWorkflow.deployedAt,
|
blockCount: debugCanvas.workflowState ? Object.keys(debugCanvas.workflowState.blocks || {}).length : 0,
|
||||||
deploymentStatuses: activeWorkflow.deploymentStatuses,
|
edgeCount: debugCanvas.workflowState ? (debugCanvas.workflowState.edges || []).length : 0
|
||||||
needsRedeployment: activeWorkflow.needsRedeployment,
|
})
|
||||||
hasActiveWebhook: activeWorkflow.hasActiveWebhook,
|
const activeWorkflow = debugCanvas.workflowState as WorkflowState
|
||||||
|
return {
|
||||||
|
blocks: activeWorkflow.blocks,
|
||||||
|
edges: activeWorkflow.edges,
|
||||||
|
loops: activeWorkflow.loops || {},
|
||||||
|
parallels: activeWorkflow.parallels || {},
|
||||||
|
lastSaved: activeWorkflow.lastSaved,
|
||||||
|
isDeployed: activeWorkflow.isDeployed,
|
||||||
|
deployedAt: activeWorkflow.deployedAt,
|
||||||
|
deploymentStatuses: activeWorkflow.deploymentStatuses,
|
||||||
|
needsRedeployment: activeWorkflow.needsRedeployment,
|
||||||
|
hasActiveWebhook: activeWorkflow.hasActiveWebhook,
|
||||||
|
isDiffMode: false,
|
||||||
|
isNormalMode: false,
|
||||||
|
isDebugCanvasMode: true,
|
||||||
|
workflowState: activeWorkflow,
|
||||||
|
getBlockById: (blockId: string) => activeWorkflow.blocks[blockId],
|
||||||
|
getBlockCount: () => Object.keys(activeWorkflow.blocks).length,
|
||||||
|
getEdgeCount: () => activeWorkflow.edges.length,
|
||||||
|
hasBlocks: () => Object.keys(activeWorkflow.blocks).length > 0,
|
||||||
|
hasEdges: () => activeWorkflow.edges.length > 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Mode information - update to reflect ready state
|
// Determine which workflow to use - only use diff if it's ready
|
||||||
isDiffMode: shouldUseDiff,
|
const hasDiffBlocks = !!diffWorkflow && Object.keys((diffWorkflow as any).blocks || {}).length > 0
|
||||||
isNormalMode: !shouldUseDiff,
|
const shouldUseDiff = isShowingDiff && isDiffReady && hasDiffBlocks
|
||||||
|
const activeWorkflow = shouldUseDiff ? diffWorkflow : normalWorkflow
|
||||||
|
|
||||||
// Full workflow state (for cases that need the complete object)
|
console.log('[useCurrentWorkflow] Not using debug canvas', {
|
||||||
workflowState: activeWorkflow,
|
debugCanvasIsActive: debugCanvas.isActive,
|
||||||
|
debugCanvasHasState: !!debugCanvas.workflowState,
|
||||||
|
usingDiff: shouldUseDiff,
|
||||||
|
normalBlockCount: Object.keys(normalWorkflow.blocks || {}).length
|
||||||
|
})
|
||||||
|
|
||||||
// Helper methods
|
return {
|
||||||
getBlockById: (blockId: string) => activeWorkflow.blocks[blockId],
|
// Current workflow state
|
||||||
getBlockCount: () => Object.keys(activeWorkflow.blocks).length,
|
blocks: activeWorkflow.blocks,
|
||||||
getEdgeCount: () => activeWorkflow.edges.length,
|
edges: activeWorkflow.edges,
|
||||||
hasBlocks: () => Object.keys(activeWorkflow.blocks).length > 0,
|
loops: activeWorkflow.loops || {},
|
||||||
hasEdges: () => activeWorkflow.edges.length > 0,
|
parallels: activeWorkflow.parallels || {},
|
||||||
}
|
lastSaved: activeWorkflow.lastSaved,
|
||||||
}, [normalWorkflow, isShowingDiff, isDiffReady, diffWorkflow])
|
isDeployed: activeWorkflow.isDeployed,
|
||||||
|
deployedAt: activeWorkflow.deployedAt,
|
||||||
|
deploymentStatuses: activeWorkflow.deploymentStatuses,
|
||||||
|
needsRedeployment: activeWorkflow.needsRedeployment,
|
||||||
|
hasActiveWebhook: activeWorkflow.hasActiveWebhook,
|
||||||
|
|
||||||
return currentWorkflow
|
// Mode information - update to reflect ready state
|
||||||
|
isDiffMode: shouldUseDiff,
|
||||||
|
isNormalMode: !shouldUseDiff,
|
||||||
|
isDebugCanvasMode: false,
|
||||||
|
|
||||||
|
// Full workflow state (for cases that need the complete object)
|
||||||
|
workflowState: activeWorkflow,
|
||||||
|
|
||||||
|
// Helper methods
|
||||||
|
getBlockById: (blockId: string) => activeWorkflow.blocks[blockId],
|
||||||
|
getBlockCount: () => Object.keys(activeWorkflow.blocks).length,
|
||||||
|
getEdgeCount: () => activeWorkflow.edges.length,
|
||||||
|
hasBlocks: () => Object.keys(activeWorkflow.blocks).length > 0,
|
||||||
|
hasEdges: () => activeWorkflow.edges.length > 0,
|
||||||
|
}
|
||||||
|
}, [normalWorkflow, isShowingDiff, isDiffReady, diffWorkflow, debugCanvas.isActive, debugCanvas.workflowState])
|
||||||
|
|
||||||
|
return currentWorkflow
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import { Executor } from '@/executor'
|
|||||||
import type { BlockLog, ExecutionResult, StreamingExecution } from '@/executor/types'
|
import type { BlockLog, ExecutionResult, StreamingExecution } from '@/executor/types'
|
||||||
import { Serializer } from '@/serializer'
|
import { Serializer } from '@/serializer'
|
||||||
import type { SerializedWorkflow } from '@/serializer/types'
|
import type { SerializedWorkflow } from '@/serializer/types'
|
||||||
|
import { useDebugSnapshotStore } from '@/stores/execution/debug-snapshots/store'
|
||||||
import { useExecutionStore } from '@/stores/execution/store'
|
import { useExecutionStore } from '@/stores/execution/store'
|
||||||
import { useConsoleStore } from '@/stores/panel/console/store'
|
import { useConsoleStore } from '@/stores/panel/console/store'
|
||||||
import { useVariablesStore } from '@/stores/panel/variables/store'
|
import { useVariablesStore } from '@/stores/panel/variables/store'
|
||||||
@@ -62,6 +63,8 @@ export function useWorkflowExecution() {
|
|||||||
setExecutor,
|
setExecutor,
|
||||||
setDebugContext,
|
setDebugContext,
|
||||||
setActiveBlocks,
|
setActiveBlocks,
|
||||||
|
setExecutingBlockIds,
|
||||||
|
startPositionIds,
|
||||||
} = useExecutionStore()
|
} = useExecutionStore()
|
||||||
const [executionResult, setExecutionResult] = useState<ExecutionResult | null>(null)
|
const [executionResult, setExecutionResult] = useState<ExecutionResult | null>(null)
|
||||||
|
|
||||||
@@ -70,7 +73,7 @@ export function useWorkflowExecution() {
|
|||||||
*/
|
*/
|
||||||
const validateDebugState = useCallback((): DebugValidationResult => {
|
const validateDebugState = useCallback((): DebugValidationResult => {
|
||||||
if (!executor || !debugContext || pendingBlocks.length === 0) {
|
if (!executor || !debugContext || pendingBlocks.length === 0) {
|
||||||
const missing = []
|
const missing = [] as string[]
|
||||||
if (!executor) missing.push('executor')
|
if (!executor) missing.push('executor')
|
||||||
if (!debugContext) missing.push('debugContext')
|
if (!debugContext) missing.push('debugContext')
|
||||||
if (pendingBlocks.length === 0) missing.push('pendingBlocks')
|
if (pendingBlocks.length === 0) missing.push('pendingBlocks')
|
||||||
@@ -93,6 +96,7 @@ export function useWorkflowExecution() {
|
|||||||
setExecutor(null)
|
setExecutor(null)
|
||||||
setPendingBlocks([])
|
setPendingBlocks([])
|
||||||
setActiveBlocks(new Set())
|
setActiveBlocks(new Set())
|
||||||
|
setExecutingBlockIds(new Set())
|
||||||
|
|
||||||
// Reset debug mode setting if it was enabled
|
// Reset debug mode setting if it was enabled
|
||||||
if (isDebugModeEnabled) {
|
if (isDebugModeEnabled) {
|
||||||
@@ -105,6 +109,7 @@ export function useWorkflowExecution() {
|
|||||||
setExecutor,
|
setExecutor,
|
||||||
setPendingBlocks,
|
setPendingBlocks,
|
||||||
setActiveBlocks,
|
setActiveBlocks,
|
||||||
|
setExecutingBlockIds,
|
||||||
isDebugModeEnabled,
|
isDebugModeEnabled,
|
||||||
])
|
])
|
||||||
|
|
||||||
@@ -120,7 +125,7 @@ export function useWorkflowExecution() {
|
|||||||
}, [])
|
}, [])
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Handles debug session completion
|
* Handles debug session completion - keep debug session open for inspection
|
||||||
*/
|
*/
|
||||||
const handleDebugSessionComplete = useCallback(
|
const handleDebugSessionComplete = useCallback(
|
||||||
async (result: ExecutionResult) => {
|
async (result: ExecutionResult) => {
|
||||||
@@ -130,10 +135,14 @@ export function useWorkflowExecution() {
|
|||||||
// Persist logs
|
// Persist logs
|
||||||
await persistLogs(uuidv4(), result)
|
await persistLogs(uuidv4(), result)
|
||||||
|
|
||||||
// Reset debug state
|
// Keep debug mode open for inspection: stop executing, clear pending
|
||||||
resetDebugState()
|
setIsExecuting(false)
|
||||||
|
setPendingBlocks([])
|
||||||
|
setExecutingBlockIds(new Set())
|
||||||
|
// Keep debugContext and executor so the panel can inspect state
|
||||||
|
// Do not reset isDebugging
|
||||||
},
|
},
|
||||||
[activeWorkflowId, resetDebugState]
|
[activeWorkflowId, setIsExecuting, setPendingBlocks, setExecutingBlockIds]
|
||||||
)
|
)
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -148,16 +157,30 @@ export function useWorkflowExecution() {
|
|||||||
// Update debug context and pending blocks
|
// Update debug context and pending blocks
|
||||||
if (result.metadata?.context) {
|
if (result.metadata?.context) {
|
||||||
setDebugContext(result.metadata.context)
|
setDebugContext(result.metadata.context)
|
||||||
|
// Capture snapshot for revert/backstep
|
||||||
|
try {
|
||||||
|
useDebugSnapshotStore.getState().captureFromContext(result.metadata.context as any)
|
||||||
|
useDebugSnapshotStore
|
||||||
|
.getState()
|
||||||
|
.pushFromContext(result.metadata.context as any, result.metadata?.pendingBlocks || [])
|
||||||
|
} catch {}
|
||||||
}
|
}
|
||||||
if (result.metadata?.pendingBlocks) {
|
if (result.metadata?.pendingBlocks) {
|
||||||
setPendingBlocks(result.metadata.pendingBlocks)
|
// Filter triggers from next pending
|
||||||
|
const filtered = (result.metadata.pendingBlocks as string[]).filter((id) => {
|
||||||
|
const block = currentWorkflow.blocks[id]
|
||||||
|
if (!block) return false
|
||||||
|
const cfg = getBlock(block.type)
|
||||||
|
return cfg?.category !== 'triggers'
|
||||||
|
})
|
||||||
|
setPendingBlocks(filtered)
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
[setDebugContext, setPendingBlocks]
|
[setDebugContext, setPendingBlocks, currentWorkflow.blocks]
|
||||||
)
|
)
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Handles debug execution errors
|
* Handles debug execution errors - keep debug open for inspection
|
||||||
*/
|
*/
|
||||||
const handleDebugExecutionError = useCallback(
|
const handleDebugExecutionError = useCallback(
|
||||||
async (error: any, operation: string) => {
|
async (error: any, operation: string) => {
|
||||||
@@ -176,10 +199,13 @@ export function useWorkflowExecution() {
|
|||||||
// Persist logs
|
// Persist logs
|
||||||
await persistLogs(uuidv4(), errorResult)
|
await persistLogs(uuidv4(), errorResult)
|
||||||
|
|
||||||
// Reset debug state
|
// Keep debug session open for inspection
|
||||||
resetDebugState()
|
setIsExecuting(false)
|
||||||
|
setPendingBlocks([])
|
||||||
|
setExecutingBlockIds(new Set())
|
||||||
|
// Keep isDebugging, debugContext, and executor intact
|
||||||
},
|
},
|
||||||
[debugContext, activeWorkflowId, resetDebugState]
|
[debugContext, activeWorkflowId, setIsExecuting, setPendingBlocks, setExecutingBlockIds]
|
||||||
)
|
)
|
||||||
|
|
||||||
const persistLogs = async (
|
const persistLogs = async (
|
||||||
@@ -268,8 +294,8 @@ export function useWorkflowExecution() {
|
|||||||
const isChatExecution =
|
const isChatExecution =
|
||||||
workflowInput && typeof workflowInput === 'object' && 'input' in workflowInput
|
workflowInput && typeof workflowInput === 'object' && 'input' in workflowInput
|
||||||
|
|
||||||
// For chat executions, we'll use a streaming approach
|
// For chat executions, use streaming only when NOT debugging
|
||||||
if (isChatExecution) {
|
if (isChatExecution && !enableDebug) {
|
||||||
const stream = new ReadableStream({
|
const stream = new ReadableStream({
|
||||||
async start(controller) {
|
async start(controller) {
|
||||||
const encoder = new TextEncoder()
|
const encoder = new TextEncoder()
|
||||||
@@ -390,7 +416,7 @@ export function useWorkflowExecution() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const result = await executeWorkflow(workflowInput, onStream, executionId)
|
const result = await executeWorkflow(workflowInput, onStream, executionId, false)
|
||||||
|
|
||||||
// Check if execution was cancelled
|
// Check if execution was cancelled
|
||||||
if (
|
if (
|
||||||
@@ -448,7 +474,6 @@ export function useWorkflowExecution() {
|
|||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
controller.error(error)
|
controller.error(error)
|
||||||
} finally {
|
} finally {
|
||||||
controller.close()
|
|
||||||
setIsExecuting(false)
|
setIsExecuting(false)
|
||||||
setIsDebugging(false)
|
setIsDebugging(false)
|
||||||
setActiveBlocks(new Set())
|
setActiveBlocks(new Set())
|
||||||
@@ -458,12 +483,23 @@ export function useWorkflowExecution() {
|
|||||||
return { success: true, stream }
|
return { success: true, stream }
|
||||||
}
|
}
|
||||||
|
|
||||||
// For manual (non-chat) execution
|
// For manual (non-streaming) execution including debug and non-chat
|
||||||
const executionId = uuidv4()
|
const executionId = uuidv4()
|
||||||
try {
|
try {
|
||||||
const result = await executeWorkflow(workflowInput, undefined, executionId)
|
const result = await executeWorkflow(workflowInput, undefined, executionId, enableDebug)
|
||||||
if (result && 'metadata' in result && result.metadata?.isDebugSession) {
|
if (result && 'metadata' in result && result.metadata?.isDebugSession) {
|
||||||
setDebugContext(result.metadata.context || null)
|
setDebugContext(result.metadata.context || null)
|
||||||
|
try {
|
||||||
|
if (result.metadata?.context) {
|
||||||
|
useDebugSnapshotStore.getState().captureFromContext(result.metadata.context as any)
|
||||||
|
useDebugSnapshotStore
|
||||||
|
.getState()
|
||||||
|
.pushFromContext(
|
||||||
|
result.metadata.context as any,
|
||||||
|
result.metadata?.pendingBlocks || []
|
||||||
|
)
|
||||||
|
}
|
||||||
|
} catch {}
|
||||||
if (result.metadata.pendingBlocks) {
|
if (result.metadata.pendingBlocks) {
|
||||||
setPendingBlocks(result.metadata.pendingBlocks)
|
setPendingBlocks(result.metadata.pendingBlocks)
|
||||||
}
|
}
|
||||||
@@ -508,13 +544,15 @@ export function useWorkflowExecution() {
|
|||||||
setExecutor,
|
setExecutor,
|
||||||
setPendingBlocks,
|
setPendingBlocks,
|
||||||
setActiveBlocks,
|
setActiveBlocks,
|
||||||
|
startPositionIds,
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
const executeWorkflow = async (
|
const executeWorkflow = async (
|
||||||
workflowInput?: any,
|
workflowInput?: any,
|
||||||
onStream?: (se: StreamingExecution) => Promise<void>,
|
onStream?: (se: StreamingExecution) => Promise<void>,
|
||||||
executionId?: string
|
executionId?: string,
|
||||||
|
debugRequested?: boolean
|
||||||
): Promise<ExecutionResult | StreamingExecution> => {
|
): Promise<ExecutionResult | StreamingExecution> => {
|
||||||
// Use currentWorkflow but check if we're in diff mode
|
// Use currentWorkflow but check if we're in diff mode
|
||||||
const {
|
const {
|
||||||
@@ -602,7 +640,7 @@ export function useWorkflowExecution() {
|
|||||||
const envVars = getAllVariables()
|
const envVars = getAllVariables()
|
||||||
const envVarValues = Object.entries(envVars).reduce(
|
const envVarValues = Object.entries(envVars).reduce(
|
||||||
(acc, [key, variable]) => {
|
(acc, [key, variable]) => {
|
||||||
acc[key] = variable.value
|
acc[key] = (variable as any).value
|
||||||
return acc
|
return acc
|
||||||
},
|
},
|
||||||
{} as Record<string, string>
|
{} as Record<string, string>
|
||||||
@@ -672,7 +710,9 @@ export function useWorkflowExecution() {
|
|||||||
setExecutor(newExecutor)
|
setExecutor(newExecutor)
|
||||||
|
|
||||||
// Execute workflow
|
// Execute workflow
|
||||||
return newExecutor.execute(activeWorkflowId || '')
|
const execResult = await newExecutor.execute(activeWorkflowId || '')
|
||||||
|
|
||||||
|
return execResult
|
||||||
}
|
}
|
||||||
|
|
||||||
const handleExecutionError = (error: any) => {
|
const handleExecutionError = (error: any) => {
|
||||||
@@ -748,14 +788,33 @@ export function useWorkflowExecution() {
|
|||||||
// Validate debug state
|
// Validate debug state
|
||||||
const validation = validateDebugState()
|
const validation = validateDebugState()
|
||||||
if (!validation.isValid) {
|
if (!validation.isValid) {
|
||||||
resetDebugState()
|
// Keep session open for inspection; simply stop executing
|
||||||
|
setIsExecuting(false)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compute executable set without triggers
|
||||||
|
const nonTriggerPending = pendingBlocks.filter((id) => {
|
||||||
|
const block = currentWorkflow.blocks[id]
|
||||||
|
if (!block) return false
|
||||||
|
const cfg = getBlock(block.type)
|
||||||
|
return cfg?.category !== 'triggers'
|
||||||
|
})
|
||||||
|
|
||||||
|
if (nonTriggerPending.length === 0) {
|
||||||
|
// Nothing executable
|
||||||
|
setIsExecuting(false)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
logger.info('Executing debug step with blocks:', pendingBlocks)
|
logger.info('Executing debug step with blocks:', nonTriggerPending)
|
||||||
const result = await executor!.continueExecution(pendingBlocks, debugContext!)
|
// Mark current pending blocks as executing for UI pulse
|
||||||
|
setExecutingBlockIds(new Set(nonTriggerPending))
|
||||||
|
const result = await executor!.continueExecution(nonTriggerPending, debugContext!)
|
||||||
logger.info('Debug step execution result:', result)
|
logger.info('Debug step execution result:', result)
|
||||||
|
// Clear executing state after step returns
|
||||||
|
setExecutingBlockIds(new Set())
|
||||||
|
|
||||||
if (isDebugSessionComplete(result)) {
|
if (isDebugSessionComplete(result)) {
|
||||||
await handleDebugSessionComplete(result)
|
await handleDebugSessionComplete(result)
|
||||||
@@ -763,6 +822,7 @@ export function useWorkflowExecution() {
|
|||||||
handleDebugSessionContinuation(result)
|
handleDebugSessionContinuation(result)
|
||||||
}
|
}
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
|
setExecutingBlockIds(new Set())
|
||||||
await handleDebugExecutionError(error, 'step')
|
await handleDebugExecutionError(error, 'step')
|
||||||
}
|
}
|
||||||
}, [
|
}, [
|
||||||
@@ -771,11 +831,13 @@ export function useWorkflowExecution() {
|
|||||||
pendingBlocks,
|
pendingBlocks,
|
||||||
activeWorkflowId,
|
activeWorkflowId,
|
||||||
validateDebugState,
|
validateDebugState,
|
||||||
resetDebugState,
|
setIsExecuting,
|
||||||
|
setExecutingBlockIds,
|
||||||
isDebugSessionComplete,
|
isDebugSessionComplete,
|
||||||
handleDebugSessionComplete,
|
handleDebugSessionComplete,
|
||||||
handleDebugSessionContinuation,
|
handleDebugSessionContinuation,
|
||||||
handleDebugExecutionError,
|
handleDebugExecutionError,
|
||||||
|
currentWorkflow.blocks,
|
||||||
])
|
])
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -791,7 +853,8 @@ export function useWorkflowExecution() {
|
|||||||
// Validate debug state
|
// Validate debug state
|
||||||
const validation = validateDebugState()
|
const validation = validateDebugState()
|
||||||
if (!validation.isValid) {
|
if (!validation.isValid) {
|
||||||
resetDebugState()
|
// Keep session open for inspection; simply stop executing
|
||||||
|
setIsExecuting(false)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -808,6 +871,14 @@ export function useWorkflowExecution() {
|
|||||||
let currentContext = { ...debugContext! }
|
let currentContext = { ...debugContext! }
|
||||||
let currentPendingBlocks = [...pendingBlocks]
|
let currentPendingBlocks = [...pendingBlocks]
|
||||||
|
|
||||||
|
// Filter initial pending
|
||||||
|
currentPendingBlocks = currentPendingBlocks.filter((id) => {
|
||||||
|
const block = currentWorkflow.blocks[id]
|
||||||
|
if (!block) return false
|
||||||
|
const cfg = getBlock(block.type)
|
||||||
|
return cfg?.category !== 'triggers'
|
||||||
|
})
|
||||||
|
|
||||||
logger.info('Starting resume execution with blocks:', currentPendingBlocks)
|
logger.info('Starting resume execution with blocks:', currentPendingBlocks)
|
||||||
|
|
||||||
// Continue execution until there are no more pending blocks
|
// Continue execution until there are no more pending blocks
|
||||||
@@ -819,7 +890,9 @@ export function useWorkflowExecution() {
|
|||||||
`Resume iteration ${iterationCount + 1}, executing ${currentPendingBlocks.length} blocks`
|
`Resume iteration ${iterationCount + 1}, executing ${currentPendingBlocks.length} blocks`
|
||||||
)
|
)
|
||||||
|
|
||||||
|
setExecutingBlockIds(new Set(currentPendingBlocks))
|
||||||
currentResult = await executor!.continueExecution(currentPendingBlocks, currentContext)
|
currentResult = await executor!.continueExecution(currentPendingBlocks, currentContext)
|
||||||
|
setExecutingBlockIds(new Set())
|
||||||
|
|
||||||
logger.info('Resume iteration result:', {
|
logger.info('Resume iteration result:', {
|
||||||
success: currentResult.success,
|
success: currentResult.success,
|
||||||
@@ -835,9 +908,14 @@ export function useWorkflowExecution() {
|
|||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update pending blocks for next iteration
|
// Update pending blocks for next iteration, filtered
|
||||||
if (currentResult.metadata?.pendingBlocks) {
|
if (currentResult.metadata?.pendingBlocks) {
|
||||||
currentPendingBlocks = currentResult.metadata.pendingBlocks
|
currentPendingBlocks = (currentResult.metadata.pendingBlocks as string[]).filter((id) => {
|
||||||
|
const block = currentWorkflow.blocks[id]
|
||||||
|
if (!block) return false
|
||||||
|
const cfg = getBlock(block.type)
|
||||||
|
return cfg?.category !== 'triggers'
|
||||||
|
})
|
||||||
} else {
|
} else {
|
||||||
logger.info('No pending blocks in result, ending resume')
|
logger.info('No pending blocks in result, ending resume')
|
||||||
break
|
break
|
||||||
@@ -864,6 +942,7 @@ export function useWorkflowExecution() {
|
|||||||
// Handle completion
|
// Handle completion
|
||||||
await handleDebugSessionComplete(currentResult)
|
await handleDebugSessionComplete(currentResult)
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
|
setExecutingBlockIds(new Set())
|
||||||
await handleDebugExecutionError(error, 'resume')
|
await handleDebugExecutionError(error, 'resume')
|
||||||
}
|
}
|
||||||
}, [
|
}, [
|
||||||
@@ -872,9 +951,11 @@ export function useWorkflowExecution() {
|
|||||||
pendingBlocks,
|
pendingBlocks,
|
||||||
activeWorkflowId,
|
activeWorkflowId,
|
||||||
validateDebugState,
|
validateDebugState,
|
||||||
resetDebugState,
|
setIsExecuting,
|
||||||
|
setExecutingBlockIds,
|
||||||
handleDebugSessionComplete,
|
handleDebugSessionComplete,
|
||||||
handleDebugExecutionError,
|
handleDebugExecutionError,
|
||||||
|
currentWorkflow.blocks,
|
||||||
])
|
])
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -131,6 +131,17 @@ export async function executeWorkflowWithLogging(
|
|||||||
// Merge subblock states from the appropriate store
|
// Merge subblock states from the appropriate store
|
||||||
const mergedStates = mergeSubblockState(validBlocks)
|
const mergedStates = mergeSubblockState(validBlocks)
|
||||||
|
|
||||||
|
// Log the current workflow state before filtering
|
||||||
|
logger.info('🔍 Current workflow state before filtering:', {
|
||||||
|
totalBlocks: Object.keys(mergedStates).length,
|
||||||
|
blocks: Object.entries(mergedStates).map(([id, block]) => ({
|
||||||
|
id,
|
||||||
|
type: block.type,
|
||||||
|
triggerMode: block.triggerMode,
|
||||||
|
category: block.type ? getBlock(block.type)?.category : undefined,
|
||||||
|
})),
|
||||||
|
})
|
||||||
|
|
||||||
// Filter out trigger blocks for manual execution
|
// Filter out trigger blocks for manual execution
|
||||||
const filteredStates = Object.entries(mergedStates).reduce(
|
const filteredStates = Object.entries(mergedStates).reduce(
|
||||||
(acc, [id, block]) => {
|
(acc, [id, block]) => {
|
||||||
@@ -142,16 +153,29 @@ export async function executeWorkflowWithLogging(
|
|||||||
|
|
||||||
const blockConfig = getBlock(block.type)
|
const blockConfig = getBlock(block.type)
|
||||||
const isTriggerBlock = blockConfig?.category === 'triggers'
|
const isTriggerBlock = blockConfig?.category === 'triggers'
|
||||||
|
const isInTriggerMode = block.triggerMode === true
|
||||||
|
|
||||||
// Skip trigger blocks during manual execution
|
// Skip trigger blocks AND blocks in trigger mode during manual execution
|
||||||
if (!isTriggerBlock) {
|
if (!isTriggerBlock && !isInTriggerMode) {
|
||||||
acc[id] = block
|
acc[id] = block
|
||||||
|
} else {
|
||||||
|
logger.info(`🚫 Filtering out block ${id} - trigger category: ${isTriggerBlock}, trigger mode: ${isInTriggerMode}`)
|
||||||
}
|
}
|
||||||
return acc
|
return acc
|
||||||
},
|
},
|
||||||
{} as typeof mergedStates
|
{} as typeof mergedStates
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Log the filtered state that will be used for execution (not snapshots)
|
||||||
|
logger.info('📦 Filtered workflow state for execution:', {
|
||||||
|
totalBlocks: Object.keys(filteredStates).length,
|
||||||
|
blocks: Object.entries(filteredStates).map(([id, block]) => ({
|
||||||
|
id,
|
||||||
|
type: block.type,
|
||||||
|
triggerMode: block.triggerMode,
|
||||||
|
})),
|
||||||
|
})
|
||||||
|
|
||||||
const currentBlockStates = Object.entries(filteredStates).reduce(
|
const currentBlockStates = Object.entries(filteredStates).reduce(
|
||||||
(acc, [id, block]) => {
|
(acc, [id, block]) => {
|
||||||
acc[id] = Object.entries(block.subBlocks).reduce(
|
acc[id] = Object.entries(block.subBlocks).reduce(
|
||||||
|
|||||||
@@ -1635,6 +1635,8 @@ const WorkflowContent = React.memo(() => {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const isReadOnly = currentWorkflow.isDebugCanvasMode === true ? true : !effectivePermissions.canEdit
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className='flex h-screen w-full flex-col overflow-hidden'>
|
<div className='flex h-screen w-full flex-col overflow-hidden'>
|
||||||
<div className='relative h-full w-full flex-1 transition-all duration-200'>
|
<div className='relative h-full w-full flex-1 transition-all duration-200'>
|
||||||
@@ -1650,11 +1652,11 @@ const WorkflowContent = React.memo(() => {
|
|||||||
edges={edgesWithSelection}
|
edges={edgesWithSelection}
|
||||||
onNodesChange={onNodesChange}
|
onNodesChange={onNodesChange}
|
||||||
onEdgesChange={onEdgesChange}
|
onEdgesChange={onEdgesChange}
|
||||||
onConnect={effectivePermissions.canEdit ? onConnect : undefined}
|
onConnect={isReadOnly ? undefined : onConnect}
|
||||||
nodeTypes={nodeTypes}
|
nodeTypes={nodeTypes}
|
||||||
edgeTypes={edgeTypes}
|
edgeTypes={edgeTypes}
|
||||||
onDrop={effectivePermissions.canEdit ? onDrop : undefined}
|
onDrop={isReadOnly ? undefined : onDrop}
|
||||||
onDragOver={effectivePermissions.canEdit ? onDragOver : undefined}
|
onDragOver={isReadOnly ? undefined : onDragOver}
|
||||||
fitView
|
fitView
|
||||||
minZoom={0.1}
|
minZoom={0.1}
|
||||||
maxZoom={1.3}
|
maxZoom={1.3}
|
||||||
@@ -1674,22 +1676,22 @@ const WorkflowContent = React.memo(() => {
|
|||||||
onEdgeClick={onEdgeClick}
|
onEdgeClick={onEdgeClick}
|
||||||
elementsSelectable={true}
|
elementsSelectable={true}
|
||||||
selectNodesOnDrag={false}
|
selectNodesOnDrag={false}
|
||||||
nodesConnectable={effectivePermissions.canEdit}
|
nodesConnectable={!isReadOnly}
|
||||||
nodesDraggable={effectivePermissions.canEdit}
|
nodesDraggable={!isReadOnly}
|
||||||
draggable={false}
|
draggable={false}
|
||||||
noWheelClassName='allow-scroll'
|
noWheelClassName='allow-scroll'
|
||||||
edgesFocusable={true}
|
edgesFocusable={true}
|
||||||
edgesUpdatable={effectivePermissions.canEdit}
|
edgesUpdatable={!isReadOnly}
|
||||||
className='workflow-container h-full'
|
className='workflow-container h-full'
|
||||||
onNodeDrag={effectivePermissions.canEdit ? onNodeDrag : undefined}
|
onNodeDrag={isReadOnly ? undefined : onNodeDrag}
|
||||||
onNodeDragStop={effectivePermissions.canEdit ? onNodeDragStop : undefined}
|
onNodeDragStop={isReadOnly ? undefined : onNodeDragStop}
|
||||||
onNodeDragStart={effectivePermissions.canEdit ? onNodeDragStart : undefined}
|
onNodeDragStart={isReadOnly ? undefined : onNodeDragStart}
|
||||||
snapToGrid={false}
|
snapToGrid={false}
|
||||||
snapGrid={[20, 20]}
|
snapGrid={[20, 20]}
|
||||||
elevateEdgesOnSelect={true}
|
elevateEdgesOnSelect={true}
|
||||||
elevateNodesOnSelect={true}
|
elevateNodesOnSelect={true}
|
||||||
autoPanOnConnect={effectivePermissions.canEdit}
|
autoPanOnConnect={!isReadOnly}
|
||||||
autoPanOnNodeDrag={effectivePermissions.canEdit}
|
autoPanOnNodeDrag={!isReadOnly}
|
||||||
>
|
>
|
||||||
<Background
|
<Background
|
||||||
color='hsl(var(--workflow-dots))'
|
color='hsl(var(--workflow-dots))'
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ import {
|
|||||||
AlertDialogTitle,
|
AlertDialogTitle,
|
||||||
} from '@/components/ui/alert-dialog'
|
} from '@/components/ui/alert-dialog'
|
||||||
import { ScrollArea } from '@/components/ui/scroll-area'
|
import { ScrollArea } from '@/components/ui/scroll-area'
|
||||||
import { MAX_TAG_SLOTS } from '@/lib/constants/knowledge'
|
import { MAX_TAG_SLOTS } from '@/lib/knowledge/consts'
|
||||||
import { createLogger } from '@/lib/logs/console/logger'
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
import { getDocumentIcon } from '@/app/workspace/[workspaceId]/knowledge/components/icons/document-icons'
|
import { getDocumentIcon } from '@/app/workspace/[workspaceId]/knowledge/components/icons/document-icons'
|
||||||
import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider'
|
import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider'
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ import {
|
|||||||
SelectValue,
|
SelectValue,
|
||||||
} from '@/components/ui'
|
} from '@/components/ui'
|
||||||
import { ScrollArea } from '@/components/ui/scroll-area'
|
import { ScrollArea } from '@/components/ui/scroll-area'
|
||||||
import { MAX_TAG_SLOTS, TAG_SLOTS, type TagSlot } from '@/lib/constants/knowledge'
|
import { MAX_TAG_SLOTS, TAG_SLOTS, type TagSlot } from '@/lib/knowledge/consts'
|
||||||
import { createLogger } from '@/lib/logs/console/logger'
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
import type { DocumentTag } from '@/app/workspace/[workspaceId]/knowledge/components/document-tag-entry/document-tag-entry'
|
import type { DocumentTag } from '@/app/workspace/[workspaceId]/knowledge/components/document-tag-entry/document-tag-entry'
|
||||||
import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider'
|
import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider'
|
||||||
|
|||||||
@@ -1007,8 +1007,11 @@ export function Sidebar() {
|
|||||||
>
|
>
|
||||||
<UsageIndicator
|
<UsageIndicator
|
||||||
onClick={() => {
|
onClick={() => {
|
||||||
const isBlocked = useSubscriptionStore.getState().getBillingStatus() === 'blocked'
|
const subscriptionStore = useSubscriptionStore.getState()
|
||||||
if (isBlocked) {
|
const isBlocked = subscriptionStore.getBillingStatus() === 'blocked'
|
||||||
|
const canUpgrade = subscriptionStore.canUpgrade()
|
||||||
|
|
||||||
|
if (isBlocked || !canUpgrade) {
|
||||||
if (typeof window !== 'undefined') {
|
if (typeof window !== 'undefined') {
|
||||||
window.dispatchEvent(
|
window.dispatchEvent(
|
||||||
new CustomEvent('open-settings', { detail: { tab: 'subscription' } })
|
new CustomEvent('open-settings', { detail: { tab: 'subscription' } })
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { task } from '@trigger.dev/sdk'
|
import { task } from '@trigger.dev/sdk'
|
||||||
|
import { env } from '@/lib/env'
|
||||||
import { processDocumentAsync } from '@/lib/knowledge/documents/service'
|
import { processDocumentAsync } from '@/lib/knowledge/documents/service'
|
||||||
import { createLogger } from '@/lib/logs/console/logger'
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
|
|
||||||
@@ -25,15 +26,15 @@ export type DocumentProcessingPayload = {
|
|||||||
|
|
||||||
export const processDocument = task({
|
export const processDocument = task({
|
||||||
id: 'knowledge-process-document',
|
id: 'knowledge-process-document',
|
||||||
maxDuration: 300,
|
maxDuration: env.KB_CONFIG_MAX_DURATION || 300,
|
||||||
retry: {
|
retry: {
|
||||||
maxAttempts: 3,
|
maxAttempts: env.KB_CONFIG_MAX_ATTEMPTS || 3,
|
||||||
factor: 2,
|
factor: env.KB_CONFIG_RETRY_FACTOR || 2,
|
||||||
minTimeoutInMs: 1000,
|
minTimeoutInMs: env.KB_CONFIG_MIN_TIMEOUT || 1000,
|
||||||
maxTimeoutInMs: 10000,
|
maxTimeoutInMs: env.KB_CONFIG_MAX_TIMEOUT || 10000,
|
||||||
},
|
},
|
||||||
queue: {
|
queue: {
|
||||||
concurrencyLimit: 20,
|
concurrencyLimit: env.KB_CONFIG_CONCURRENCY_LIMIT || 20,
|
||||||
name: 'document-processing-queue',
|
name: 'document-processing-queue',
|
||||||
},
|
},
|
||||||
run: async (payload: DocumentProcessingPayload) => {
|
run: async (payload: DocumentProcessingPayload) => {
|
||||||
|
|||||||
@@ -99,6 +99,10 @@ export async function executeWebhookJob(payload: WebhookExecutionPayload) {
|
|||||||
userId: payload.userId,
|
userId: payload.userId,
|
||||||
workspaceId: '', // TODO: Get from workflow if needed
|
workspaceId: '', // TODO: Get from workflow if needed
|
||||||
variables: decryptedEnvVars,
|
variables: decryptedEnvVars,
|
||||||
|
initialInput: payload.body || {},
|
||||||
|
triggerData: { provider: payload.provider, blockId: payload.blockId },
|
||||||
|
startBlockId: payload.blockId,
|
||||||
|
executionType: 'webhook',
|
||||||
})
|
})
|
||||||
|
|
||||||
// Merge subblock states (matching workflow-execution pattern)
|
// Merge subblock states (matching workflow-execution pattern)
|
||||||
|
|||||||
@@ -118,6 +118,72 @@ export const MySQLBlock: BlockConfig<MySQLResponse> = {
|
|||||||
placeholder: 'SELECT * FROM users WHERE active = true',
|
placeholder: 'SELECT * FROM users WHERE active = true',
|
||||||
condition: { field: 'operation', value: 'query' },
|
condition: { field: 'operation', value: 'query' },
|
||||||
required: true,
|
required: true,
|
||||||
|
wandConfig: {
|
||||||
|
enabled: true,
|
||||||
|
maintainHistory: true,
|
||||||
|
prompt: `You are an expert MySQL database developer. Write MySQL SQL queries based on the user's request.
|
||||||
|
|
||||||
|
### CONTEXT
|
||||||
|
{context}
|
||||||
|
|
||||||
|
### CRITICAL INSTRUCTION
|
||||||
|
Return ONLY the SQL query. Do not include any explanations, markdown formatting, comments, or additional text. Just the raw SQL query.
|
||||||
|
|
||||||
|
### QUERY GUIDELINES
|
||||||
|
1. **Syntax**: Use MySQL-specific syntax and functions
|
||||||
|
2. **Performance**: Write efficient queries with proper indexing considerations
|
||||||
|
3. **Security**: Use parameterized queries when applicable
|
||||||
|
4. **Readability**: Format queries with proper indentation and spacing
|
||||||
|
5. **Best Practices**: Follow MySQL naming conventions
|
||||||
|
|
||||||
|
### MYSQL FEATURES
|
||||||
|
- Use MySQL-specific functions (IFNULL, DATE_FORMAT, CONCAT, etc.)
|
||||||
|
- Leverage MySQL features like GROUP_CONCAT, AUTO_INCREMENT
|
||||||
|
- Use proper MySQL data types (VARCHAR, DATETIME, DECIMAL, JSON, etc.)
|
||||||
|
- Include appropriate LIMIT clauses for large result sets
|
||||||
|
|
||||||
|
### EXAMPLES
|
||||||
|
|
||||||
|
**Simple Select**: "Get all active users"
|
||||||
|
→ SELECT id, name, email, created_at
|
||||||
|
FROM users
|
||||||
|
WHERE active = 1
|
||||||
|
ORDER BY created_at DESC;
|
||||||
|
|
||||||
|
**Complex Join**: "Get users with their order counts and total spent"
|
||||||
|
→ SELECT
|
||||||
|
u.id,
|
||||||
|
u.name,
|
||||||
|
u.email,
|
||||||
|
COUNT(o.id) as order_count,
|
||||||
|
IFNULL(SUM(o.total), 0) as total_spent
|
||||||
|
FROM users u
|
||||||
|
LEFT JOIN orders o ON u.id = o.user_id
|
||||||
|
WHERE u.active = 1
|
||||||
|
GROUP BY u.id, u.name, u.email
|
||||||
|
HAVING COUNT(o.id) > 0
|
||||||
|
ORDER BY total_spent DESC;
|
||||||
|
|
||||||
|
**With Subquery**: "Get top 10 products by sales"
|
||||||
|
→ SELECT
|
||||||
|
p.id,
|
||||||
|
p.name,
|
||||||
|
(SELECT SUM(oi.quantity * oi.price)
|
||||||
|
FROM order_items oi
|
||||||
|
JOIN orders o ON oi.order_id = o.id
|
||||||
|
WHERE oi.product_id = p.id
|
||||||
|
AND o.created_at >= DATE_SUB(NOW(), INTERVAL 30 DAY)
|
||||||
|
) as total_sales
|
||||||
|
FROM products p
|
||||||
|
WHERE p.active = 1
|
||||||
|
ORDER BY total_sales DESC
|
||||||
|
LIMIT 10;
|
||||||
|
|
||||||
|
### REMEMBER
|
||||||
|
Return ONLY the SQL query - no explanations, no markdown, no extra text.`,
|
||||||
|
placeholder: 'Describe the SQL query you need...',
|
||||||
|
generationType: 'sql-query',
|
||||||
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
id: 'query',
|
id: 'query',
|
||||||
@@ -127,6 +193,72 @@ export const MySQLBlock: BlockConfig<MySQLResponse> = {
|
|||||||
placeholder: 'SELECT * FROM table_name',
|
placeholder: 'SELECT * FROM table_name',
|
||||||
condition: { field: 'operation', value: 'execute' },
|
condition: { field: 'operation', value: 'execute' },
|
||||||
required: true,
|
required: true,
|
||||||
|
wandConfig: {
|
||||||
|
enabled: true,
|
||||||
|
maintainHistory: true,
|
||||||
|
prompt: `You are an expert MySQL database developer. Write MySQL SQL queries based on the user's request.
|
||||||
|
|
||||||
|
### CONTEXT
|
||||||
|
{context}
|
||||||
|
|
||||||
|
### CRITICAL INSTRUCTION
|
||||||
|
Return ONLY the SQL query. Do not include any explanations, markdown formatting, comments, or additional text. Just the raw SQL query.
|
||||||
|
|
||||||
|
### QUERY GUIDELINES
|
||||||
|
1. **Syntax**: Use MySQL-specific syntax and functions
|
||||||
|
2. **Performance**: Write efficient queries with proper indexing considerations
|
||||||
|
3. **Security**: Use parameterized queries when applicable
|
||||||
|
4. **Readability**: Format queries with proper indentation and spacing
|
||||||
|
5. **Best Practices**: Follow MySQL naming conventions
|
||||||
|
|
||||||
|
### MYSQL FEATURES
|
||||||
|
- Use MySQL-specific functions (IFNULL, DATE_FORMAT, CONCAT, etc.)
|
||||||
|
- Leverage MySQL features like GROUP_CONCAT, AUTO_INCREMENT
|
||||||
|
- Use proper MySQL data types (VARCHAR, DATETIME, DECIMAL, JSON, etc.)
|
||||||
|
- Include appropriate LIMIT clauses for large result sets
|
||||||
|
|
||||||
|
### EXAMPLES
|
||||||
|
|
||||||
|
**Simple Select**: "Get all active users"
|
||||||
|
→ SELECT id, name, email, created_at
|
||||||
|
FROM users
|
||||||
|
WHERE active = 1
|
||||||
|
ORDER BY created_at DESC;
|
||||||
|
|
||||||
|
**Complex Join**: "Get users with their order counts and total spent"
|
||||||
|
→ SELECT
|
||||||
|
u.id,
|
||||||
|
u.name,
|
||||||
|
u.email,
|
||||||
|
COUNT(o.id) as order_count,
|
||||||
|
IFNULL(SUM(o.total), 0) as total_spent
|
||||||
|
FROM users u
|
||||||
|
LEFT JOIN orders o ON u.id = o.user_id
|
||||||
|
WHERE u.active = 1
|
||||||
|
GROUP BY u.id, u.name, u.email
|
||||||
|
HAVING COUNT(o.id) > 0
|
||||||
|
ORDER BY total_spent DESC;
|
||||||
|
|
||||||
|
**With Subquery**: "Get top 10 products by sales"
|
||||||
|
→ SELECT
|
||||||
|
p.id,
|
||||||
|
p.name,
|
||||||
|
(SELECT SUM(oi.quantity * oi.price)
|
||||||
|
FROM order_items oi
|
||||||
|
JOIN orders o ON oi.order_id = o.id
|
||||||
|
WHERE oi.product_id = p.id
|
||||||
|
AND o.created_at >= DATE_SUB(NOW(), INTERVAL 30 DAY)
|
||||||
|
) as total_sales
|
||||||
|
FROM products p
|
||||||
|
WHERE p.active = 1
|
||||||
|
ORDER BY total_sales DESC
|
||||||
|
LIMIT 10;
|
||||||
|
|
||||||
|
### REMEMBER
|
||||||
|
Return ONLY the SQL query - no explanations, no markdown, no extra text.`,
|
||||||
|
placeholder: 'Describe the SQL query you need...',
|
||||||
|
generationType: 'sql-query',
|
||||||
|
},
|
||||||
},
|
},
|
||||||
// Data for insert operations
|
// Data for insert operations
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -118,6 +118,73 @@ export const PostgreSQLBlock: BlockConfig<PostgresResponse> = {
|
|||||||
placeholder: 'SELECT * FROM users WHERE active = true',
|
placeholder: 'SELECT * FROM users WHERE active = true',
|
||||||
condition: { field: 'operation', value: 'query' },
|
condition: { field: 'operation', value: 'query' },
|
||||||
required: true,
|
required: true,
|
||||||
|
wandConfig: {
|
||||||
|
enabled: true,
|
||||||
|
maintainHistory: true,
|
||||||
|
prompt: `You are an expert PostgreSQL database developer. Write PostgreSQL SQL queries based on the user's request.
|
||||||
|
|
||||||
|
### CONTEXT
|
||||||
|
{context}
|
||||||
|
|
||||||
|
### CRITICAL INSTRUCTION
|
||||||
|
Return ONLY the SQL query. Do not include any explanations, markdown formatting, comments, or additional text. Just the raw SQL query.
|
||||||
|
|
||||||
|
### QUERY GUIDELINES
|
||||||
|
1. **Syntax**: Use PostgreSQL-specific syntax and functions
|
||||||
|
2. **Performance**: Write efficient queries with proper indexing considerations
|
||||||
|
3. **Security**: Use parameterized queries when applicable
|
||||||
|
4. **Readability**: Format queries with proper indentation and spacing
|
||||||
|
5. **Best Practices**: Follow PostgreSQL naming conventions
|
||||||
|
|
||||||
|
### POSTGRESQL FEATURES
|
||||||
|
- Use PostgreSQL-specific functions (COALESCE, EXTRACT, etc.)
|
||||||
|
- Leverage advanced features like CTEs, window functions, arrays
|
||||||
|
- Use proper PostgreSQL data types (TEXT, TIMESTAMPTZ, JSONB, etc.)
|
||||||
|
- Include appropriate LIMIT clauses for large result sets
|
||||||
|
|
||||||
|
### EXAMPLES
|
||||||
|
|
||||||
|
**Simple Select**: "Get all active users"
|
||||||
|
→ SELECT id, name, email, created_at
|
||||||
|
FROM users
|
||||||
|
WHERE active = true
|
||||||
|
ORDER BY created_at DESC;
|
||||||
|
|
||||||
|
**Complex Join**: "Get users with their order counts and total spent"
|
||||||
|
→ SELECT
|
||||||
|
u.id,
|
||||||
|
u.name,
|
||||||
|
u.email,
|
||||||
|
COUNT(o.id) as order_count,
|
||||||
|
COALESCE(SUM(o.total), 0) as total_spent
|
||||||
|
FROM users u
|
||||||
|
LEFT JOIN orders o ON u.id = o.user_id
|
||||||
|
WHERE u.active = true
|
||||||
|
GROUP BY u.id, u.name, u.email
|
||||||
|
HAVING COUNT(o.id) > 0
|
||||||
|
ORDER BY total_spent DESC;
|
||||||
|
|
||||||
|
**With CTE**: "Get top 10 products by sales"
|
||||||
|
→ WITH product_sales AS (
|
||||||
|
SELECT
|
||||||
|
p.id,
|
||||||
|
p.name,
|
||||||
|
SUM(oi.quantity * oi.price) as total_sales
|
||||||
|
FROM products p
|
||||||
|
JOIN order_items oi ON p.id = oi.product_id
|
||||||
|
JOIN orders o ON oi.order_id = o.id
|
||||||
|
WHERE o.created_at >= CURRENT_DATE - INTERVAL '30 days'
|
||||||
|
GROUP BY p.id, p.name
|
||||||
|
)
|
||||||
|
SELECT * FROM product_sales
|
||||||
|
ORDER BY total_sales DESC
|
||||||
|
LIMIT 10;
|
||||||
|
|
||||||
|
### REMEMBER
|
||||||
|
Return ONLY the SQL query - no explanations, no markdown, no extra text.`,
|
||||||
|
placeholder: 'Describe the SQL query you need...',
|
||||||
|
generationType: 'sql-query',
|
||||||
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
id: 'query',
|
id: 'query',
|
||||||
@@ -127,6 +194,73 @@ export const PostgreSQLBlock: BlockConfig<PostgresResponse> = {
|
|||||||
placeholder: 'SELECT * FROM table_name',
|
placeholder: 'SELECT * FROM table_name',
|
||||||
condition: { field: 'operation', value: 'execute' },
|
condition: { field: 'operation', value: 'execute' },
|
||||||
required: true,
|
required: true,
|
||||||
|
wandConfig: {
|
||||||
|
enabled: true,
|
||||||
|
maintainHistory: true,
|
||||||
|
prompt: `You are an expert PostgreSQL database developer. Write PostgreSQL SQL queries based on the user's request.
|
||||||
|
|
||||||
|
### CONTEXT
|
||||||
|
{context}
|
||||||
|
|
||||||
|
### CRITICAL INSTRUCTION
|
||||||
|
Return ONLY the SQL query. Do not include any explanations, markdown formatting, comments, or additional text. Just the raw SQL query.
|
||||||
|
|
||||||
|
### QUERY GUIDELINES
|
||||||
|
1. **Syntax**: Use PostgreSQL-specific syntax and functions
|
||||||
|
2. **Performance**: Write efficient queries with proper indexing considerations
|
||||||
|
3. **Security**: Use parameterized queries when applicable
|
||||||
|
4. **Readability**: Format queries with proper indentation and spacing
|
||||||
|
5. **Best Practices**: Follow PostgreSQL naming conventions
|
||||||
|
|
||||||
|
### POSTGRESQL FEATURES
|
||||||
|
- Use PostgreSQL-specific functions (COALESCE, EXTRACT, etc.)
|
||||||
|
- Leverage advanced features like CTEs, window functions, arrays
|
||||||
|
- Use proper PostgreSQL data types (TEXT, TIMESTAMPTZ, JSONB, etc.)
|
||||||
|
- Include appropriate LIMIT clauses for large result sets
|
||||||
|
|
||||||
|
### EXAMPLES
|
||||||
|
|
||||||
|
**Simple Select**: "Get all active users"
|
||||||
|
→ SELECT id, name, email, created_at
|
||||||
|
FROM users
|
||||||
|
WHERE active = true
|
||||||
|
ORDER BY created_at DESC;
|
||||||
|
|
||||||
|
**Complex Join**: "Get users with their order counts and total spent"
|
||||||
|
→ SELECT
|
||||||
|
u.id,
|
||||||
|
u.name,
|
||||||
|
u.email,
|
||||||
|
COUNT(o.id) as order_count,
|
||||||
|
COALESCE(SUM(o.total), 0) as total_spent
|
||||||
|
FROM users u
|
||||||
|
LEFT JOIN orders o ON u.id = o.user_id
|
||||||
|
WHERE u.active = true
|
||||||
|
GROUP BY u.id, u.name, u.email
|
||||||
|
HAVING COUNT(o.id) > 0
|
||||||
|
ORDER BY total_spent DESC;
|
||||||
|
|
||||||
|
**With CTE**: "Get top 10 products by sales"
|
||||||
|
→ WITH product_sales AS (
|
||||||
|
SELECT
|
||||||
|
p.id,
|
||||||
|
p.name,
|
||||||
|
SUM(oi.quantity * oi.price) as total_sales
|
||||||
|
FROM products p
|
||||||
|
JOIN order_items oi ON p.id = oi.product_id
|
||||||
|
JOIN orders o ON oi.order_id = o.id
|
||||||
|
WHERE o.created_at >= CURRENT_DATE - INTERVAL '30 days'
|
||||||
|
GROUP BY p.id, p.name
|
||||||
|
)
|
||||||
|
SELECT * FROM product_sales
|
||||||
|
ORDER BY total_sales DESC
|
||||||
|
LIMIT 10;
|
||||||
|
|
||||||
|
### REMEMBER
|
||||||
|
Return ONLY the SQL query - no explanations, no markdown, no extra text.`,
|
||||||
|
placeholder: 'Describe the SQL query you need...',
|
||||||
|
generationType: 'sql-query',
|
||||||
|
},
|
||||||
},
|
},
|
||||||
// Data for insert operations
|
// Data for insert operations
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -94,6 +94,66 @@ export const SupabaseBlock: BlockConfig<SupabaseResponse> = {
|
|||||||
placeholder: 'id=eq.123',
|
placeholder: 'id=eq.123',
|
||||||
condition: { field: 'operation', value: 'get_row' },
|
condition: { field: 'operation', value: 'get_row' },
|
||||||
required: true,
|
required: true,
|
||||||
|
wandConfig: {
|
||||||
|
enabled: true,
|
||||||
|
maintainHistory: true,
|
||||||
|
prompt: `You are an expert in PostgREST API syntax. Generate PostgREST filter expressions based on the user's request.
|
||||||
|
|
||||||
|
### CONTEXT
|
||||||
|
{context}
|
||||||
|
|
||||||
|
### CRITICAL INSTRUCTION
|
||||||
|
Return ONLY the PostgREST filter expression. Do not include any explanations, markdown formatting, or additional text. Just the raw filter expression.
|
||||||
|
|
||||||
|
### POSTGREST FILTER SYNTAX
|
||||||
|
PostgREST uses a specific syntax for filtering data. The format is:
|
||||||
|
column=operator.value
|
||||||
|
|
||||||
|
### OPERATORS
|
||||||
|
- **eq** - equals: \`id=eq.123\`
|
||||||
|
- **neq** - not equals: \`status=neq.inactive\`
|
||||||
|
- **gt** - greater than: \`age=gt.18\`
|
||||||
|
- **gte** - greater than or equal: \`score=gte.80\`
|
||||||
|
- **lt** - less than: \`price=lt.100\`
|
||||||
|
- **lte** - less than or equal: \`rating=lte.5\`
|
||||||
|
- **like** - pattern matching: \`name=like.*john*\`
|
||||||
|
- **ilike** - case-insensitive like: \`email=ilike.*@gmail.com\`
|
||||||
|
- **in** - in list: \`category=in.(tech,science,art)\`
|
||||||
|
- **is** - is null/not null: \`deleted_at=is.null\`
|
||||||
|
- **not** - negation: \`not.and=(status.eq.active,verified.eq.true)\`
|
||||||
|
|
||||||
|
### COMBINING FILTERS
|
||||||
|
- **AND**: Use \`&\` or \`and=(...)\`: \`id=eq.123&status=eq.active\`
|
||||||
|
- **OR**: Use \`or=(...)\`: \`or=(status.eq.active,status.eq.pending)\`
|
||||||
|
|
||||||
|
### EXAMPLES
|
||||||
|
|
||||||
|
**Simple equality**: "Find user with ID 123"
|
||||||
|
→ id=eq.123
|
||||||
|
|
||||||
|
**Text search**: "Find users with Gmail addresses"
|
||||||
|
→ email=ilike.*@gmail.com
|
||||||
|
|
||||||
|
**Range filter**: "Find products under $50"
|
||||||
|
→ price=lt.50
|
||||||
|
|
||||||
|
**Multiple conditions**: "Find active users over 18"
|
||||||
|
→ age=gt.18&status=eq.active
|
||||||
|
|
||||||
|
**OR condition**: "Find active or pending orders"
|
||||||
|
→ or=(status.eq.active,status.eq.pending)
|
||||||
|
|
||||||
|
**In list**: "Find posts in specific categories"
|
||||||
|
→ category=in.(tech,science,health)
|
||||||
|
|
||||||
|
**Null check**: "Find users without a profile picture"
|
||||||
|
→ profile_image=is.null
|
||||||
|
|
||||||
|
### REMEMBER
|
||||||
|
Return ONLY the PostgREST filter expression - no explanations, no markdown, no extra text.`,
|
||||||
|
placeholder: 'Describe the filter condition you need...',
|
||||||
|
generationType: 'postgrest',
|
||||||
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
id: 'filter',
|
id: 'filter',
|
||||||
@@ -103,6 +163,66 @@ export const SupabaseBlock: BlockConfig<SupabaseResponse> = {
|
|||||||
placeholder: 'id=eq.123',
|
placeholder: 'id=eq.123',
|
||||||
condition: { field: 'operation', value: 'update' },
|
condition: { field: 'operation', value: 'update' },
|
||||||
required: true,
|
required: true,
|
||||||
|
wandConfig: {
|
||||||
|
enabled: true,
|
||||||
|
maintainHistory: true,
|
||||||
|
prompt: `You are an expert in PostgREST API syntax. Generate PostgREST filter expressions based on the user's request.
|
||||||
|
|
||||||
|
### CONTEXT
|
||||||
|
{context}
|
||||||
|
|
||||||
|
### CRITICAL INSTRUCTION
|
||||||
|
Return ONLY the PostgREST filter expression. Do not include any explanations, markdown formatting, or additional text. Just the raw filter expression.
|
||||||
|
|
||||||
|
### POSTGREST FILTER SYNTAX
|
||||||
|
PostgREST uses a specific syntax for filtering data. The format is:
|
||||||
|
column=operator.value
|
||||||
|
|
||||||
|
### OPERATORS
|
||||||
|
- **eq** - equals: \`id=eq.123\`
|
||||||
|
- **neq** - not equals: \`status=neq.inactive\`
|
||||||
|
- **gt** - greater than: \`age=gt.18\`
|
||||||
|
- **gte** - greater than or equal: \`score=gte.80\`
|
||||||
|
- **lt** - less than: \`price=lt.100\`
|
||||||
|
- **lte** - less than or equal: \`rating=lte.5\`
|
||||||
|
- **like** - pattern matching: \`name=like.*john*\`
|
||||||
|
- **ilike** - case-insensitive like: \`email=ilike.*@gmail.com\`
|
||||||
|
- **in** - in list: \`category=in.(tech,science,art)\`
|
||||||
|
- **is** - is null/not null: \`deleted_at=is.null\`
|
||||||
|
- **not** - negation: \`not.and=(status.eq.active,verified.eq.true)\`
|
||||||
|
|
||||||
|
### COMBINING FILTERS
|
||||||
|
- **AND**: Use \`&\` or \`and=(...)\`: \`id=eq.123&status=eq.active\`
|
||||||
|
- **OR**: Use \`or=(...)\`: \`or=(status.eq.active,status.eq.pending)\`
|
||||||
|
|
||||||
|
### EXAMPLES
|
||||||
|
|
||||||
|
**Simple equality**: "Find user with ID 123"
|
||||||
|
→ id=eq.123
|
||||||
|
|
||||||
|
**Text search**: "Find users with Gmail addresses"
|
||||||
|
→ email=ilike.*@gmail.com
|
||||||
|
|
||||||
|
**Range filter**: "Find products under $50"
|
||||||
|
→ price=lt.50
|
||||||
|
|
||||||
|
**Multiple conditions**: "Find active users over 18"
|
||||||
|
→ age=gt.18&status=eq.active
|
||||||
|
|
||||||
|
**OR condition**: "Find active or pending orders"
|
||||||
|
→ or=(status.eq.active,status.eq.pending)
|
||||||
|
|
||||||
|
**In list**: "Find posts in specific categories"
|
||||||
|
→ category=in.(tech,science,health)
|
||||||
|
|
||||||
|
**Null check**: "Find users without a profile picture"
|
||||||
|
→ profile_image=is.null
|
||||||
|
|
||||||
|
### REMEMBER
|
||||||
|
Return ONLY the PostgREST filter expression - no explanations, no markdown, no extra text.`,
|
||||||
|
placeholder: 'Describe the filter condition you need...',
|
||||||
|
generationType: 'postgrest',
|
||||||
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
id: 'filter',
|
id: 'filter',
|
||||||
@@ -112,6 +232,66 @@ export const SupabaseBlock: BlockConfig<SupabaseResponse> = {
|
|||||||
placeholder: 'id=eq.123',
|
placeholder: 'id=eq.123',
|
||||||
condition: { field: 'operation', value: 'delete' },
|
condition: { field: 'operation', value: 'delete' },
|
||||||
required: true,
|
required: true,
|
||||||
|
wandConfig: {
|
||||||
|
enabled: true,
|
||||||
|
maintainHistory: true,
|
||||||
|
prompt: `You are an expert in PostgREST API syntax. Generate PostgREST filter expressions based on the user's request.
|
||||||
|
|
||||||
|
### CONTEXT
|
||||||
|
{context}
|
||||||
|
|
||||||
|
### CRITICAL INSTRUCTION
|
||||||
|
Return ONLY the PostgREST filter expression. Do not include any explanations, markdown formatting, or additional text. Just the raw filter expression.
|
||||||
|
|
||||||
|
### POSTGREST FILTER SYNTAX
|
||||||
|
PostgREST uses a specific syntax for filtering data. The format is:
|
||||||
|
column=operator.value
|
||||||
|
|
||||||
|
### OPERATORS
|
||||||
|
- **eq** - equals: \`id=eq.123\`
|
||||||
|
- **neq** - not equals: \`status=neq.inactive\`
|
||||||
|
- **gt** - greater than: \`age=gt.18\`
|
||||||
|
- **gte** - greater than or equal: \`score=gte.80\`
|
||||||
|
- **lt** - less than: \`price=lt.100\`
|
||||||
|
- **lte** - less than or equal: \`rating=lte.5\`
|
||||||
|
- **like** - pattern matching: \`name=like.*john*\`
|
||||||
|
- **ilike** - case-insensitive like: \`email=ilike.*@gmail.com\`
|
||||||
|
- **in** - in list: \`category=in.(tech,science,art)\`
|
||||||
|
- **is** - is null/not null: \`deleted_at=is.null\`
|
||||||
|
- **not** - negation: \`not.and=(status.eq.active,verified.eq.true)\`
|
||||||
|
|
||||||
|
### COMBINING FILTERS
|
||||||
|
- **AND**: Use \`&\` or \`and=(...)\`: \`id=eq.123&status=eq.active\`
|
||||||
|
- **OR**: Use \`or=(...)\`: \`or=(status.eq.active,status.eq.pending)\`
|
||||||
|
|
||||||
|
### EXAMPLES
|
||||||
|
|
||||||
|
**Simple equality**: "Find user with ID 123"
|
||||||
|
→ id=eq.123
|
||||||
|
|
||||||
|
**Text search**: "Find users with Gmail addresses"
|
||||||
|
→ email=ilike.*@gmail.com
|
||||||
|
|
||||||
|
**Range filter**: "Find products under $50"
|
||||||
|
→ price=lt.50
|
||||||
|
|
||||||
|
**Multiple conditions**: "Find active users over 18"
|
||||||
|
→ age=gt.18&status=eq.active
|
||||||
|
|
||||||
|
**OR condition**: "Find active or pending orders"
|
||||||
|
→ or=(status.eq.active,status.eq.pending)
|
||||||
|
|
||||||
|
**In list**: "Find posts in specific categories"
|
||||||
|
→ category=in.(tech,science,health)
|
||||||
|
|
||||||
|
**Null check**: "Find users without a profile picture"
|
||||||
|
→ profile_image=is.null
|
||||||
|
|
||||||
|
### REMEMBER
|
||||||
|
Return ONLY the PostgREST filter expression - no explanations, no markdown, no extra text.`,
|
||||||
|
placeholder: 'Describe the filter condition you need...',
|
||||||
|
generationType: 'postgrest',
|
||||||
|
},
|
||||||
},
|
},
|
||||||
// Optional filter for query operation
|
// Optional filter for query operation
|
||||||
{
|
{
|
||||||
@@ -121,6 +301,66 @@ export const SupabaseBlock: BlockConfig<SupabaseResponse> = {
|
|||||||
layout: 'full',
|
layout: 'full',
|
||||||
placeholder: 'status=eq.active',
|
placeholder: 'status=eq.active',
|
||||||
condition: { field: 'operation', value: 'query' },
|
condition: { field: 'operation', value: 'query' },
|
||||||
|
wandConfig: {
|
||||||
|
enabled: true,
|
||||||
|
maintainHistory: true,
|
||||||
|
prompt: `You are an expert in PostgREST API syntax. Generate PostgREST filter expressions based on the user's request.
|
||||||
|
|
||||||
|
### CONTEXT
|
||||||
|
{context}
|
||||||
|
|
||||||
|
### CRITICAL INSTRUCTION
|
||||||
|
Return ONLY the PostgREST filter expression. Do not include any explanations, markdown formatting, or additional text. Just the raw filter expression.
|
||||||
|
|
||||||
|
### POSTGREST FILTER SYNTAX
|
||||||
|
PostgREST uses a specific syntax for filtering data. The format is:
|
||||||
|
column=operator.value
|
||||||
|
|
||||||
|
### OPERATORS
|
||||||
|
- **eq** - equals: \`id=eq.123\`
|
||||||
|
- **neq** - not equals: \`status=neq.inactive\`
|
||||||
|
- **gt** - greater than: \`age=gt.18\`
|
||||||
|
- **gte** - greater than or equal: \`score=gte.80\`
|
||||||
|
- **lt** - less than: \`price=lt.100\`
|
||||||
|
- **lte** - less than or equal: \`rating=lte.5\`
|
||||||
|
- **like** - pattern matching: \`name=like.*john*\`
|
||||||
|
- **ilike** - case-insensitive like: \`email=ilike.*@gmail.com\`
|
||||||
|
- **in** - in list: \`category=in.(tech,science,art)\`
|
||||||
|
- **is** - is null/not null: \`deleted_at=is.null\`
|
||||||
|
- **not** - negation: \`not.and=(status.eq.active,verified.eq.true)\`
|
||||||
|
|
||||||
|
### COMBINING FILTERS
|
||||||
|
- **AND**: Use \`&\` or \`and=(...)\`: \`id=eq.123&status=eq.active\`
|
||||||
|
- **OR**: Use \`or=(...)\`: \`or=(status.eq.active,status.eq.pending)\`
|
||||||
|
|
||||||
|
### EXAMPLES
|
||||||
|
|
||||||
|
**Simple equality**: "Find user with ID 123"
|
||||||
|
→ id=eq.123
|
||||||
|
|
||||||
|
**Text search**: "Find users with Gmail addresses"
|
||||||
|
→ email=ilike.*@gmail.com
|
||||||
|
|
||||||
|
**Range filter**: "Find products under $50"
|
||||||
|
→ price=lt.50
|
||||||
|
|
||||||
|
**Multiple conditions**: "Find active users over 18"
|
||||||
|
→ age=gt.18&status=eq.active
|
||||||
|
|
||||||
|
**OR condition**: "Find active or pending orders"
|
||||||
|
→ or=(status.eq.active,status.eq.pending)
|
||||||
|
|
||||||
|
**In list**: "Find posts in specific categories"
|
||||||
|
→ category=in.(tech,science,health)
|
||||||
|
|
||||||
|
**Null check**: "Find users without a profile picture"
|
||||||
|
→ profile_image=is.null
|
||||||
|
|
||||||
|
### REMEMBER
|
||||||
|
Return ONLY the PostgREST filter expression - no explanations, no markdown, no extra text.`,
|
||||||
|
placeholder: 'Describe the filter condition...',
|
||||||
|
generationType: 'postgrest',
|
||||||
|
},
|
||||||
},
|
},
|
||||||
// Optional order by for query operation
|
// Optional order by for query operation
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -17,6 +17,8 @@ export type GenerationType =
|
|||||||
| 'json-object'
|
| 'json-object'
|
||||||
| 'system-prompt'
|
| 'system-prompt'
|
||||||
| 'custom-tool-schema'
|
| 'custom-tool-schema'
|
||||||
|
| 'sql-query'
|
||||||
|
| 'postgrest'
|
||||||
|
|
||||||
// SubBlock types
|
// SubBlock types
|
||||||
export type SubBlockType =
|
export type SubBlockType =
|
||||||
|
|||||||
@@ -14,9 +14,9 @@ const Slider = React.forwardRef<
|
|||||||
{...props}
|
{...props}
|
||||||
>
|
>
|
||||||
<SliderPrimitive.Track className='relative h-2 w-full grow overflow-hidden rounded-full bg-secondary'>
|
<SliderPrimitive.Track className='relative h-2 w-full grow overflow-hidden rounded-full bg-secondary'>
|
||||||
<SliderPrimitive.Range className='absolute h-full bg-primary' />
|
<SliderPrimitive.Range className='absolute h-full bg-primary dark:bg-white' />
|
||||||
</SliderPrimitive.Track>
|
</SliderPrimitive.Track>
|
||||||
<SliderPrimitive.Thumb className='block h-5 w-5 rounded-full border-2 border-primary bg-background ring-offset-background transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50' />
|
<SliderPrimitive.Thumb className='block h-5 w-5 rounded-full border-2 border-primary bg-background ring-offset-background transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 dark:border-white dark:bg-black' />
|
||||||
</SliderPrimitive.Root>
|
</SliderPrimitive.Root>
|
||||||
))
|
))
|
||||||
Slider.displayName = SliderPrimitive.Root.displayName
|
Slider.displayName = SliderPrimitive.Root.displayName
|
||||||
|
|||||||
@@ -961,9 +961,13 @@ export class Executor {
|
|||||||
const connectedToStartBlock = this.actualWorkflow.connections
|
const connectedToStartBlock = this.actualWorkflow.connections
|
||||||
.filter((conn) => conn.source === initBlock.id)
|
.filter((conn) => conn.source === initBlock.id)
|
||||||
.map((conn) => conn.target)
|
.map((conn) => conn.target)
|
||||||
|
// Skip trigger-category targets when seeding from starter (manual/debug runs)
|
||||||
connectedToStartBlock.forEach((blockId) => {
|
connectedToStartBlock.forEach((blockId) => {
|
||||||
context.activeExecutionPath.add(blockId)
|
const targetBlock = this.actualWorkflow.blocks.find((b) => b.id === blockId)
|
||||||
|
const isTriggerCategory = (targetBlock as any)?.metadata?.category === 'triggers'
|
||||||
|
if (!isTriggerCategory) {
|
||||||
|
context.activeExecutionPath.add(blockId)
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -203,14 +203,18 @@ export class PathTracker {
|
|||||||
if (!context.activeExecutionPath.has(conn.target)) {
|
if (!context.activeExecutionPath.has(conn.target)) {
|
||||||
const targetBlock = this.getBlock(conn.target)
|
const targetBlock = this.getBlock(conn.target)
|
||||||
const targetBlockType = targetBlock?.metadata?.id
|
const targetBlockType = targetBlock?.metadata?.id
|
||||||
|
const isTriggerCategory = (targetBlock as any)?.metadata?.category === 'triggers'
|
||||||
|
|
||||||
// Use routing strategy to determine if this connection should be activated
|
// Use routing strategy to determine if this connection should be activated
|
||||||
if (!Routing.shouldSkipConnection(conn.sourceHandle, targetBlockType || '')) {
|
if (!Routing.shouldSkipConnection(conn.sourceHandle, targetBlockType || '')) {
|
||||||
context.activeExecutionPath.add(conn.target)
|
// Do not activate or traverse trigger blocks during downstream activation from manual paths
|
||||||
|
if (!isTriggerCategory) {
|
||||||
|
context.activeExecutionPath.add(conn.target)
|
||||||
|
|
||||||
// Recursively activate downstream paths if the target block should activate downstream
|
// Recursively activate downstream paths if the target block should activate downstream
|
||||||
if (Routing.shouldActivateDownstream(targetBlockType || '')) {
|
if (Routing.shouldActivateDownstream(targetBlockType || '')) {
|
||||||
this.activateDownstreamPathsSelectively(conn.target, context)
|
this.activateDownstreamPathsSelectively(conn.target, context)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -233,7 +237,11 @@ export class PathTracker {
|
|||||||
)
|
)
|
||||||
|
|
||||||
for (const conn of targetConnections) {
|
for (const conn of targetConnections) {
|
||||||
context.activeExecutionPath.add(conn.target)
|
const targetBlock = this.getBlock(conn.target)
|
||||||
|
const isTriggerCategory = (targetBlock as any)?.metadata?.category === 'triggers'
|
||||||
|
if (!isTriggerCategory) {
|
||||||
|
context.activeExecutionPath.add(conn.target)
|
||||||
|
}
|
||||||
logger.debug(`Condition ${block.id} activated path to: ${conn.target}`)
|
logger.debug(`Condition ${block.id} activated path to: ${conn.target}`)
|
||||||
|
|
||||||
// Check if the selected target should activate downstream paths
|
// Check if the selected target should activate downstream paths
|
||||||
@@ -282,13 +290,16 @@ export class PathTracker {
|
|||||||
if (this.shouldActivateConnection(conn, hasError, isPartOfLoop, blockLoops, context)) {
|
if (this.shouldActivateConnection(conn, hasError, isPartOfLoop, blockLoops, context)) {
|
||||||
const targetBlock = this.getBlock(conn.target)
|
const targetBlock = this.getBlock(conn.target)
|
||||||
const targetBlockType = targetBlock?.metadata?.id
|
const targetBlockType = targetBlock?.metadata?.id
|
||||||
|
const isTriggerCategory = (targetBlock as any)?.metadata?.category === 'triggers'
|
||||||
|
|
||||||
// Use routing strategy to determine if this connection should be activated
|
// Use routing strategy to determine if this connection should be activated
|
||||||
if (Routing.shouldSkipConnection(conn.sourceHandle, targetBlockType || '')) {
|
if (Routing.shouldSkipConnection(conn.sourceHandle, targetBlockType || '')) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
context.activeExecutionPath.add(conn.target)
|
if (!isTriggerCategory) {
|
||||||
|
context.activeExecutionPath.add(conn.target)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -782,6 +782,16 @@ export function useCollaborativeWorkflow() {
|
|||||||
|
|
||||||
const newTriggerMode = !currentBlock.triggerMode
|
const newTriggerMode = !currentBlock.triggerMode
|
||||||
|
|
||||||
|
// If enabling trigger mode, proactively remove incoming edges for consistency across clients
|
||||||
|
if (newTriggerMode) {
|
||||||
|
const incomingEdges = Object.values(workflowStore.edges).filter((e) => e.target === id)
|
||||||
|
for (const edge of incomingEdges) {
|
||||||
|
executeQueuedOperation('remove', 'edge', { id: edge.id }, () =>
|
||||||
|
workflowStore.removeEdge(edge.id)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
executeQueuedOperation(
|
executeQueuedOperation(
|
||||||
'update-trigger-mode',
|
'update-trigger-mode',
|
||||||
'block',
|
'block',
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
'use client'
|
'use client'
|
||||||
|
|
||||||
import { useCallback, useEffect, useState } from 'react'
|
import { useCallback, useEffect, useState } from 'react'
|
||||||
import type { TagSlot } from '@/lib/constants/knowledge'
|
import type { TagSlot } from '@/lib/knowledge/consts'
|
||||||
import { createLogger } from '@/lib/logs/console/logger'
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
|
|
||||||
const logger = createLogger('useKnowledgeBaseTagDefinitions')
|
const logger = createLogger('useKnowledgeBaseTagDefinitions')
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
'use client'
|
'use client'
|
||||||
|
|
||||||
import { useCallback, useEffect, useState } from 'react'
|
import { useCallback, useEffect, useState } from 'react'
|
||||||
import type { TagSlot } from '@/lib/constants/knowledge'
|
import type { TagSlot } from '@/lib/knowledge/consts'
|
||||||
import { createLogger } from '@/lib/logs/console/logger'
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
|
|
||||||
const logger = createLogger('useTagDefinitions')
|
const logger = createLogger('useTagDefinitions')
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import { createAuthClient } from 'better-auth/react'
|
|||||||
import type { auth } from '@/lib/auth'
|
import type { auth } from '@/lib/auth'
|
||||||
import { env, getEnv } from '@/lib/env'
|
import { env, getEnv } from '@/lib/env'
|
||||||
import { isProd } from '@/lib/environment'
|
import { isProd } from '@/lib/environment'
|
||||||
import { SessionContext, type SessionHookResult } from '@/lib/session-context'
|
import { SessionContext, type SessionHookResult } from '@/lib/session/session-context'
|
||||||
|
|
||||||
export function getBaseURL() {
|
export function getBaseURL() {
|
||||||
let baseURL
|
let baseURL
|
||||||
|
|||||||
@@ -1,53 +0,0 @@
|
|||||||
/**
|
|
||||||
* Knowledge base and document constants
|
|
||||||
*/
|
|
||||||
|
|
||||||
// Tag slot configuration by field type
|
|
||||||
// Each field type maps to specific database columns
|
|
||||||
export const TAG_SLOT_CONFIG = {
|
|
||||||
text: {
|
|
||||||
slots: ['tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6', 'tag7'] as const,
|
|
||||||
maxSlots: 7,
|
|
||||||
},
|
|
||||||
// Future field types would be added here with their own database columns
|
|
||||||
// date: {
|
|
||||||
// slots: ['tag8', 'tag9'] as const,
|
|
||||||
// maxSlots: 2,
|
|
||||||
// },
|
|
||||||
// number: {
|
|
||||||
// slots: ['tag10', 'tag11'] as const,
|
|
||||||
// maxSlots: 2,
|
|
||||||
// },
|
|
||||||
} as const
|
|
||||||
|
|
||||||
// Currently supported field types
|
|
||||||
export const SUPPORTED_FIELD_TYPES = Object.keys(TAG_SLOT_CONFIG) as Array<
|
|
||||||
keyof typeof TAG_SLOT_CONFIG
|
|
||||||
>
|
|
||||||
|
|
||||||
// All tag slots (for backward compatibility)
|
|
||||||
export const TAG_SLOTS = TAG_SLOT_CONFIG.text.slots
|
|
||||||
|
|
||||||
// Maximum number of tag slots for text type (for backward compatibility)
|
|
||||||
export const MAX_TAG_SLOTS = TAG_SLOT_CONFIG.text.maxSlots
|
|
||||||
|
|
||||||
// Type for tag slot names
|
|
||||||
export type TagSlot = (typeof TAG_SLOTS)[number]
|
|
||||||
|
|
||||||
// Helper function to get available slots for a field type
|
|
||||||
export function getSlotsForFieldType(fieldType: string): readonly string[] {
|
|
||||||
const config = TAG_SLOT_CONFIG[fieldType as keyof typeof TAG_SLOT_CONFIG]
|
|
||||||
if (!config) {
|
|
||||||
return [] // Return empty array for unsupported field types - system will naturally handle this
|
|
||||||
}
|
|
||||||
return config.slots
|
|
||||||
}
|
|
||||||
|
|
||||||
// Helper function to get max slots for a field type
|
|
||||||
export function getMaxSlotsForFieldType(fieldType: string): number {
|
|
||||||
const config = TAG_SLOT_CONFIG[fieldType as keyof typeof TAG_SLOT_CONFIG]
|
|
||||||
if (!config) {
|
|
||||||
return 0 // Return 0 for unsupported field types
|
|
||||||
}
|
|
||||||
return config.maxSlots
|
|
||||||
}
|
|
||||||
@@ -139,6 +139,17 @@ export const env = createEnv({
|
|||||||
RATE_LIMIT_ENTERPRISE_SYNC: z.string().optional().default('150'), // Enterprise tier sync API executions per minute
|
RATE_LIMIT_ENTERPRISE_SYNC: z.string().optional().default('150'), // Enterprise tier sync API executions per minute
|
||||||
RATE_LIMIT_ENTERPRISE_ASYNC: z.string().optional().default('1000'), // Enterprise tier async API executions per minute
|
RATE_LIMIT_ENTERPRISE_ASYNC: z.string().optional().default('1000'), // Enterprise tier async API executions per minute
|
||||||
|
|
||||||
|
// Knowledge Base Processing Configuration - Shared across all processing methods
|
||||||
|
KB_CONFIG_MAX_DURATION: z.number().optional().default(300), // Max processing duration in s
|
||||||
|
KB_CONFIG_MAX_ATTEMPTS: z.number().optional().default(3), // Max retry attempts
|
||||||
|
KB_CONFIG_RETRY_FACTOR: z.number().optional().default(2), // Retry backoff factor
|
||||||
|
KB_CONFIG_MIN_TIMEOUT: z.number().optional().default(1000), // Min timeout in ms
|
||||||
|
KB_CONFIG_MAX_TIMEOUT: z.number().optional().default(10000), // Max timeout in ms
|
||||||
|
KB_CONFIG_CONCURRENCY_LIMIT: z.number().optional().default(20), // Queue concurrency limit
|
||||||
|
KB_CONFIG_BATCH_SIZE: z.number().optional().default(20), // Processing batch size
|
||||||
|
KB_CONFIG_DELAY_BETWEEN_BATCHES: z.number().optional().default(100), // Delay between batches in ms
|
||||||
|
KB_CONFIG_DELAY_BETWEEN_DOCUMENTS: z.number().optional().default(50), // Delay between documents in ms
|
||||||
|
|
||||||
// Real-time Communication
|
// Real-time Communication
|
||||||
SOCKET_SERVER_URL: z.string().url().optional(), // WebSocket server URL for real-time features
|
SOCKET_SERVER_URL: z.string().url().optional(), // WebSocket server URL for real-time features
|
||||||
SOCKET_PORT: z.number().optional(), // Port for WebSocket server
|
SOCKET_PORT: z.number().optional(), // Port for WebSocket server
|
||||||
|
|||||||
@@ -1,139 +1,108 @@
|
|||||||
import { createReadStream, existsSync } from 'fs'
|
import { existsSync, readFileSync } from 'fs'
|
||||||
import { Readable } from 'stream'
|
import * as Papa from 'papaparse'
|
||||||
import csvParser from 'csv-parser'
|
|
||||||
import type { FileParseResult, FileParser } from '@/lib/file-parsers/types'
|
import type { FileParseResult, FileParser } from '@/lib/file-parsers/types'
|
||||||
import { sanitizeTextForUTF8 } from '@/lib/file-parsers/utils'
|
import { sanitizeTextForUTF8 } from '@/lib/file-parsers/utils'
|
||||||
import { createLogger } from '@/lib/logs/console/logger'
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
|
|
||||||
const logger = createLogger('CsvParser')
|
const logger = createLogger('CsvParser')
|
||||||
|
|
||||||
|
const PARSE_OPTIONS = {
|
||||||
|
header: true,
|
||||||
|
skipEmptyLines: true,
|
||||||
|
transformHeader: (header: string) => sanitizeTextForUTF8(String(header)),
|
||||||
|
transform: (value: string) => sanitizeTextForUTF8(String(value || '')),
|
||||||
|
}
|
||||||
|
|
||||||
export class CsvParser implements FileParser {
|
export class CsvParser implements FileParser {
|
||||||
async parseFile(filePath: string): Promise<FileParseResult> {
|
async parseFile(filePath: string): Promise<FileParseResult> {
|
||||||
return new Promise((resolve, reject) => {
|
try {
|
||||||
try {
|
if (!filePath) {
|
||||||
// Validate input
|
throw new Error('No file path provided')
|
||||||
if (!filePath) {
|
|
||||||
return reject(new Error('No file path provided'))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if file exists
|
|
||||||
if (!existsSync(filePath)) {
|
|
||||||
return reject(new Error(`File not found: ${filePath}`))
|
|
||||||
}
|
|
||||||
|
|
||||||
const results: Record<string, any>[] = []
|
|
||||||
const headers: string[] = []
|
|
||||||
|
|
||||||
createReadStream(filePath)
|
|
||||||
.on('error', (error: Error) => {
|
|
||||||
logger.error('CSV stream error:', error)
|
|
||||||
reject(new Error(`Failed to read CSV file: ${error.message}`))
|
|
||||||
})
|
|
||||||
.pipe(csvParser())
|
|
||||||
.on('headers', (headerList: string[]) => {
|
|
||||||
headers.push(...headerList)
|
|
||||||
})
|
|
||||||
.on('data', (data: Record<string, any>) => {
|
|
||||||
results.push(data)
|
|
||||||
})
|
|
||||||
.on('end', () => {
|
|
||||||
// Convert CSV data to a formatted string representation
|
|
||||||
let content = ''
|
|
||||||
|
|
||||||
// Add headers
|
|
||||||
if (headers.length > 0) {
|
|
||||||
const cleanHeaders = headers.map((h) => sanitizeTextForUTF8(String(h)))
|
|
||||||
content += `${cleanHeaders.join(', ')}\n`
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add rows
|
|
||||||
results.forEach((row) => {
|
|
||||||
const cleanValues = Object.values(row).map((v) =>
|
|
||||||
sanitizeTextForUTF8(String(v || ''))
|
|
||||||
)
|
|
||||||
content += `${cleanValues.join(', ')}\n`
|
|
||||||
})
|
|
||||||
|
|
||||||
resolve({
|
|
||||||
content: sanitizeTextForUTF8(content),
|
|
||||||
metadata: {
|
|
||||||
rowCount: results.length,
|
|
||||||
headers: headers,
|
|
||||||
rawData: results,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.on('error', (error: Error) => {
|
|
||||||
logger.error('CSV parsing error:', error)
|
|
||||||
reject(new Error(`Failed to parse CSV file: ${error.message}`))
|
|
||||||
})
|
|
||||||
} catch (error) {
|
|
||||||
logger.error('CSV general error:', error)
|
|
||||||
reject(new Error(`Failed to process CSV file: ${(error as Error).message}`))
|
|
||||||
}
|
}
|
||||||
})
|
|
||||||
|
if (!existsSync(filePath)) {
|
||||||
|
throw new Error(`File not found: ${filePath}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
const fileContent = readFileSync(filePath, 'utf8')
|
||||||
|
|
||||||
|
const parseResult = Papa.parse(fileContent, PARSE_OPTIONS)
|
||||||
|
|
||||||
|
if (parseResult.errors && parseResult.errors.length > 0) {
|
||||||
|
const errorMessages = parseResult.errors.map((err) => err.message).join(', ')
|
||||||
|
logger.error('CSV parsing errors:', parseResult.errors)
|
||||||
|
throw new Error(`Failed to parse CSV file: ${errorMessages}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
const results = parseResult.data as Record<string, any>[]
|
||||||
|
const headers = parseResult.meta.fields || []
|
||||||
|
|
||||||
|
let content = ''
|
||||||
|
|
||||||
|
if (headers.length > 0) {
|
||||||
|
const cleanHeaders = headers.map((h) => sanitizeTextForUTF8(String(h)))
|
||||||
|
content += `${cleanHeaders.join(', ')}\n`
|
||||||
|
}
|
||||||
|
|
||||||
|
results.forEach((row) => {
|
||||||
|
const cleanValues = Object.values(row).map((v) => sanitizeTextForUTF8(String(v || '')))
|
||||||
|
content += `${cleanValues.join(', ')}\n`
|
||||||
|
})
|
||||||
|
|
||||||
|
return {
|
||||||
|
content: sanitizeTextForUTF8(content),
|
||||||
|
metadata: {
|
||||||
|
rowCount: results.length,
|
||||||
|
headers: headers,
|
||||||
|
rawData: results,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('CSV general error:', error)
|
||||||
|
throw new Error(`Failed to process CSV file: ${(error as Error).message}`)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async parseBuffer(buffer: Buffer): Promise<FileParseResult> {
|
async parseBuffer(buffer: Buffer): Promise<FileParseResult> {
|
||||||
return new Promise((resolve, reject) => {
|
try {
|
||||||
try {
|
logger.info('Parsing buffer, size:', buffer.length)
|
||||||
logger.info('Parsing buffer, size:', buffer.length)
|
|
||||||
|
|
||||||
const results: Record<string, any>[] = []
|
const fileContent = buffer.toString('utf8')
|
||||||
const headers: string[] = []
|
|
||||||
|
|
||||||
// Create a readable stream from the buffer
|
const parseResult = Papa.parse(fileContent, PARSE_OPTIONS)
|
||||||
const bufferStream = new Readable()
|
|
||||||
bufferStream.push(buffer)
|
|
||||||
bufferStream.push(null) // Signal the end of the stream
|
|
||||||
|
|
||||||
bufferStream
|
if (parseResult.errors && parseResult.errors.length > 0) {
|
||||||
.on('error', (error: Error) => {
|
const errorMessages = parseResult.errors.map((err) => err.message).join(', ')
|
||||||
logger.error('CSV buffer stream error:', error)
|
logger.error('CSV parsing errors:', parseResult.errors)
|
||||||
reject(new Error(`Failed to read CSV buffer: ${error.message}`))
|
throw new Error(`Failed to parse CSV buffer: ${errorMessages}`)
|
||||||
})
|
|
||||||
.pipe(csvParser())
|
|
||||||
.on('headers', (headerList: string[]) => {
|
|
||||||
headers.push(...headerList)
|
|
||||||
})
|
|
||||||
.on('data', (data: Record<string, any>) => {
|
|
||||||
results.push(data)
|
|
||||||
})
|
|
||||||
.on('end', () => {
|
|
||||||
// Convert CSV data to a formatted string representation
|
|
||||||
let content = ''
|
|
||||||
|
|
||||||
// Add headers
|
|
||||||
if (headers.length > 0) {
|
|
||||||
const cleanHeaders = headers.map((h) => sanitizeTextForUTF8(String(h)))
|
|
||||||
content += `${cleanHeaders.join(', ')}\n`
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add rows
|
|
||||||
results.forEach((row) => {
|
|
||||||
const cleanValues = Object.values(row).map((v) =>
|
|
||||||
sanitizeTextForUTF8(String(v || ''))
|
|
||||||
)
|
|
||||||
content += `${cleanValues.join(', ')}\n`
|
|
||||||
})
|
|
||||||
|
|
||||||
resolve({
|
|
||||||
content: sanitizeTextForUTF8(content),
|
|
||||||
metadata: {
|
|
||||||
rowCount: results.length,
|
|
||||||
headers: headers,
|
|
||||||
rawData: results,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.on('error', (error: Error) => {
|
|
||||||
logger.error('CSV parsing error:', error)
|
|
||||||
reject(new Error(`Failed to parse CSV buffer: ${error.message}`))
|
|
||||||
})
|
|
||||||
} catch (error) {
|
|
||||||
logger.error('CSV buffer parsing error:', error)
|
|
||||||
reject(new Error(`Failed to process CSV buffer: ${(error as Error).message}`))
|
|
||||||
}
|
}
|
||||||
})
|
|
||||||
|
const results = parseResult.data as Record<string, any>[]
|
||||||
|
const headers = parseResult.meta.fields || []
|
||||||
|
|
||||||
|
let content = ''
|
||||||
|
|
||||||
|
if (headers.length > 0) {
|
||||||
|
const cleanHeaders = headers.map((h) => sanitizeTextForUTF8(String(h)))
|
||||||
|
content += `${cleanHeaders.join(', ')}\n`
|
||||||
|
}
|
||||||
|
|
||||||
|
results.forEach((row) => {
|
||||||
|
const cleanValues = Object.values(row).map((v) => sanitizeTextForUTF8(String(v || '')))
|
||||||
|
content += `${cleanValues.join(', ')}\n`
|
||||||
|
})
|
||||||
|
|
||||||
|
return {
|
||||||
|
content: sanitizeTextForUTF8(content),
|
||||||
|
metadata: {
|
||||||
|
rowCount: results.length,
|
||||||
|
headers: headers,
|
||||||
|
rawData: results,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('CSV buffer parsing error:', error)
|
||||||
|
throw new Error(`Failed to process CSV buffer: ${(error as Error).message}`)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,19 +9,16 @@ const logger = createLogger('DocParser')
|
|||||||
export class DocParser implements FileParser {
|
export class DocParser implements FileParser {
|
||||||
async parseFile(filePath: string): Promise<FileParseResult> {
|
async parseFile(filePath: string): Promise<FileParseResult> {
|
||||||
try {
|
try {
|
||||||
// Validate input
|
|
||||||
if (!filePath) {
|
if (!filePath) {
|
||||||
throw new Error('No file path provided')
|
throw new Error('No file path provided')
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if file exists
|
|
||||||
if (!existsSync(filePath)) {
|
if (!existsSync(filePath)) {
|
||||||
throw new Error(`File not found: ${filePath}`)
|
throw new Error(`File not found: ${filePath}`)
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.info(`Parsing DOC file: ${filePath}`)
|
logger.info(`Parsing DOC file: ${filePath}`)
|
||||||
|
|
||||||
// Read the file
|
|
||||||
const buffer = await readFile(filePath)
|
const buffer = await readFile(filePath)
|
||||||
return this.parseBuffer(buffer)
|
return this.parseBuffer(buffer)
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -38,45 +35,37 @@ export class DocParser implements FileParser {
|
|||||||
throw new Error('Empty buffer provided')
|
throw new Error('Empty buffer provided')
|
||||||
}
|
}
|
||||||
|
|
||||||
// Try to dynamically import the word extractor
|
let parseOfficeAsync
|
||||||
let WordExtractor
|
|
||||||
try {
|
try {
|
||||||
WordExtractor = (await import('word-extractor')).default
|
const officeParser = await import('officeparser')
|
||||||
|
parseOfficeAsync = officeParser.parseOfficeAsync
|
||||||
} catch (importError) {
|
} catch (importError) {
|
||||||
logger.warn('word-extractor not available, using fallback extraction')
|
logger.warn('officeparser not available, using fallback extraction')
|
||||||
return this.fallbackExtraction(buffer)
|
return this.fallbackExtraction(buffer)
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const extractor = new WordExtractor()
|
const result = await parseOfficeAsync(buffer)
|
||||||
const extracted = await extractor.extract(buffer)
|
|
||||||
|
|
||||||
const content = sanitizeTextForUTF8(extracted.getBody())
|
if (!result) {
|
||||||
const headers = extracted.getHeaders()
|
throw new Error('officeparser returned no result')
|
||||||
const footers = extracted.getFooters()
|
|
||||||
|
|
||||||
// Combine body with headers/footers if they exist
|
|
||||||
let fullContent = content
|
|
||||||
if (headers?.trim()) {
|
|
||||||
fullContent = `${sanitizeTextForUTF8(headers)}\n\n${fullContent}`
|
|
||||||
}
|
|
||||||
if (footers?.trim()) {
|
|
||||||
fullContent = `${fullContent}\n\n${sanitizeTextForUTF8(footers)}`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.info('DOC parsing completed successfully')
|
const resultString = typeof result === 'string' ? result : String(result)
|
||||||
|
|
||||||
|
const content = sanitizeTextForUTF8(resultString.trim())
|
||||||
|
|
||||||
|
logger.info('DOC parsing completed successfully with officeparser')
|
||||||
|
|
||||||
return {
|
return {
|
||||||
content: fullContent.trim(),
|
content: content,
|
||||||
metadata: {
|
metadata: {
|
||||||
hasHeaders: !!headers?.trim(),
|
characterCount: content.length,
|
||||||
hasFooters: !!footers?.trim(),
|
extractionMethod: 'officeparser',
|
||||||
characterCount: fullContent.length,
|
|
||||||
extractionMethod: 'word-extractor',
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
} catch (extractError) {
|
} catch (extractError) {
|
||||||
logger.warn('word-extractor failed, using fallback:', extractError)
|
logger.warn('officeparser failed, using fallback:', extractError)
|
||||||
return this.fallbackExtraction(buffer)
|
return this.fallbackExtraction(buffer)
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -85,25 +74,16 @@ export class DocParser implements FileParser {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Fallback extraction method for when word-extractor is not available
|
|
||||||
* This is a very basic extraction that looks for readable text in the binary
|
|
||||||
*/
|
|
||||||
private fallbackExtraction(buffer: Buffer): FileParseResult {
|
private fallbackExtraction(buffer: Buffer): FileParseResult {
|
||||||
logger.info('Using fallback text extraction for DOC file')
|
logger.info('Using fallback text extraction for DOC file')
|
||||||
|
|
||||||
// Convert buffer to string and try to extract readable text
|
const text = buffer.toString('utf8', 0, Math.min(buffer.length, 100000))
|
||||||
// This is very basic and won't work well for complex DOC files
|
|
||||||
const text = buffer.toString('utf8', 0, Math.min(buffer.length, 100000)) // Limit to first 100KB
|
|
||||||
|
|
||||||
// Extract sequences of printable ASCII characters
|
|
||||||
const readableText = text
|
const readableText = text
|
||||||
.match(/[\x20-\x7E\s]{4,}/g) // Find sequences of 4+ printable characters
|
.match(/[\x20-\x7E\s]{4,}/g)
|
||||||
?.filter(
|
?.filter(
|
||||||
(chunk) =>
|
(chunk) =>
|
||||||
chunk.trim().length > 10 && // Minimum length
|
chunk.trim().length > 10 && /[a-zA-Z]/.test(chunk) && !/^[\x00-\x1F]*$/.test(chunk)
|
||||||
/[a-zA-Z]/.test(chunk) && // Must contain letters
|
|
||||||
!/^[\x00-\x1F]*$/.test(chunk) // Not just control characters
|
|
||||||
)
|
)
|
||||||
.join(' ')
|
.join(' ')
|
||||||
.replace(/\s+/g, ' ')
|
.replace(/\s+/g, ' ')
|
||||||
@@ -118,8 +98,7 @@ export class DocParser implements FileParser {
|
|||||||
metadata: {
|
metadata: {
|
||||||
extractionMethod: 'fallback',
|
extractionMethod: 'fallback',
|
||||||
characterCount: content.length,
|
characterCount: content.length,
|
||||||
warning:
|
warning: 'Basic text extraction used. For better results, convert to DOCX format.',
|
||||||
'Basic text extraction used. For better results, install word-extractor package or convert to DOCX format.',
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,15 +14,12 @@ interface MammothResult {
|
|||||||
export class DocxParser implements FileParser {
|
export class DocxParser implements FileParser {
|
||||||
async parseFile(filePath: string): Promise<FileParseResult> {
|
async parseFile(filePath: string): Promise<FileParseResult> {
|
||||||
try {
|
try {
|
||||||
// Validate input
|
|
||||||
if (!filePath) {
|
if (!filePath) {
|
||||||
throw new Error('No file path provided')
|
throw new Error('No file path provided')
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read the file
|
|
||||||
const buffer = await readFile(filePath)
|
const buffer = await readFile(filePath)
|
||||||
|
|
||||||
// Use parseBuffer for consistent implementation
|
|
||||||
return this.parseBuffer(buffer)
|
return this.parseBuffer(buffer)
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error('DOCX file error:', error)
|
logger.error('DOCX file error:', error)
|
||||||
@@ -34,10 +31,8 @@ export class DocxParser implements FileParser {
|
|||||||
try {
|
try {
|
||||||
logger.info('Parsing buffer, size:', buffer.length)
|
logger.info('Parsing buffer, size:', buffer.length)
|
||||||
|
|
||||||
// Extract text with mammoth
|
|
||||||
const result = await mammoth.extractRawText({ buffer })
|
const result = await mammoth.extractRawText({ buffer })
|
||||||
|
|
||||||
// Extract HTML for metadata (optional - won't fail if this fails)
|
|
||||||
let htmlResult: MammothResult = { value: '', messages: [] }
|
let htmlResult: MammothResult = { value: '', messages: [] }
|
||||||
try {
|
try {
|
||||||
htmlResult = await mammoth.convertToHtml({ buffer })
|
htmlResult = await mammoth.convertToHtml({ buffer })
|
||||||
|
|||||||
283
apps/sim/lib/file-parsers/html-parser.ts
Normal file
283
apps/sim/lib/file-parsers/html-parser.ts
Normal file
@@ -0,0 +1,283 @@
|
|||||||
|
import { readFile } from 'fs/promises'
|
||||||
|
import * as cheerio from 'cheerio'
|
||||||
|
import type { FileParseResult, FileParser } from '@/lib/file-parsers/types'
|
||||||
|
import { sanitizeTextForUTF8 } from '@/lib/file-parsers/utils'
|
||||||
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
|
|
||||||
|
const logger = createLogger('HtmlParser')
|
||||||
|
|
||||||
|
export class HtmlParser implements FileParser {
|
||||||
|
async parseFile(filePath: string): Promise<FileParseResult> {
|
||||||
|
try {
|
||||||
|
if (!filePath) {
|
||||||
|
throw new Error('No file path provided')
|
||||||
|
}
|
||||||
|
|
||||||
|
const buffer = await readFile(filePath)
|
||||||
|
return this.parseBuffer(buffer)
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('HTML file error:', error)
|
||||||
|
throw new Error(`Failed to parse HTML file: ${(error as Error).message}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async parseBuffer(buffer: Buffer): Promise<FileParseResult> {
|
||||||
|
try {
|
||||||
|
logger.info('Parsing HTML buffer, size:', buffer.length)
|
||||||
|
|
||||||
|
const htmlContent = buffer.toString('utf-8')
|
||||||
|
const $ = cheerio.load(htmlContent)
|
||||||
|
|
||||||
|
// Extract meta information before removing tags
|
||||||
|
const title = $('title').text().trim()
|
||||||
|
const metaDescription = $('meta[name="description"]').attr('content') || ''
|
||||||
|
|
||||||
|
$('script, style, noscript, meta, link, iframe, object, embed, svg').remove()
|
||||||
|
|
||||||
|
$.root()
|
||||||
|
.contents()
|
||||||
|
.filter(function () {
|
||||||
|
return this.type === 'comment'
|
||||||
|
})
|
||||||
|
.remove()
|
||||||
|
|
||||||
|
const content = this.extractStructuredText($)
|
||||||
|
|
||||||
|
const sanitizedContent = sanitizeTextForUTF8(content)
|
||||||
|
|
||||||
|
const characterCount = sanitizedContent.length
|
||||||
|
const wordCount = sanitizedContent.split(/\s+/).filter((word) => word.length > 0).length
|
||||||
|
const estimatedTokenCount = Math.ceil(characterCount / 4)
|
||||||
|
|
||||||
|
const headings = this.extractHeadings($)
|
||||||
|
|
||||||
|
const links = this.extractLinks($)
|
||||||
|
|
||||||
|
return {
|
||||||
|
content: sanitizedContent,
|
||||||
|
metadata: {
|
||||||
|
title,
|
||||||
|
metaDescription,
|
||||||
|
characterCount,
|
||||||
|
wordCount,
|
||||||
|
tokenCount: estimatedTokenCount,
|
||||||
|
headings,
|
||||||
|
links: links.slice(0, 50),
|
||||||
|
hasImages: $('img').length > 0,
|
||||||
|
imageCount: $('img').length,
|
||||||
|
hasTable: $('table').length > 0,
|
||||||
|
tableCount: $('table').length,
|
||||||
|
hasList: $('ul, ol').length > 0,
|
||||||
|
listCount: $('ul, ol').length,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('HTML buffer parsing error:', error)
|
||||||
|
throw new Error(`Failed to parse HTML buffer: ${(error as Error).message}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extract structured text content preserving document hierarchy
|
||||||
|
*/
|
||||||
|
private extractStructuredText($: cheerio.CheerioAPI): string {
|
||||||
|
const contentParts: string[] = []
|
||||||
|
|
||||||
|
const rootElement = $('body').length > 0 ? $('body') : $.root()
|
||||||
|
|
||||||
|
this.processElement($, rootElement, contentParts, 0)
|
||||||
|
|
||||||
|
return contentParts.join('\n').trim()
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Recursively process elements to extract text with structure
|
||||||
|
*/
|
||||||
|
private processElement(
|
||||||
|
$: cheerio.CheerioAPI,
|
||||||
|
element: cheerio.Cheerio<any>,
|
||||||
|
contentParts: string[],
|
||||||
|
depth: number
|
||||||
|
): void {
|
||||||
|
element.contents().each((_, node) => {
|
||||||
|
if (node.type === 'text') {
|
||||||
|
const text = $(node).text().trim()
|
||||||
|
if (text) {
|
||||||
|
contentParts.push(text)
|
||||||
|
}
|
||||||
|
} else if (node.type === 'tag') {
|
||||||
|
const $node = $(node)
|
||||||
|
const tagName = node.tagName?.toLowerCase()
|
||||||
|
|
||||||
|
switch (tagName) {
|
||||||
|
case 'h1':
|
||||||
|
case 'h2':
|
||||||
|
case 'h3':
|
||||||
|
case 'h4':
|
||||||
|
case 'h5':
|
||||||
|
case 'h6': {
|
||||||
|
const headingText = $node.text().trim()
|
||||||
|
if (headingText) {
|
||||||
|
contentParts.push(`\n${headingText}\n`)
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
case 'p': {
|
||||||
|
const paragraphText = $node.text().trim()
|
||||||
|
if (paragraphText) {
|
||||||
|
contentParts.push(`${paragraphText}\n`)
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
case 'br':
|
||||||
|
contentParts.push('\n')
|
||||||
|
break
|
||||||
|
|
||||||
|
case 'hr':
|
||||||
|
contentParts.push('\n---\n')
|
||||||
|
break
|
||||||
|
|
||||||
|
case 'li': {
|
||||||
|
const listItemText = $node.text().trim()
|
||||||
|
if (listItemText) {
|
||||||
|
const indent = ' '.repeat(Math.min(depth, 3))
|
||||||
|
contentParts.push(`${indent}• ${listItemText}`)
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
case 'ul':
|
||||||
|
case 'ol':
|
||||||
|
contentParts.push('\n')
|
||||||
|
this.processElement($, $node, contentParts, depth + 1)
|
||||||
|
contentParts.push('\n')
|
||||||
|
break
|
||||||
|
|
||||||
|
case 'table':
|
||||||
|
this.processTable($, $node, contentParts)
|
||||||
|
break
|
||||||
|
|
||||||
|
case 'blockquote': {
|
||||||
|
const quoteText = $node.text().trim()
|
||||||
|
if (quoteText) {
|
||||||
|
contentParts.push(`\n> ${quoteText}\n`)
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
case 'pre':
|
||||||
|
case 'code': {
|
||||||
|
const codeText = $node.text().trim()
|
||||||
|
if (codeText) {
|
||||||
|
contentParts.push(`\n\`\`\`\n${codeText}\n\`\`\`\n`)
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
case 'div':
|
||||||
|
case 'section':
|
||||||
|
case 'article':
|
||||||
|
case 'main':
|
||||||
|
case 'aside':
|
||||||
|
case 'nav':
|
||||||
|
case 'header':
|
||||||
|
case 'footer':
|
||||||
|
this.processElement($, $node, contentParts, depth)
|
||||||
|
break
|
||||||
|
|
||||||
|
case 'a': {
|
||||||
|
const linkText = $node.text().trim()
|
||||||
|
const href = $node.attr('href')
|
||||||
|
if (linkText) {
|
||||||
|
if (href?.startsWith('http')) {
|
||||||
|
contentParts.push(`${linkText} (${href})`)
|
||||||
|
} else {
|
||||||
|
contentParts.push(linkText)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
case 'img': {
|
||||||
|
const alt = $node.attr('alt')
|
||||||
|
if (alt) {
|
||||||
|
contentParts.push(`[Image: ${alt}]`)
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
default:
|
||||||
|
this.processElement($, $node, contentParts, depth)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Process table elements to extract structured data
|
||||||
|
*/
|
||||||
|
private processTable(
|
||||||
|
$: cheerio.CheerioAPI,
|
||||||
|
table: cheerio.Cheerio<any>,
|
||||||
|
contentParts: string[]
|
||||||
|
): void {
|
||||||
|
contentParts.push('\n[Table]')
|
||||||
|
|
||||||
|
table.find('tr').each((_, row) => {
|
||||||
|
const $row = $(row)
|
||||||
|
const cells: string[] = []
|
||||||
|
|
||||||
|
$row.find('td, th').each((_, cell) => {
|
||||||
|
const cellText = $(cell).text().trim()
|
||||||
|
cells.push(cellText || '')
|
||||||
|
})
|
||||||
|
|
||||||
|
if (cells.length > 0) {
|
||||||
|
contentParts.push(`| ${cells.join(' | ')} |`)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
contentParts.push('[/Table]\n')
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extract heading structure for metadata
|
||||||
|
*/
|
||||||
|
private extractHeadings($: cheerio.CheerioAPI): Array<{ level: number; text: string }> {
|
||||||
|
const headings: Array<{ level: number; text: string }> = []
|
||||||
|
|
||||||
|
$('h1, h2, h3, h4, h5, h6').each((_, element) => {
|
||||||
|
const $element = $(element)
|
||||||
|
const tagName = element.tagName?.toLowerCase()
|
||||||
|
const level = Number.parseInt(tagName?.charAt(1) || '1', 10)
|
||||||
|
const text = $element.text().trim()
|
||||||
|
|
||||||
|
if (text) {
|
||||||
|
headings.push({ level, text })
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
return headings
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extract links from the document
|
||||||
|
*/
|
||||||
|
private extractLinks($: cheerio.CheerioAPI): Array<{ text: string; href: string }> {
|
||||||
|
const links: Array<{ text: string; href: string }> = []
|
||||||
|
|
||||||
|
$('a[href]').each((_, element) => {
|
||||||
|
const $element = $(element)
|
||||||
|
const href = $element.attr('href')
|
||||||
|
const text = $element.text().trim()
|
||||||
|
|
||||||
|
if (href && text && href.startsWith('http')) {
|
||||||
|
links.push({ text, href })
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
return links
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -51,6 +51,23 @@ const mockMdParseFile = vi.fn().mockResolvedValue({
|
|||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
|
const mockPptxParseFile = vi.fn().mockResolvedValue({
|
||||||
|
content: 'Parsed PPTX content',
|
||||||
|
metadata: {
|
||||||
|
slideCount: 5,
|
||||||
|
extractionMethod: 'officeparser',
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
const mockHtmlParseFile = vi.fn().mockResolvedValue({
|
||||||
|
content: 'Parsed HTML content',
|
||||||
|
metadata: {
|
||||||
|
title: 'Test HTML Document',
|
||||||
|
headingCount: 3,
|
||||||
|
linkCount: 2,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
const createMockModule = () => {
|
const createMockModule = () => {
|
||||||
const mockParsers: Record<string, FileParser> = {
|
const mockParsers: Record<string, FileParser> = {
|
||||||
pdf: { parseFile: mockPdfParseFile },
|
pdf: { parseFile: mockPdfParseFile },
|
||||||
@@ -58,6 +75,10 @@ const createMockModule = () => {
|
|||||||
docx: { parseFile: mockDocxParseFile },
|
docx: { parseFile: mockDocxParseFile },
|
||||||
txt: { parseFile: mockTxtParseFile },
|
txt: { parseFile: mockTxtParseFile },
|
||||||
md: { parseFile: mockMdParseFile },
|
md: { parseFile: mockMdParseFile },
|
||||||
|
pptx: { parseFile: mockPptxParseFile },
|
||||||
|
ppt: { parseFile: mockPptxParseFile },
|
||||||
|
html: { parseFile: mockHtmlParseFile },
|
||||||
|
htm: { parseFile: mockHtmlParseFile },
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@@ -143,6 +164,18 @@ describe('File Parsers', () => {
|
|||||||
})),
|
})),
|
||||||
}))
|
}))
|
||||||
|
|
||||||
|
vi.doMock('@/lib/file-parsers/pptx-parser', () => ({
|
||||||
|
PptxParser: vi.fn().mockImplementation(() => ({
|
||||||
|
parseFile: mockPptxParseFile,
|
||||||
|
})),
|
||||||
|
}))
|
||||||
|
|
||||||
|
vi.doMock('@/lib/file-parsers/html-parser', () => ({
|
||||||
|
HtmlParser: vi.fn().mockImplementation(() => ({
|
||||||
|
parseFile: mockHtmlParseFile,
|
||||||
|
})),
|
||||||
|
}))
|
||||||
|
|
||||||
global.console = {
|
global.console = {
|
||||||
...console,
|
...console,
|
||||||
log: vi.fn(),
|
log: vi.fn(),
|
||||||
@@ -261,6 +294,82 @@ describe('File Parsers', () => {
|
|||||||
|
|
||||||
const { parseFile } = await import('@/lib/file-parsers/index')
|
const { parseFile } = await import('@/lib/file-parsers/index')
|
||||||
const result = await parseFile('/test/files/document.md')
|
const result = await parseFile('/test/files/document.md')
|
||||||
|
|
||||||
|
expect(result).toEqual(expectedResult)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should parse PPTX files successfully', async () => {
|
||||||
|
const expectedResult = {
|
||||||
|
content: 'Parsed PPTX content',
|
||||||
|
metadata: {
|
||||||
|
slideCount: 5,
|
||||||
|
extractionMethod: 'officeparser',
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
mockPptxParseFile.mockResolvedValueOnce(expectedResult)
|
||||||
|
mockExistsSync.mockReturnValue(true)
|
||||||
|
|
||||||
|
const { parseFile } = await import('@/lib/file-parsers/index')
|
||||||
|
const result = await parseFile('/test/files/presentation.pptx')
|
||||||
|
|
||||||
|
expect(result).toEqual(expectedResult)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should parse PPT files successfully', async () => {
|
||||||
|
const expectedResult = {
|
||||||
|
content: 'Parsed PPTX content',
|
||||||
|
metadata: {
|
||||||
|
slideCount: 5,
|
||||||
|
extractionMethod: 'officeparser',
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
mockPptxParseFile.mockResolvedValueOnce(expectedResult)
|
||||||
|
mockExistsSync.mockReturnValue(true)
|
||||||
|
|
||||||
|
const { parseFile } = await import('@/lib/file-parsers/index')
|
||||||
|
const result = await parseFile('/test/files/presentation.ppt')
|
||||||
|
|
||||||
|
expect(result).toEqual(expectedResult)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should parse HTML files successfully', async () => {
|
||||||
|
const expectedResult = {
|
||||||
|
content: 'Parsed HTML content',
|
||||||
|
metadata: {
|
||||||
|
title: 'Test HTML Document',
|
||||||
|
headingCount: 3,
|
||||||
|
linkCount: 2,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
mockHtmlParseFile.mockResolvedValueOnce(expectedResult)
|
||||||
|
mockExistsSync.mockReturnValue(true)
|
||||||
|
|
||||||
|
const { parseFile } = await import('@/lib/file-parsers/index')
|
||||||
|
const result = await parseFile('/test/files/document.html')
|
||||||
|
|
||||||
|
expect(result).toEqual(expectedResult)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should parse HTM files successfully', async () => {
|
||||||
|
const expectedResult = {
|
||||||
|
content: 'Parsed HTML content',
|
||||||
|
metadata: {
|
||||||
|
title: 'Test HTML Document',
|
||||||
|
headingCount: 3,
|
||||||
|
linkCount: 2,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
mockHtmlParseFile.mockResolvedValueOnce(expectedResult)
|
||||||
|
mockExistsSync.mockReturnValue(true)
|
||||||
|
|
||||||
|
const { parseFile } = await import('@/lib/file-parsers/index')
|
||||||
|
const result = await parseFile('/test/files/document.htm')
|
||||||
|
|
||||||
|
expect(result).toEqual(expectedResult)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should throw error for unsupported file types', async () => {
|
it('should throw error for unsupported file types', async () => {
|
||||||
@@ -292,6 +401,10 @@ describe('File Parsers', () => {
|
|||||||
expect(isSupportedFileType('docx')).toBe(true)
|
expect(isSupportedFileType('docx')).toBe(true)
|
||||||
expect(isSupportedFileType('txt')).toBe(true)
|
expect(isSupportedFileType('txt')).toBe(true)
|
||||||
expect(isSupportedFileType('md')).toBe(true)
|
expect(isSupportedFileType('md')).toBe(true)
|
||||||
|
expect(isSupportedFileType('pptx')).toBe(true)
|
||||||
|
expect(isSupportedFileType('ppt')).toBe(true)
|
||||||
|
expect(isSupportedFileType('html')).toBe(true)
|
||||||
|
expect(isSupportedFileType('htm')).toBe(true)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should return false for unsupported file types', async () => {
|
it('should return false for unsupported file types', async () => {
|
||||||
@@ -308,6 +421,8 @@ describe('File Parsers', () => {
|
|||||||
expect(isSupportedFileType('CSV')).toBe(true)
|
expect(isSupportedFileType('CSV')).toBe(true)
|
||||||
expect(isSupportedFileType('TXT')).toBe(true)
|
expect(isSupportedFileType('TXT')).toBe(true)
|
||||||
expect(isSupportedFileType('MD')).toBe(true)
|
expect(isSupportedFileType('MD')).toBe(true)
|
||||||
|
expect(isSupportedFileType('PPTX')).toBe(true)
|
||||||
|
expect(isSupportedFileType('HTML')).toBe(true)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should handle errors gracefully', async () => {
|
it('should handle errors gracefully', async () => {
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import { createLogger } from '@/lib/logs/console/logger'
|
|||||||
|
|
||||||
const logger = createLogger('FileParser')
|
const logger = createLogger('FileParser')
|
||||||
|
|
||||||
// Lazy-loaded parsers to avoid initialization issues
|
|
||||||
let parserInstances: Record<string, FileParser> | null = null
|
let parserInstances: Record<string, FileParser> | null = null
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -18,25 +17,20 @@ function getParserInstances(): Record<string, FileParser> {
|
|||||||
parserInstances = {}
|
parserInstances = {}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Import parsers only when needed - with try/catch for each one
|
|
||||||
try {
|
try {
|
||||||
logger.info('Attempting to load PDF parser...')
|
logger.info('Attempting to load PDF parser...')
|
||||||
try {
|
try {
|
||||||
// First try to use the pdf-parse library
|
|
||||||
// Import the PdfParser using ES module import to avoid test file access
|
|
||||||
const { PdfParser } = require('@/lib/file-parsers/pdf-parser')
|
const { PdfParser } = require('@/lib/file-parsers/pdf-parser')
|
||||||
parserInstances.pdf = new PdfParser()
|
parserInstances.pdf = new PdfParser()
|
||||||
logger.info('PDF parser loaded successfully')
|
logger.info('PDF parser loaded successfully')
|
||||||
} catch (pdfParseError) {
|
} catch (pdfLibError) {
|
||||||
// If that fails, fallback to our raw PDF parser
|
logger.error('Failed to load primary PDF parser:', pdfLibError)
|
||||||
logger.error('Failed to load primary PDF parser:', pdfParseError)
|
|
||||||
logger.info('Falling back to raw PDF parser')
|
logger.info('Falling back to raw PDF parser')
|
||||||
parserInstances.pdf = new RawPdfParser()
|
parserInstances.pdf = new RawPdfParser()
|
||||||
logger.info('Raw PDF parser loaded successfully')
|
logger.info('Raw PDF parser loaded successfully')
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error('Failed to load any PDF parser:', error)
|
logger.error('Failed to load any PDF parser:', error)
|
||||||
// Create a simple fallback that just returns the file size and a message
|
|
||||||
parserInstances.pdf = {
|
parserInstances.pdf = {
|
||||||
async parseFile(filePath: string): Promise<FileParseResult> {
|
async parseFile(filePath: string): Promise<FileParseResult> {
|
||||||
const buffer = await readFile(filePath)
|
const buffer = await readFile(filePath)
|
||||||
@@ -100,10 +94,26 @@ function getParserInstances(): Record<string, FileParser> {
|
|||||||
try {
|
try {
|
||||||
const { XlsxParser } = require('@/lib/file-parsers/xlsx-parser')
|
const { XlsxParser } = require('@/lib/file-parsers/xlsx-parser')
|
||||||
parserInstances.xlsx = new XlsxParser()
|
parserInstances.xlsx = new XlsxParser()
|
||||||
parserInstances.xls = new XlsxParser() // Both xls and xlsx use the same parser
|
parserInstances.xls = new XlsxParser()
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error('Failed to load XLSX parser:', error)
|
logger.error('Failed to load XLSX parser:', error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const { PptxParser } = require('@/lib/file-parsers/pptx-parser')
|
||||||
|
parserInstances.pptx = new PptxParser()
|
||||||
|
parserInstances.ppt = new PptxParser()
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Failed to load PPTX parser:', error)
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const { HtmlParser } = require('@/lib/file-parsers/html-parser')
|
||||||
|
parserInstances.html = new HtmlParser()
|
||||||
|
parserInstances.htm = new HtmlParser()
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Failed to load HTML parser:', error)
|
||||||
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error('Error loading file parsers:', error)
|
logger.error('Error loading file parsers:', error)
|
||||||
}
|
}
|
||||||
@@ -119,12 +129,10 @@ function getParserInstances(): Record<string, FileParser> {
|
|||||||
*/
|
*/
|
||||||
export async function parseFile(filePath: string): Promise<FileParseResult> {
|
export async function parseFile(filePath: string): Promise<FileParseResult> {
|
||||||
try {
|
try {
|
||||||
// Validate input
|
|
||||||
if (!filePath) {
|
if (!filePath) {
|
||||||
throw new Error('No file path provided')
|
throw new Error('No file path provided')
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if file exists
|
|
||||||
if (!existsSync(filePath)) {
|
if (!existsSync(filePath)) {
|
||||||
throw new Error(`File not found: ${filePath}`)
|
throw new Error(`File not found: ${filePath}`)
|
||||||
}
|
}
|
||||||
@@ -158,7 +166,6 @@ export async function parseFile(filePath: string): Promise<FileParseResult> {
|
|||||||
*/
|
*/
|
||||||
export async function parseBuffer(buffer: Buffer, extension: string): Promise<FileParseResult> {
|
export async function parseBuffer(buffer: Buffer, extension: string): Promise<FileParseResult> {
|
||||||
try {
|
try {
|
||||||
// Validate input
|
|
||||||
if (!buffer || buffer.length === 0) {
|
if (!buffer || buffer.length === 0) {
|
||||||
throw new Error('Empty buffer provided')
|
throw new Error('Empty buffer provided')
|
||||||
}
|
}
|
||||||
@@ -182,7 +189,6 @@ export async function parseBuffer(buffer: Buffer, extension: string): Promise<Fi
|
|||||||
logger.info('Using parser for extension:', normalizedExtension)
|
logger.info('Using parser for extension:', normalizedExtension)
|
||||||
const parser = parsers[normalizedExtension]
|
const parser = parsers[normalizedExtension]
|
||||||
|
|
||||||
// Check if parser supports buffer parsing
|
|
||||||
if (parser.parseBuffer) {
|
if (parser.parseBuffer) {
|
||||||
return await parser.parseBuffer(buffer)
|
return await parser.parseBuffer(buffer)
|
||||||
}
|
}
|
||||||
@@ -207,5 +213,4 @@ export function isSupportedFileType(extension: string): extension is SupportedFi
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Type exports
|
|
||||||
export type { FileParseResult, FileParser, SupportedFileType }
|
export type { FileParseResult, FileParser, SupportedFileType }
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import { readFile } from 'fs/promises'
|
import { readFile } from 'fs/promises'
|
||||||
import type { FileParseResult, FileParser } from '@/lib/file-parsers/types'
|
import type { FileParseResult, FileParser } from '@/lib/file-parsers/types'
|
||||||
|
import { sanitizeTextForUTF8 } from '@/lib/file-parsers/utils'
|
||||||
import { createLogger } from '@/lib/logs/console/logger'
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
|
|
||||||
const logger = createLogger('MdParser')
|
const logger = createLogger('MdParser')
|
||||||
@@ -7,15 +8,12 @@ const logger = createLogger('MdParser')
|
|||||||
export class MdParser implements FileParser {
|
export class MdParser implements FileParser {
|
||||||
async parseFile(filePath: string): Promise<FileParseResult> {
|
async parseFile(filePath: string): Promise<FileParseResult> {
|
||||||
try {
|
try {
|
||||||
// Validate input
|
|
||||||
if (!filePath) {
|
if (!filePath) {
|
||||||
throw new Error('No file path provided')
|
throw new Error('No file path provided')
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read the file
|
|
||||||
const buffer = await readFile(filePath)
|
const buffer = await readFile(filePath)
|
||||||
|
|
||||||
// Use parseBuffer for consistent implementation
|
|
||||||
return this.parseBuffer(buffer)
|
return this.parseBuffer(buffer)
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error('MD file error:', error)
|
logger.error('MD file error:', error)
|
||||||
@@ -27,14 +25,14 @@ export class MdParser implements FileParser {
|
|||||||
try {
|
try {
|
||||||
logger.info('Parsing buffer, size:', buffer.length)
|
logger.info('Parsing buffer, size:', buffer.length)
|
||||||
|
|
||||||
// Extract content
|
|
||||||
const result = buffer.toString('utf-8')
|
const result = buffer.toString('utf-8')
|
||||||
|
const content = sanitizeTextForUTF8(result)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
content: result,
|
content,
|
||||||
metadata: {
|
metadata: {
|
||||||
characterCount: result.length,
|
characterCount: content.length,
|
||||||
tokenCount: result.length / 4,
|
tokenCount: Math.floor(content.length / 4),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|||||||
@@ -1,22 +1,21 @@
|
|||||||
import { readFile } from 'fs/promises'
|
import { readFile } from 'fs/promises'
|
||||||
// @ts-ignore
|
import { PDFDocument } from 'pdf-lib'
|
||||||
import * as pdfParseLib from 'pdf-parse/lib/pdf-parse.js'
|
|
||||||
import type { FileParseResult, FileParser } from '@/lib/file-parsers/types'
|
import type { FileParseResult, FileParser } from '@/lib/file-parsers/types'
|
||||||
import { createLogger } from '@/lib/logs/console/logger'
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
|
import { RawPdfParser } from './raw-pdf-parser'
|
||||||
|
|
||||||
const logger = createLogger('PdfParser')
|
const logger = createLogger('PdfParser')
|
||||||
|
const rawPdfParser = new RawPdfParser()
|
||||||
|
|
||||||
export class PdfParser implements FileParser {
|
export class PdfParser implements FileParser {
|
||||||
async parseFile(filePath: string): Promise<FileParseResult> {
|
async parseFile(filePath: string): Promise<FileParseResult> {
|
||||||
try {
|
try {
|
||||||
logger.info('Starting to parse file:', filePath)
|
logger.info('Starting to parse file:', filePath)
|
||||||
|
|
||||||
// Make sure we're only parsing the provided file path
|
|
||||||
if (!filePath) {
|
if (!filePath) {
|
||||||
throw new Error('No file path provided')
|
throw new Error('No file path provided')
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read the file
|
|
||||||
logger.info('Reading file...')
|
logger.info('Reading file...')
|
||||||
const dataBuffer = await readFile(filePath)
|
const dataBuffer = await readFile(filePath)
|
||||||
logger.info('File read successfully, size:', dataBuffer.length)
|
logger.info('File read successfully, size:', dataBuffer.length)
|
||||||
@@ -32,93 +31,66 @@ export class PdfParser implements FileParser {
|
|||||||
try {
|
try {
|
||||||
logger.info('Starting to parse buffer, size:', dataBuffer.length)
|
logger.info('Starting to parse buffer, size:', dataBuffer.length)
|
||||||
|
|
||||||
// Try to parse with pdf-parse library first
|
|
||||||
try {
|
try {
|
||||||
logger.info('Attempting to parse with pdf-parse library...')
|
logger.info('Attempting to parse with pdf-lib library...')
|
||||||
|
|
||||||
// Parse PDF with direct function call to avoid test file access
|
|
||||||
logger.info('Starting PDF parsing...')
|
logger.info('Starting PDF parsing...')
|
||||||
const data = await pdfParseLib.default(dataBuffer)
|
const pdfDoc = await PDFDocument.load(dataBuffer)
|
||||||
logger.info('PDF parsed successfully with pdf-parse, pages:', data.numpages)
|
const pages = pdfDoc.getPages()
|
||||||
|
const pageCount = pages.length
|
||||||
|
|
||||||
|
logger.info('PDF parsed successfully with pdf-lib, pages:', pageCount)
|
||||||
|
|
||||||
|
const metadata: Record<string, any> = {
|
||||||
|
pageCount,
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const title = pdfDoc.getTitle()
|
||||||
|
const author = pdfDoc.getAuthor()
|
||||||
|
const subject = pdfDoc.getSubject()
|
||||||
|
const creator = pdfDoc.getCreator()
|
||||||
|
const producer = pdfDoc.getProducer()
|
||||||
|
const creationDate = pdfDoc.getCreationDate()
|
||||||
|
const modificationDate = pdfDoc.getModificationDate()
|
||||||
|
|
||||||
|
if (title) metadata.title = title
|
||||||
|
if (author) metadata.author = author
|
||||||
|
if (subject) metadata.subject = subject
|
||||||
|
if (creator) metadata.creator = creator
|
||||||
|
if (producer) metadata.producer = producer
|
||||||
|
if (creationDate) metadata.creationDate = creationDate.toISOString()
|
||||||
|
if (modificationDate) metadata.modificationDate = modificationDate.toISOString()
|
||||||
|
} catch (metadataError) {
|
||||||
|
logger.warn('Could not extract PDF metadata:', metadataError)
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
'pdf-lib loaded successfully, but text extraction requires fallback to raw parser'
|
||||||
|
)
|
||||||
|
const rawResult = await rawPdfParser.parseBuffer(dataBuffer)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
content: data.text,
|
content: rawResult.content,
|
||||||
metadata: {
|
metadata: {
|
||||||
pageCount: data.numpages,
|
...rawResult.metadata,
|
||||||
info: data.info,
|
...metadata,
|
||||||
version: data.version,
|
source: 'pdf-lib + raw-parser',
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
} catch (pdfParseError: unknown) {
|
} catch (pdfLibError: unknown) {
|
||||||
logger.error('PDF-parse library failed:', pdfParseError)
|
logger.error('PDF-lib library failed:', pdfLibError)
|
||||||
|
|
||||||
// Fallback to manual text extraction
|
logger.info('Falling back to raw PDF parser...')
|
||||||
logger.info('Falling back to manual text extraction...')
|
const rawResult = await rawPdfParser.parseBuffer(dataBuffer)
|
||||||
|
|
||||||
// Extract basic PDF info from raw content
|
|
||||||
const rawContent = dataBuffer.toString('utf-8', 0, Math.min(10000, dataBuffer.length))
|
|
||||||
|
|
||||||
let version = 'Unknown'
|
|
||||||
let pageCount = 0
|
|
||||||
|
|
||||||
// Try to extract PDF version
|
|
||||||
const versionMatch = rawContent.match(/%PDF-(\d+\.\d+)/)
|
|
||||||
if (versionMatch?.[1]) {
|
|
||||||
version = versionMatch[1]
|
|
||||||
}
|
|
||||||
|
|
||||||
// Try to get page count
|
|
||||||
const pageMatches = rawContent.match(/\/Type\s*\/Page\b/g)
|
|
||||||
if (pageMatches) {
|
|
||||||
pageCount = pageMatches.length
|
|
||||||
}
|
|
||||||
|
|
||||||
// Try to extract text by looking for text-related operators in the PDF
|
|
||||||
let extractedText = ''
|
|
||||||
|
|
||||||
// Look for text in the PDF content using common patterns
|
|
||||||
const textMatches = rawContent.match(/BT[\s\S]*?ET/g)
|
|
||||||
if (textMatches && textMatches.length > 0) {
|
|
||||||
extractedText = textMatches
|
|
||||||
.map((textBlock) => {
|
|
||||||
// Extract text objects (Tj, TJ) from the text block
|
|
||||||
const textObjects = textBlock.match(/\([^)]*\)\s*Tj|\[[^\]]*\]\s*TJ/g)
|
|
||||||
if (textObjects) {
|
|
||||||
return textObjects
|
|
||||||
.map((obj) => {
|
|
||||||
// Clean up text objects
|
|
||||||
return (
|
|
||||||
obj
|
|
||||||
.replace(
|
|
||||||
/\(([^)]*)\)\s*Tj|\[([^\]]*)\]\s*TJ/g,
|
|
||||||
(match, p1, p2) => p1 || p2 || ''
|
|
||||||
)
|
|
||||||
// Clean up PDF escape sequences
|
|
||||||
.replace(/\\(\d{3}|[()\\])/g, '')
|
|
||||||
.replace(/\\\\/g, '\\')
|
|
||||||
.replace(/\\\(/g, '(')
|
|
||||||
.replace(/\\\)/g, ')')
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.join(' ')
|
|
||||||
}
|
|
||||||
return ''
|
|
||||||
})
|
|
||||||
.join('\n')
|
|
||||||
}
|
|
||||||
|
|
||||||
// If we couldn't extract text or the text is too short, return a fallback message
|
|
||||||
if (!extractedText || extractedText.length < 50) {
|
|
||||||
extractedText = `This PDF contains ${pageCount} page(s) but text extraction was not successful.`
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
content: extractedText,
|
...rawResult,
|
||||||
metadata: {
|
metadata: {
|
||||||
pageCount,
|
...rawResult.metadata,
|
||||||
version,
|
|
||||||
fallback: true,
|
fallback: true,
|
||||||
error: (pdfParseError as Error).message || 'Unknown error',
|
source: 'raw-parser-only',
|
||||||
|
error: (pdfLibError as Error).message || 'Unknown error',
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
106
apps/sim/lib/file-parsers/pptx-parser.ts
Normal file
106
apps/sim/lib/file-parsers/pptx-parser.ts
Normal file
@@ -0,0 +1,106 @@
|
|||||||
|
import { existsSync } from 'fs'
|
||||||
|
import { readFile } from 'fs/promises'
|
||||||
|
import type { FileParseResult, FileParser } from '@/lib/file-parsers/types'
|
||||||
|
import { sanitizeTextForUTF8 } from '@/lib/file-parsers/utils'
|
||||||
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
|
|
||||||
|
const logger = createLogger('PptxParser')
|
||||||
|
|
||||||
|
export class PptxParser implements FileParser {
|
||||||
|
async parseFile(filePath: string): Promise<FileParseResult> {
|
||||||
|
try {
|
||||||
|
if (!filePath) {
|
||||||
|
throw new Error('No file path provided')
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!existsSync(filePath)) {
|
||||||
|
throw new Error(`File not found: ${filePath}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(`Parsing PowerPoint file: ${filePath}`)
|
||||||
|
|
||||||
|
const buffer = await readFile(filePath)
|
||||||
|
return this.parseBuffer(buffer)
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('PowerPoint file parsing error:', error)
|
||||||
|
throw new Error(`Failed to parse PowerPoint file: ${(error as Error).message}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async parseBuffer(buffer: Buffer): Promise<FileParseResult> {
|
||||||
|
try {
|
||||||
|
logger.info('Parsing PowerPoint buffer, size:', buffer.length)
|
||||||
|
|
||||||
|
if (!buffer || buffer.length === 0) {
|
||||||
|
throw new Error('Empty buffer provided')
|
||||||
|
}
|
||||||
|
|
||||||
|
let parseOfficeAsync
|
||||||
|
try {
|
||||||
|
const officeParser = await import('officeparser')
|
||||||
|
parseOfficeAsync = officeParser.parseOfficeAsync
|
||||||
|
} catch (importError) {
|
||||||
|
logger.warn('officeparser not available, using fallback extraction')
|
||||||
|
return this.fallbackExtraction(buffer)
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await parseOfficeAsync(buffer)
|
||||||
|
|
||||||
|
if (!result || typeof result !== 'string') {
|
||||||
|
throw new Error('officeparser returned invalid result')
|
||||||
|
}
|
||||||
|
|
||||||
|
const content = sanitizeTextForUTF8(result.trim())
|
||||||
|
|
||||||
|
logger.info('PowerPoint parsing completed successfully with officeparser')
|
||||||
|
|
||||||
|
return {
|
||||||
|
content: content,
|
||||||
|
metadata: {
|
||||||
|
characterCount: content.length,
|
||||||
|
extractionMethod: 'officeparser',
|
||||||
|
},
|
||||||
|
}
|
||||||
|
} catch (extractError) {
|
||||||
|
logger.warn('officeparser failed, using fallback:', extractError)
|
||||||
|
return this.fallbackExtraction(buffer)
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('PowerPoint buffer parsing error:', error)
|
||||||
|
throw new Error(`Failed to parse PowerPoint buffer: ${(error as Error).message}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fallbackExtraction(buffer: Buffer): FileParseResult {
|
||||||
|
logger.info('Using fallback text extraction for PowerPoint file')
|
||||||
|
|
||||||
|
const text = buffer.toString('utf8', 0, Math.min(buffer.length, 200000))
|
||||||
|
|
||||||
|
const readableText = text
|
||||||
|
.match(/[\x20-\x7E\s]{4,}/g)
|
||||||
|
?.filter(
|
||||||
|
(chunk) =>
|
||||||
|
chunk.trim().length > 10 &&
|
||||||
|
/[a-zA-Z]/.test(chunk) &&
|
||||||
|
!/^[\x00-\x1F]*$/.test(chunk) &&
|
||||||
|
!/^[^\w\s]*$/.test(chunk)
|
||||||
|
)
|
||||||
|
.join(' ')
|
||||||
|
.replace(/\s+/g, ' ')
|
||||||
|
.trim()
|
||||||
|
|
||||||
|
const content = readableText
|
||||||
|
? sanitizeTextForUTF8(readableText)
|
||||||
|
: 'Unable to extract text from PowerPoint file. Please ensure the file contains readable text content.'
|
||||||
|
|
||||||
|
return {
|
||||||
|
content,
|
||||||
|
metadata: {
|
||||||
|
extractionMethod: 'fallback',
|
||||||
|
characterCount: content.length,
|
||||||
|
warning: 'Basic text extraction used',
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -6,14 +6,9 @@ import { createLogger } from '@/lib/logs/console/logger'
|
|||||||
|
|
||||||
const logger = createLogger('RawPdfParser')
|
const logger = createLogger('RawPdfParser')
|
||||||
|
|
||||||
// Promisify zlib functions
|
|
||||||
const inflateAsync = promisify(zlib.inflate)
|
const inflateAsync = promisify(zlib.inflate)
|
||||||
const unzipAsync = promisify(zlib.unzip)
|
const unzipAsync = promisify(zlib.unzip)
|
||||||
|
|
||||||
/**
|
|
||||||
* A simple PDF parser that extracts readable text from a PDF file.
|
|
||||||
* This is used as a fallback when the pdf-parse library fails.
|
|
||||||
*/
|
|
||||||
export class RawPdfParser implements FileParser {
|
export class RawPdfParser implements FileParser {
|
||||||
async parseFile(filePath: string): Promise<FileParseResult> {
|
async parseFile(filePath: string): Promise<FileParseResult> {
|
||||||
try {
|
try {
|
||||||
@@ -23,7 +18,6 @@ export class RawPdfParser implements FileParser {
|
|||||||
throw new Error('No file path provided')
|
throw new Error('No file path provided')
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read the file
|
|
||||||
logger.info('Reading file...')
|
logger.info('Reading file...')
|
||||||
const dataBuffer = await readFile(filePath)
|
const dataBuffer = await readFile(filePath)
|
||||||
logger.info('File read successfully, size:', dataBuffer.length)
|
logger.info('File read successfully, size:', dataBuffer.length)
|
||||||
@@ -46,31 +40,22 @@ export class RawPdfParser implements FileParser {
|
|||||||
try {
|
try {
|
||||||
logger.info('Starting to parse buffer, size:', dataBuffer.length)
|
logger.info('Starting to parse buffer, size:', dataBuffer.length)
|
||||||
|
|
||||||
// Instead of trying to parse the binary PDF data directly,
|
|
||||||
// we'll extract only the text sections that are readable
|
|
||||||
|
|
||||||
// First convert to string but only for pattern matching, not for display
|
|
||||||
const rawContent = dataBuffer.toString('utf-8')
|
const rawContent = dataBuffer.toString('utf-8')
|
||||||
|
|
||||||
// Extract basic PDF info
|
|
||||||
let version = 'Unknown'
|
let version = 'Unknown'
|
||||||
let pageCount = 0
|
let pageCount = 0
|
||||||
|
|
||||||
// Try to extract PDF version
|
|
||||||
const versionMatch = rawContent.match(/%PDF-(\d+\.\d+)/)
|
const versionMatch = rawContent.match(/%PDF-(\d+\.\d+)/)
|
||||||
if (versionMatch?.[1]) {
|
if (versionMatch?.[1]) {
|
||||||
version = versionMatch[1]
|
version = versionMatch[1]
|
||||||
}
|
}
|
||||||
|
|
||||||
// Count pages using multiple methods for redundancy
|
|
||||||
// Method 1: Count "/Type /Page" occurrences (most reliable)
|
|
||||||
const typePageMatches = rawContent.match(/\/Type\s*\/Page\b/gi)
|
const typePageMatches = rawContent.match(/\/Type\s*\/Page\b/gi)
|
||||||
if (typePageMatches) {
|
if (typePageMatches) {
|
||||||
pageCount = typePageMatches.length
|
pageCount = typePageMatches.length
|
||||||
logger.info('Found page count using /Type /Page:', pageCount)
|
logger.info('Found page count using /Type /Page:', pageCount)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Method 2: Look for "/Page" dictionary references
|
|
||||||
if (pageCount === 0) {
|
if (pageCount === 0) {
|
||||||
const pageMatches = rawContent.match(/\/Page\s*\//gi)
|
const pageMatches = rawContent.match(/\/Page\s*\//gi)
|
||||||
if (pageMatches) {
|
if (pageMatches) {
|
||||||
@@ -79,19 +64,15 @@ export class RawPdfParser implements FileParser {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Method 3: Look for "/Pages" object references
|
|
||||||
if (pageCount === 0) {
|
if (pageCount === 0) {
|
||||||
const pagesObjMatches = rawContent.match(/\/Pages\s+\d+\s+\d+\s+R/gi)
|
const pagesObjMatches = rawContent.match(/\/Pages\s+\d+\s+\d+\s+R/gi)
|
||||||
if (pagesObjMatches && pagesObjMatches.length > 0) {
|
if (pagesObjMatches && pagesObjMatches.length > 0) {
|
||||||
// Extract the object reference
|
|
||||||
const pagesObjRef = pagesObjMatches[0].match(/\/Pages\s+(\d+)\s+\d+\s+R/i)
|
const pagesObjRef = pagesObjMatches[0].match(/\/Pages\s+(\d+)\s+\d+\s+R/i)
|
||||||
if (pagesObjRef?.[1]) {
|
if (pagesObjRef?.[1]) {
|
||||||
const objNum = pagesObjRef[1]
|
const objNum = pagesObjRef[1]
|
||||||
// Find the referenced object
|
|
||||||
const objRegex = new RegExp(`${objNum}\\s+0\\s+obj[\\s\\S]*?endobj`, 'i')
|
const objRegex = new RegExp(`${objNum}\\s+0\\s+obj[\\s\\S]*?endobj`, 'i')
|
||||||
const objMatch = rawContent.match(objRegex)
|
const objMatch = rawContent.match(objRegex)
|
||||||
if (objMatch) {
|
if (objMatch) {
|
||||||
// Look for /Count within the Pages object
|
|
||||||
const countMatch = objMatch[0].match(/\/Count\s+(\d+)/i)
|
const countMatch = objMatch[0].match(/\/Count\s+(\d+)/i)
|
||||||
if (countMatch?.[1]) {
|
if (countMatch?.[1]) {
|
||||||
pageCount = Number.parseInt(countMatch[1], 10)
|
pageCount = Number.parseInt(countMatch[1], 10)
|
||||||
@@ -102,50 +83,40 @@ export class RawPdfParser implements FileParser {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Method 4: Count trailer references to get an approximate count
|
|
||||||
if (pageCount === 0) {
|
if (pageCount === 0) {
|
||||||
const trailerMatches = rawContent.match(/trailer/gi)
|
const trailerMatches = rawContent.match(/trailer/gi)
|
||||||
if (trailerMatches) {
|
if (trailerMatches) {
|
||||||
// This is just a rough estimate, not accurate
|
|
||||||
pageCount = Math.max(1, Math.ceil(trailerMatches.length / 2))
|
pageCount = Math.max(1, Math.ceil(trailerMatches.length / 2))
|
||||||
logger.info('Estimated page count using trailer references:', pageCount)
|
logger.info('Estimated page count using trailer references:', pageCount)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Default to at least 1 page if we couldn't find any
|
|
||||||
if (pageCount === 0) {
|
if (pageCount === 0) {
|
||||||
pageCount = 1
|
pageCount = 1
|
||||||
logger.info('Defaulting to 1 page as no count was found')
|
logger.info('Defaulting to 1 page as no count was found')
|
||||||
}
|
}
|
||||||
|
|
||||||
// Extract text content using text markers commonly found in PDFs
|
|
||||||
let extractedText = ''
|
let extractedText = ''
|
||||||
|
|
||||||
// Method 1: Extract text between BT (Begin Text) and ET (End Text) markers
|
|
||||||
const textMatches = rawContent.match(/BT[\s\S]*?ET/g)
|
const textMatches = rawContent.match(/BT[\s\S]*?ET/g)
|
||||||
if (textMatches && textMatches.length > 0) {
|
if (textMatches && textMatches.length > 0) {
|
||||||
logger.info('Found', textMatches.length, 'text blocks')
|
logger.info('Found', textMatches.length, 'text blocks')
|
||||||
|
|
||||||
extractedText = textMatches
|
extractedText = textMatches
|
||||||
.map((textBlock) => {
|
.map((textBlock) => {
|
||||||
// Extract text objects (Tj, TJ) from the text block
|
|
||||||
const textObjects = textBlock.match(/(\([^)]*\)|\[[^\]]*\])\s*(Tj|TJ)/g)
|
const textObjects = textBlock.match(/(\([^)]*\)|\[[^\]]*\])\s*(Tj|TJ)/g)
|
||||||
if (textObjects && textObjects.length > 0) {
|
if (textObjects && textObjects.length > 0) {
|
||||||
return textObjects
|
return textObjects
|
||||||
.map((obj) => {
|
.map((obj) => {
|
||||||
// Clean up text objects
|
|
||||||
let text = ''
|
let text = ''
|
||||||
if (obj.includes('Tj')) {
|
if (obj.includes('Tj')) {
|
||||||
// Handle Tj operator (simple string)
|
|
||||||
const match = obj.match(/\(([^)]*)\)\s*Tj/)
|
const match = obj.match(/\(([^)]*)\)\s*Tj/)
|
||||||
if (match?.[1]) {
|
if (match?.[1]) {
|
||||||
text = match[1]
|
text = match[1]
|
||||||
}
|
}
|
||||||
} else if (obj.includes('TJ')) {
|
} else if (obj.includes('TJ')) {
|
||||||
// Handle TJ operator (array of strings and positioning)
|
|
||||||
const match = obj.match(/\[(.*)\]\s*TJ/)
|
const match = obj.match(/\[(.*)\]\s*TJ/)
|
||||||
if (match?.[1]) {
|
if (match?.[1]) {
|
||||||
// Extract only the string parts from the array
|
|
||||||
const parts = match[1].match(/\([^)]*\)/g)
|
const parts = match[1].match(/\([^)]*\)/g)
|
||||||
if (parts) {
|
if (parts) {
|
||||||
text = parts.map((p) => p.slice(1, -1)).join(' ')
|
text = parts.map((p) => p.slice(1, -1)).join(' ')
|
||||||
@@ -153,7 +124,6 @@ export class RawPdfParser implements FileParser {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Clean up PDF escape sequences
|
|
||||||
return text
|
return text
|
||||||
.replace(/\\(\d{3})/g, (_, octal) =>
|
.replace(/\\(\d{3})/g, (_, octal) =>
|
||||||
String.fromCharCode(Number.parseInt(octal, 8))
|
String.fromCharCode(Number.parseInt(octal, 8))
|
||||||
@@ -170,50 +140,42 @@ export class RawPdfParser implements FileParser {
|
|||||||
.trim()
|
.trim()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Try to extract metadata from XML
|
|
||||||
let metadataText = ''
|
let metadataText = ''
|
||||||
const xmlMatch = rawContent.match(/<x:xmpmeta[\s\S]*?<\/x:xmpmeta>/)
|
const xmlMatch = rawContent.match(/<x:xmpmeta[\s\S]*?<\/x:xmpmeta>/)
|
||||||
if (xmlMatch) {
|
if (xmlMatch) {
|
||||||
const xmlContent = xmlMatch[0]
|
const xmlContent = xmlMatch[0]
|
||||||
logger.info('Found XML metadata')
|
logger.info('Found XML metadata')
|
||||||
|
|
||||||
// Extract document title
|
|
||||||
const titleMatch = xmlContent.match(/<dc:title>[\s\S]*?<rdf:li[^>]*>(.*?)<\/rdf:li>/i)
|
const titleMatch = xmlContent.match(/<dc:title>[\s\S]*?<rdf:li[^>]*>(.*?)<\/rdf:li>/i)
|
||||||
if (titleMatch?.[1]) {
|
if (titleMatch?.[1]) {
|
||||||
const title = titleMatch[1].replace(/<[^>]+>/g, '').trim()
|
const title = titleMatch[1].replace(/<[^>]+>/g, '').trim()
|
||||||
metadataText += `Document Title: ${title}\n\n`
|
metadataText += `Document Title: ${title}\n\n`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Extract creator/author
|
|
||||||
const creatorMatch = xmlContent.match(/<dc:creator>[\s\S]*?<rdf:li[^>]*>(.*?)<\/rdf:li>/i)
|
const creatorMatch = xmlContent.match(/<dc:creator>[\s\S]*?<rdf:li[^>]*>(.*?)<\/rdf:li>/i)
|
||||||
if (creatorMatch?.[1]) {
|
if (creatorMatch?.[1]) {
|
||||||
const creator = creatorMatch[1].replace(/<[^>]+>/g, '').trim()
|
const creator = creatorMatch[1].replace(/<[^>]+>/g, '').trim()
|
||||||
metadataText += `Author: ${creator}\n`
|
metadataText += `Author: ${creator}\n`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Extract creation date
|
|
||||||
const dateMatch = xmlContent.match(/<xmp:CreateDate>(.*?)<\/xmp:CreateDate>/i)
|
const dateMatch = xmlContent.match(/<xmp:CreateDate>(.*?)<\/xmp:CreateDate>/i)
|
||||||
if (dateMatch?.[1]) {
|
if (dateMatch?.[1]) {
|
||||||
metadataText += `Created: ${dateMatch[1].trim()}\n`
|
metadataText += `Created: ${dateMatch[1].trim()}\n`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Extract producer
|
|
||||||
const producerMatch = xmlContent.match(/<pdf:Producer>(.*?)<\/pdf:Producer>/i)
|
const producerMatch = xmlContent.match(/<pdf:Producer>(.*?)<\/pdf:Producer>/i)
|
||||||
if (producerMatch?.[1]) {
|
if (producerMatch?.[1]) {
|
||||||
metadataText += `Producer: ${producerMatch[1].trim()}\n`
|
metadataText += `Producer: ${producerMatch[1].trim()}\n`
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Try to extract actual text content from content streams
|
|
||||||
if (!extractedText || extractedText.length < 100 || extractedText.includes('/Type /Page')) {
|
if (!extractedText || extractedText.length < 100 || extractedText.includes('/Type /Page')) {
|
||||||
logger.info('Trying advanced text extraction from content streams')
|
logger.info('Trying advanced text extraction from content streams')
|
||||||
|
|
||||||
// Find content stream references
|
|
||||||
const contentRefs = rawContent.match(/\/Contents\s+\[?\s*(\d+)\s+\d+\s+R\s*\]?/g)
|
const contentRefs = rawContent.match(/\/Contents\s+\[?\s*(\d+)\s+\d+\s+R\s*\]?/g)
|
||||||
if (contentRefs && contentRefs.length > 0) {
|
if (contentRefs && contentRefs.length > 0) {
|
||||||
logger.info('Found', contentRefs.length, 'content stream references')
|
logger.info('Found', contentRefs.length, 'content stream references')
|
||||||
|
|
||||||
// Extract object numbers from content references
|
|
||||||
const objNumbers = contentRefs
|
const objNumbers = contentRefs
|
||||||
.map((ref) => {
|
.map((ref) => {
|
||||||
const match = ref.match(/\/Contents\s+\[?\s*(\d+)\s+\d+\s+R\s*\]?/)
|
const match = ref.match(/\/Contents\s+\[?\s*(\d+)\s+\d+\s+R\s*\]?/)
|
||||||
@@ -223,7 +185,6 @@ export class RawPdfParser implements FileParser {
|
|||||||
|
|
||||||
logger.info('Content stream object numbers:', objNumbers)
|
logger.info('Content stream object numbers:', objNumbers)
|
||||||
|
|
||||||
// Try to find those objects in the content
|
|
||||||
if (objNumbers.length > 0) {
|
if (objNumbers.length > 0) {
|
||||||
let textFromStreams = ''
|
let textFromStreams = ''
|
||||||
|
|
||||||
@@ -232,12 +193,10 @@ export class RawPdfParser implements FileParser {
|
|||||||
const objMatch = rawContent.match(objRegex)
|
const objMatch = rawContent.match(objRegex)
|
||||||
|
|
||||||
if (objMatch) {
|
if (objMatch) {
|
||||||
// Look for stream content within the object
|
|
||||||
const streamMatch = objMatch[0].match(/stream\r?\n([\s\S]*?)\r?\nendstream/)
|
const streamMatch = objMatch[0].match(/stream\r?\n([\s\S]*?)\r?\nendstream/)
|
||||||
if (streamMatch?.[1]) {
|
if (streamMatch?.[1]) {
|
||||||
const streamContent = streamMatch[1]
|
const streamContent = streamMatch[1]
|
||||||
|
|
||||||
// Look for text operations in the stream (Tj, TJ, etc.)
|
|
||||||
const textFragments = streamContent.match(/\([^)]+\)\s*Tj|\[[^\]]*\]\s*TJ/g)
|
const textFragments = streamContent.match(/\([^)]+\)\s*Tj|\[[^\]]*\]\s*TJ/g)
|
||||||
if (textFragments && textFragments.length > 0) {
|
if (textFragments && textFragments.length > 0) {
|
||||||
const extractedFragments = textFragments
|
const extractedFragments = textFragments
|
||||||
@@ -290,35 +249,27 @@ export class RawPdfParser implements FileParser {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Try to decompress PDF streams
|
|
||||||
// This is especially helpful for PDFs with compressed content
|
|
||||||
if (!extractedText || extractedText.length < 100) {
|
if (!extractedText || extractedText.length < 100) {
|
||||||
logger.info('Trying to decompress PDF streams')
|
logger.info('Trying to decompress PDF streams')
|
||||||
|
|
||||||
// Find compressed streams (FlateDecode)
|
|
||||||
const compressedStreams = rawContent.match(
|
const compressedStreams = rawContent.match(
|
||||||
/\/Filter\s*\/FlateDecode[\s\S]*?stream[\s\S]*?endstream/g
|
/\/Filter\s*\/FlateDecode[\s\S]*?stream[\s\S]*?endstream/g
|
||||||
)
|
)
|
||||||
if (compressedStreams && compressedStreams.length > 0) {
|
if (compressedStreams && compressedStreams.length > 0) {
|
||||||
logger.info('Found', compressedStreams.length, 'compressed streams')
|
logger.info('Found', compressedStreams.length, 'compressed streams')
|
||||||
|
|
||||||
// Process each stream
|
|
||||||
const decompressedContents = await Promise.all(
|
const decompressedContents = await Promise.all(
|
||||||
compressedStreams.map(async (stream) => {
|
compressedStreams.map(async (stream) => {
|
||||||
try {
|
try {
|
||||||
// Extract stream content between stream and endstream
|
|
||||||
const streamMatch = stream.match(/stream\r?\n([\s\S]*?)\r?\nendstream/)
|
const streamMatch = stream.match(/stream\r?\n([\s\S]*?)\r?\nendstream/)
|
||||||
if (!streamMatch || !streamMatch[1]) return ''
|
if (!streamMatch || !streamMatch[1]) return ''
|
||||||
|
|
||||||
const compressedData = Buffer.from(streamMatch[1], 'binary')
|
const compressedData = Buffer.from(streamMatch[1], 'binary')
|
||||||
|
|
||||||
// Try different decompression methods
|
|
||||||
try {
|
try {
|
||||||
// Try inflate (most common)
|
|
||||||
const decompressed = await inflateAsync(compressedData)
|
const decompressed = await inflateAsync(compressedData)
|
||||||
const content = decompressed.toString('utf-8')
|
const content = decompressed.toString('utf-8')
|
||||||
|
|
||||||
// Check if it contains readable text
|
|
||||||
const readable = content.replace(/[^\x20-\x7E\r\n]/g, ' ').trim()
|
const readable = content.replace(/[^\x20-\x7E\r\n]/g, ' ').trim()
|
||||||
if (
|
if (
|
||||||
readable.length > 50 &&
|
readable.length > 50 &&
|
||||||
@@ -329,12 +280,10 @@ export class RawPdfParser implements FileParser {
|
|||||||
return readable
|
return readable
|
||||||
}
|
}
|
||||||
} catch (_inflateErr) {
|
} catch (_inflateErr) {
|
||||||
// Try unzip as fallback
|
|
||||||
try {
|
try {
|
||||||
const decompressed = await unzipAsync(compressedData)
|
const decompressed = await unzipAsync(compressedData)
|
||||||
const content = decompressed.toString('utf-8')
|
const content = decompressed.toString('utf-8')
|
||||||
|
|
||||||
// Check if it contains readable text
|
|
||||||
const readable = content.replace(/[^\x20-\x7E\r\n]/g, ' ').trim()
|
const readable = content.replace(/[^\x20-\x7E\r\n]/g, ' ').trim()
|
||||||
if (
|
if (
|
||||||
readable.length > 50 &&
|
readable.length > 50 &&
|
||||||
@@ -345,12 +294,10 @@ export class RawPdfParser implements FileParser {
|
|||||||
return readable
|
return readable
|
||||||
}
|
}
|
||||||
} catch (_unzipErr) {
|
} catch (_unzipErr) {
|
||||||
// Both methods failed, continue to next stream
|
|
||||||
return ''
|
return ''
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (_error) {
|
} catch (_error) {
|
||||||
// Error processing this stream, skip it
|
|
||||||
return ''
|
return ''
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -358,7 +305,6 @@ export class RawPdfParser implements FileParser {
|
|||||||
})
|
})
|
||||||
)
|
)
|
||||||
|
|
||||||
// Filter out empty results and combine
|
|
||||||
const decompressedText = decompressedContents
|
const decompressedText = decompressedContents
|
||||||
.filter((text) => text && text.length > 0)
|
.filter((text) => text && text.length > 0)
|
||||||
.join('\n\n')
|
.join('\n\n')
|
||||||
@@ -370,26 +316,19 @@ export class RawPdfParser implements FileParser {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Method 2: Look for text stream data
|
|
||||||
if (!extractedText || extractedText.length < 50) {
|
if (!extractedText || extractedText.length < 50) {
|
||||||
logger.info('Trying alternative text extraction method with streams')
|
logger.info('Trying alternative text extraction method with streams')
|
||||||
|
|
||||||
// Find text streams
|
|
||||||
const streamMatches = rawContent.match(/stream[\s\S]*?endstream/g)
|
const streamMatches = rawContent.match(/stream[\s\S]*?endstream/g)
|
||||||
if (streamMatches && streamMatches.length > 0) {
|
if (streamMatches && streamMatches.length > 0) {
|
||||||
logger.info('Found', streamMatches.length, 'streams')
|
logger.info('Found', streamMatches.length, 'streams')
|
||||||
|
|
||||||
// Process each stream to look for text content
|
|
||||||
const textContent = streamMatches
|
const textContent = streamMatches
|
||||||
.map((stream) => {
|
.map((stream) => {
|
||||||
// Remove 'stream' and 'endstream' markers
|
|
||||||
const content = stream.replace(/^stream\r?\n|\r?\nendstream$/g, '')
|
const content = stream.replace(/^stream\r?\n|\r?\nendstream$/g, '')
|
||||||
|
|
||||||
// Look for readable ASCII text (more strict heuristic)
|
|
||||||
// Only keep ASCII printable characters
|
|
||||||
const readable = content.replace(/[^\x20-\x7E\r\n]/g, ' ').trim()
|
const readable = content.replace(/[^\x20-\x7E\r\n]/g, ' ').trim()
|
||||||
|
|
||||||
// Only keep content that looks like real text (has spaces, periods, etc.)
|
|
||||||
if (
|
if (
|
||||||
readable.length > 20 &&
|
readable.length > 20 &&
|
||||||
readable.includes(' ') &&
|
readable.includes(' ') &&
|
||||||
@@ -400,7 +339,7 @@ export class RawPdfParser implements FileParser {
|
|||||||
}
|
}
|
||||||
return ''
|
return ''
|
||||||
})
|
})
|
||||||
.filter((text) => text.length > 0 && text.split(' ').length > 5) // Must have at least 5 words
|
.filter((text) => text.length > 0 && text.split(' ').length > 5)
|
||||||
.join('\n\n')
|
.join('\n\n')
|
||||||
|
|
||||||
if (textContent.length > 0) {
|
if (textContent.length > 0) {
|
||||||
@@ -409,22 +348,17 @@ export class RawPdfParser implements FileParser {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Method 3: Look for object streams
|
|
||||||
if (!extractedText || extractedText.length < 50) {
|
if (!extractedText || extractedText.length < 50) {
|
||||||
logger.info('Trying object streams for text')
|
logger.info('Trying object streams for text')
|
||||||
|
|
||||||
// Find object stream content
|
|
||||||
const objMatches = rawContent.match(/\d+\s+\d+\s+obj[\s\S]*?endobj/g)
|
const objMatches = rawContent.match(/\d+\s+\d+\s+obj[\s\S]*?endobj/g)
|
||||||
if (objMatches && objMatches.length > 0) {
|
if (objMatches && objMatches.length > 0) {
|
||||||
logger.info('Found', objMatches.length, 'objects')
|
logger.info('Found', objMatches.length, 'objects')
|
||||||
|
|
||||||
// Process objects looking for text content
|
|
||||||
const textContent = objMatches
|
const textContent = objMatches
|
||||||
.map((obj) => {
|
.map((obj) => {
|
||||||
// Find readable text in the object - only keep ASCII printable characters
|
|
||||||
const readable = obj.replace(/[^\x20-\x7E\r\n]/g, ' ').trim()
|
const readable = obj.replace(/[^\x20-\x7E\r\n]/g, ' ').trim()
|
||||||
|
|
||||||
// Only include if it looks like actual text (strict heuristic)
|
|
||||||
if (
|
if (
|
||||||
readable.length > 50 &&
|
readable.length > 50 &&
|
||||||
readable.includes(' ') &&
|
readable.includes(' ') &&
|
||||||
@@ -445,8 +379,6 @@ export class RawPdfParser implements FileParser {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// If what we extracted is just PDF structure information rather than readable text,
|
|
||||||
// provide a clearer message
|
|
||||||
if (
|
if (
|
||||||
extractedText &&
|
extractedText &&
|
||||||
(extractedText.includes('endobj') ||
|
(extractedText.includes('endobj') ||
|
||||||
@@ -459,53 +391,41 @@ export class RawPdfParser implements FileParser {
|
|||||||
)
|
)
|
||||||
extractedText = metadataText
|
extractedText = metadataText
|
||||||
} else if (metadataText && !extractedText.includes('Document Title:')) {
|
} else if (metadataText && !extractedText.includes('Document Title:')) {
|
||||||
// Prepend metadata to extracted text if available
|
|
||||||
extractedText = metadataText + (extractedText ? `\n\n${extractedText}` : '')
|
extractedText = metadataText + (extractedText ? `\n\n${extractedText}` : '')
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate that the extracted text looks meaningful
|
|
||||||
// Count how many recognizable words/characters it contains
|
|
||||||
const validCharCount = (extractedText || '').replace(/[^\x20-\x7E\r\n]/g, '').length
|
const validCharCount = (extractedText || '').replace(/[^\x20-\x7E\r\n]/g, '').length
|
||||||
const totalCharCount = (extractedText || '').length
|
const totalCharCount = (extractedText || '').length
|
||||||
const validRatio = validCharCount / (totalCharCount || 1)
|
const validRatio = validCharCount / (totalCharCount || 1)
|
||||||
|
|
||||||
// Check for common PDF artifacts that indicate binary corruption
|
|
||||||
const hasBinaryArtifacts =
|
const hasBinaryArtifacts =
|
||||||
extractedText &&
|
extractedText &&
|
||||||
(extractedText.includes('\\u') ||
|
(extractedText.includes('\\u') ||
|
||||||
extractedText.includes('\\x') ||
|
extractedText.includes('\\x') ||
|
||||||
extractedText.includes('\0') ||
|
extractedText.includes('\0') ||
|
||||||
/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F-\xFF]{10,}/g.test(extractedText) ||
|
/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F-\xFF]{10,}/g.test(extractedText) ||
|
||||||
validRatio < 0.7) // Less than 70% valid characters
|
validRatio < 0.7)
|
||||||
|
|
||||||
// Check if the content looks like gibberish
|
|
||||||
const looksLikeGibberish =
|
const looksLikeGibberish =
|
||||||
extractedText &&
|
extractedText &&
|
||||||
// Too many special characters
|
|
||||||
(extractedText.replace(/[a-zA-Z0-9\s.,:'"()[\]{}]/g, '').length / extractedText.length >
|
(extractedText.replace(/[a-zA-Z0-9\s.,:'"()[\]{}]/g, '').length / extractedText.length >
|
||||||
0.3 ||
|
0.3 ||
|
||||||
// Not enough spaces (real text has spaces between words)
|
|
||||||
extractedText.split(' ').length < extractedText.length / 20)
|
extractedText.split(' ').length < extractedText.length / 20)
|
||||||
|
|
||||||
// If no text was extracted, or if it's binary/gibberish,
|
|
||||||
// provide a helpful message instead
|
|
||||||
if (!extractedText || extractedText.length < 50 || hasBinaryArtifacts || looksLikeGibberish) {
|
if (!extractedText || extractedText.length < 50 || hasBinaryArtifacts || looksLikeGibberish) {
|
||||||
logger.info('Could not extract meaningful text, providing fallback message')
|
logger.info('Could not extract meaningful text, providing fallback message')
|
||||||
logger.info('Valid character ratio:', validRatio)
|
logger.info('Valid character ratio:', validRatio)
|
||||||
logger.info('Has binary artifacts:', hasBinaryArtifacts)
|
logger.info('Has binary artifacts:', hasBinaryArtifacts)
|
||||||
logger.info('Looks like gibberish:', looksLikeGibberish)
|
logger.info('Looks like gibberish:', looksLikeGibberish)
|
||||||
|
|
||||||
// Start with metadata if available
|
|
||||||
if (metadataText) {
|
if (metadataText) {
|
||||||
extractedText = `${metadataText}\n`
|
extractedText = `${metadataText}\n`
|
||||||
} else {
|
} else {
|
||||||
extractedText = ''
|
extractedText = ''
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add basic PDF info
|
|
||||||
extractedText += `This is a PDF document with ${pageCount} page(s) and version ${version}.\n\n`
|
extractedText += `This is a PDF document with ${pageCount} page(s) and version ${version}.\n\n`
|
||||||
|
|
||||||
// Try to find a title in the PDF structure that we might have missed
|
|
||||||
const titleInStructure =
|
const titleInStructure =
|
||||||
rawContent.match(/title\s*:\s*([^\n]+)/i) ||
|
rawContent.match(/title\s*:\s*([^\n]+)/i) ||
|
||||||
rawContent.match(/Microsoft Word -\s*([^\n]+)/i)
|
rawContent.match(/Microsoft Word -\s*([^\n]+)/i)
|
||||||
|
|||||||
@@ -8,15 +8,12 @@ const logger = createLogger('TxtParser')
|
|||||||
export class TxtParser implements FileParser {
|
export class TxtParser implements FileParser {
|
||||||
async parseFile(filePath: string): Promise<FileParseResult> {
|
async parseFile(filePath: string): Promise<FileParseResult> {
|
||||||
try {
|
try {
|
||||||
// Validate input
|
|
||||||
if (!filePath) {
|
if (!filePath) {
|
||||||
throw new Error('No file path provided')
|
throw new Error('No file path provided')
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read the file
|
|
||||||
const buffer = await readFile(filePath)
|
const buffer = await readFile(filePath)
|
||||||
|
|
||||||
// Use parseBuffer for consistent implementation
|
|
||||||
return this.parseBuffer(buffer)
|
return this.parseBuffer(buffer)
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error('TXT file error:', error)
|
logger.error('TXT file error:', error)
|
||||||
@@ -28,7 +25,6 @@ export class TxtParser implements FileParser {
|
|||||||
try {
|
try {
|
||||||
logger.info('Parsing buffer, size:', buffer.length)
|
logger.info('Parsing buffer, size:', buffer.length)
|
||||||
|
|
||||||
// Extract content and sanitize for UTF-8 storage
|
|
||||||
const rawContent = buffer.toString('utf-8')
|
const rawContent = buffer.toString('utf-8')
|
||||||
const result = sanitizeTextForUTF8(rawContent)
|
const result = sanitizeTextForUTF8(rawContent)
|
||||||
|
|
||||||
|
|||||||
@@ -8,4 +8,16 @@ export interface FileParser {
|
|||||||
parseBuffer?(buffer: Buffer): Promise<FileParseResult>
|
parseBuffer?(buffer: Buffer): Promise<FileParseResult>
|
||||||
}
|
}
|
||||||
|
|
||||||
export type SupportedFileType = 'pdf' | 'csv' | 'doc' | 'docx' | 'txt' | 'md' | 'xlsx' | 'xls'
|
export type SupportedFileType =
|
||||||
|
| 'pdf'
|
||||||
|
| 'csv'
|
||||||
|
| 'doc'
|
||||||
|
| 'docx'
|
||||||
|
| 'txt'
|
||||||
|
| 'md'
|
||||||
|
| 'xlsx'
|
||||||
|
| 'xls'
|
||||||
|
| 'html'
|
||||||
|
| 'htm'
|
||||||
|
| 'pptx'
|
||||||
|
| 'ppt'
|
||||||
|
|||||||
@@ -9,19 +9,16 @@ const logger = createLogger('XlsxParser')
|
|||||||
export class XlsxParser implements FileParser {
|
export class XlsxParser implements FileParser {
|
||||||
async parseFile(filePath: string): Promise<FileParseResult> {
|
async parseFile(filePath: string): Promise<FileParseResult> {
|
||||||
try {
|
try {
|
||||||
// Validate input
|
|
||||||
if (!filePath) {
|
if (!filePath) {
|
||||||
throw new Error('No file path provided')
|
throw new Error('No file path provided')
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if file exists
|
|
||||||
if (!existsSync(filePath)) {
|
if (!existsSync(filePath)) {
|
||||||
throw new Error(`File not found: ${filePath}`)
|
throw new Error(`File not found: ${filePath}`)
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.info(`Parsing XLSX file: ${filePath}`)
|
logger.info(`Parsing XLSX file: ${filePath}`)
|
||||||
|
|
||||||
// Read the workbook
|
|
||||||
const workbook = XLSX.readFile(filePath)
|
const workbook = XLSX.readFile(filePath)
|
||||||
return this.processWorkbook(workbook)
|
return this.processWorkbook(workbook)
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -38,7 +35,6 @@ export class XlsxParser implements FileParser {
|
|||||||
throw new Error('Empty buffer provided')
|
throw new Error('Empty buffer provided')
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read the workbook from buffer
|
|
||||||
const workbook = XLSX.read(buffer, { type: 'buffer' })
|
const workbook = XLSX.read(buffer, { type: 'buffer' })
|
||||||
return this.processWorkbook(workbook)
|
return this.processWorkbook(workbook)
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -53,25 +49,20 @@ export class XlsxParser implements FileParser {
|
|||||||
let content = ''
|
let content = ''
|
||||||
let totalRows = 0
|
let totalRows = 0
|
||||||
|
|
||||||
// Process each worksheet
|
|
||||||
for (const sheetName of sheetNames) {
|
for (const sheetName of sheetNames) {
|
||||||
const worksheet = workbook.Sheets[sheetName]
|
const worksheet = workbook.Sheets[sheetName]
|
||||||
|
|
||||||
// Convert to array of objects
|
|
||||||
const sheetData = XLSX.utils.sheet_to_json(worksheet, { header: 1 })
|
const sheetData = XLSX.utils.sheet_to_json(worksheet, { header: 1 })
|
||||||
sheets[sheetName] = sheetData
|
sheets[sheetName] = sheetData
|
||||||
totalRows += sheetData.length
|
totalRows += sheetData.length
|
||||||
|
|
||||||
// Add sheet content to the overall content string (clean sheet name)
|
|
||||||
const cleanSheetName = sanitizeTextForUTF8(sheetName)
|
const cleanSheetName = sanitizeTextForUTF8(sheetName)
|
||||||
content += `Sheet: ${cleanSheetName}\n`
|
content += `Sheet: ${cleanSheetName}\n`
|
||||||
content += `=${'='.repeat(cleanSheetName.length + 6)}\n\n`
|
content += `=${'='.repeat(cleanSheetName.length + 6)}\n\n`
|
||||||
|
|
||||||
if (sheetData.length > 0) {
|
if (sheetData.length > 0) {
|
||||||
// Process each row
|
|
||||||
sheetData.forEach((row: unknown, rowIndex: number) => {
|
sheetData.forEach((row: unknown, rowIndex: number) => {
|
||||||
if (Array.isArray(row) && row.length > 0) {
|
if (Array.isArray(row) && row.length > 0) {
|
||||||
// Convert row to string, handling undefined/null values and cleaning non-UTF8 characters
|
|
||||||
const rowString = row
|
const rowString = row
|
||||||
.map((cell) => {
|
.map((cell) => {
|
||||||
if (cell === null || cell === undefined) {
|
if (cell === null || cell === undefined) {
|
||||||
@@ -93,7 +84,6 @@ export class XlsxParser implements FileParser {
|
|||||||
|
|
||||||
logger.info(`XLSX parsing completed: ${sheetNames.length} sheets, ${totalRows} total rows`)
|
logger.info(`XLSX parsing completed: ${sheetNames.length} sheets, ${totalRows} total rows`)
|
||||||
|
|
||||||
// Final cleanup of the entire content to ensure UTF-8 compatibility
|
|
||||||
const cleanContent = sanitizeTextForUTF8(content).trim()
|
const cleanContent = sanitizeTextForUTF8(content).trim()
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
|||||||
24
apps/sim/lib/knowledge/consts.ts
Normal file
24
apps/sim/lib/knowledge/consts.ts
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
export const TAG_SLOT_CONFIG = {
|
||||||
|
text: {
|
||||||
|
slots: ['tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6', 'tag7'] as const,
|
||||||
|
maxSlots: 7,
|
||||||
|
},
|
||||||
|
} as const
|
||||||
|
|
||||||
|
export const SUPPORTED_FIELD_TYPES = Object.keys(TAG_SLOT_CONFIG) as Array<
|
||||||
|
keyof typeof TAG_SLOT_CONFIG
|
||||||
|
>
|
||||||
|
|
||||||
|
export const TAG_SLOTS = TAG_SLOT_CONFIG.text.slots
|
||||||
|
|
||||||
|
export const MAX_TAG_SLOTS = TAG_SLOT_CONFIG.text.maxSlots
|
||||||
|
|
||||||
|
export type TagSlot = (typeof TAG_SLOTS)[number]
|
||||||
|
|
||||||
|
export function getSlotsForFieldType(fieldType: string): readonly string[] {
|
||||||
|
const config = TAG_SLOT_CONFIG[fieldType as keyof typeof TAG_SLOT_CONFIG]
|
||||||
|
if (!config) {
|
||||||
|
return []
|
||||||
|
}
|
||||||
|
return config.slots
|
||||||
|
}
|
||||||
@@ -1,9 +1,9 @@
|
|||||||
import crypto, { randomUUID } from 'crypto'
|
import crypto, { randomUUID } from 'crypto'
|
||||||
import { tasks } from '@trigger.dev/sdk'
|
import { tasks } from '@trigger.dev/sdk'
|
||||||
import { and, asc, desc, eq, inArray, isNull, sql } from 'drizzle-orm'
|
import { and, asc, desc, eq, inArray, isNull, sql } from 'drizzle-orm'
|
||||||
import { getSlotsForFieldType, type TAG_SLOT_CONFIG } from '@/lib/constants/knowledge'
|
|
||||||
import { generateEmbeddings } from '@/lib/embeddings/utils'
|
import { generateEmbeddings } from '@/lib/embeddings/utils'
|
||||||
import { env } from '@/lib/env'
|
import { env } from '@/lib/env'
|
||||||
|
import { getSlotsForFieldType, type TAG_SLOT_CONFIG } from '@/lib/knowledge/consts'
|
||||||
import { processDocument } from '@/lib/knowledge/documents/document-processor'
|
import { processDocument } from '@/lib/knowledge/documents/document-processor'
|
||||||
import { getNextAvailableSlot } from '@/lib/knowledge/tags/service'
|
import { getNextAvailableSlot } from '@/lib/knowledge/tags/service'
|
||||||
import { createLogger } from '@/lib/logs/console/logger'
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
@@ -17,8 +17,8 @@ import type { DocumentSortField, SortOrder } from './types'
|
|||||||
const logger = createLogger('DocumentService')
|
const logger = createLogger('DocumentService')
|
||||||
|
|
||||||
const TIMEOUTS = {
|
const TIMEOUTS = {
|
||||||
OVERALL_PROCESSING: 600000,
|
OVERALL_PROCESSING: (env.KB_CONFIG_MAX_DURATION || 300) * 1000,
|
||||||
EMBEDDINGS_API: 180000,
|
EMBEDDINGS_API: (env.KB_CONFIG_MAX_TIMEOUT || 10000) * 18,
|
||||||
} as const
|
} as const
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -38,17 +38,17 @@ function withTimeout<T>(
|
|||||||
}
|
}
|
||||||
|
|
||||||
const PROCESSING_CONFIG = {
|
const PROCESSING_CONFIG = {
|
||||||
maxConcurrentDocuments: 4,
|
maxConcurrentDocuments: Math.max(1, Math.floor((env.KB_CONFIG_CONCURRENCY_LIMIT || 20) / 5)) || 4,
|
||||||
batchSize: 10,
|
batchSize: Math.max(1, Math.floor((env.KB_CONFIG_BATCH_SIZE || 20) / 2)) || 10,
|
||||||
delayBetweenBatches: 200,
|
delayBetweenBatches: (env.KB_CONFIG_DELAY_BETWEEN_BATCHES || 100) * 2,
|
||||||
delayBetweenDocuments: 100,
|
delayBetweenDocuments: (env.KB_CONFIG_DELAY_BETWEEN_DOCUMENTS || 50) * 2,
|
||||||
}
|
}
|
||||||
|
|
||||||
const REDIS_PROCESSING_CONFIG = {
|
const REDIS_PROCESSING_CONFIG = {
|
||||||
maxConcurrentDocuments: 12,
|
maxConcurrentDocuments: env.KB_CONFIG_CONCURRENCY_LIMIT || 20,
|
||||||
batchSize: 20,
|
batchSize: env.KB_CONFIG_BATCH_SIZE || 20,
|
||||||
delayBetweenBatches: 100,
|
delayBetweenBatches: env.KB_CONFIG_DELAY_BETWEEN_BATCHES || 100,
|
||||||
delayBetweenDocuments: 50,
|
delayBetweenDocuments: env.KB_CONFIG_DELAY_BETWEEN_DOCUMENTS || 50,
|
||||||
}
|
}
|
||||||
|
|
||||||
let documentQueue: DocumentProcessingQueue | null = null
|
let documentQueue: DocumentProcessingQueue | null = null
|
||||||
@@ -59,8 +59,8 @@ export function getDocumentQueue(): DocumentProcessingQueue {
|
|||||||
const config = redisClient ? REDIS_PROCESSING_CONFIG : PROCESSING_CONFIG
|
const config = redisClient ? REDIS_PROCESSING_CONFIG : PROCESSING_CONFIG
|
||||||
documentQueue = new DocumentProcessingQueue({
|
documentQueue = new DocumentProcessingQueue({
|
||||||
maxConcurrent: config.maxConcurrentDocuments,
|
maxConcurrent: config.maxConcurrentDocuments,
|
||||||
retryDelay: 2000,
|
retryDelay: env.KB_CONFIG_MIN_TIMEOUT || 1000,
|
||||||
maxRetries: 5,
|
maxRetries: env.KB_CONFIG_MAX_ATTEMPTS || 3,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return documentQueue
|
return documentQueue
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import {
|
|||||||
getSlotsForFieldType,
|
getSlotsForFieldType,
|
||||||
SUPPORTED_FIELD_TYPES,
|
SUPPORTED_FIELD_TYPES,
|
||||||
type TAG_SLOT_CONFIG,
|
type TAG_SLOT_CONFIG,
|
||||||
} from '@/lib/constants/knowledge'
|
} from '@/lib/knowledge/consts'
|
||||||
import type { BulkTagDefinitionsData, DocumentTagDefinition } from '@/lib/knowledge/tags/types'
|
import type { BulkTagDefinitionsData, DocumentTagDefinition } from '@/lib/knowledge/tags/types'
|
||||||
import type {
|
import type {
|
||||||
CreateTagDefinitionData,
|
CreateTagDefinitionData,
|
||||||
|
|||||||
@@ -36,11 +36,14 @@ export class ExecutionLogger implements IExecutionLoggerService {
|
|||||||
trigger: ExecutionTrigger
|
trigger: ExecutionTrigger
|
||||||
environment: ExecutionEnvironment
|
environment: ExecutionEnvironment
|
||||||
workflowState: WorkflowState
|
workflowState: WorkflowState
|
||||||
|
initialInput?: Record<string, unknown>
|
||||||
|
startedFromBlockId?: string
|
||||||
|
executionType?: 'api' | 'webhook' | 'schedule' | 'manual' | 'chat'
|
||||||
}): Promise<{
|
}): Promise<{
|
||||||
workflowLog: WorkflowExecutionLog
|
workflowLog: WorkflowExecutionLog
|
||||||
snapshot: WorkflowExecutionSnapshot
|
snapshot: WorkflowExecutionSnapshot
|
||||||
}> {
|
}> {
|
||||||
const { workflowId, executionId, trigger, environment, workflowState } = params
|
const { workflowId, executionId, trigger, environment, workflowState, initialInput, startedFromBlockId, executionType } = params
|
||||||
|
|
||||||
logger.debug(`Starting workflow execution ${executionId} for workflow ${workflowId}`)
|
logger.debug(`Starting workflow execution ${executionId} for workflow ${workflowId}`)
|
||||||
|
|
||||||
@@ -66,6 +69,9 @@ export class ExecutionLogger implements IExecutionLoggerService {
|
|||||||
executionData: {
|
executionData: {
|
||||||
environment,
|
environment,
|
||||||
trigger,
|
trigger,
|
||||||
|
initialInput: initialInput || {},
|
||||||
|
startedFromBlockId: startedFromBlockId || undefined,
|
||||||
|
executionType: executionType || trigger.type,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
.returning()
|
.returning()
|
||||||
@@ -137,6 +143,39 @@ export class ExecutionLogger implements IExecutionLoggerService {
|
|||||||
// Extract files from trace spans and final output
|
// Extract files from trace spans and final output
|
||||||
const executionFiles = this.extractFilesFromExecution(traceSpans, finalOutput)
|
const executionFiles = this.extractFilesFromExecution(traceSpans, finalOutput)
|
||||||
|
|
||||||
|
// Read the existing executionData so we can merge new fields without losing initialInput/environment/trigger
|
||||||
|
const [existing] = await db
|
||||||
|
.select({ executionData: workflowExecutionLogs.executionData })
|
||||||
|
.from(workflowExecutionLogs)
|
||||||
|
.where(eq(workflowExecutionLogs.executionId, executionId))
|
||||||
|
.limit(1)
|
||||||
|
|
||||||
|
const existingExecutionData = (existing?.executionData as any) || {}
|
||||||
|
|
||||||
|
// Build simple block execution summaries from trace spans (flat list)
|
||||||
|
const blockExecutions: any[] = []
|
||||||
|
const collectBlocks = (spans?: any[]) => {
|
||||||
|
if (!Array.isArray(spans)) return
|
||||||
|
spans.forEach((span) => {
|
||||||
|
if (span?.blockId) {
|
||||||
|
blockExecutions.push({
|
||||||
|
id: span.id,
|
||||||
|
blockId: span.blockId,
|
||||||
|
blockName: span.name,
|
||||||
|
blockType: span.type,
|
||||||
|
startedAt: span.startTime,
|
||||||
|
endedAt: span.endTime,
|
||||||
|
durationMs: span.duration,
|
||||||
|
status: span.status || 'success',
|
||||||
|
inputData: span.input || {},
|
||||||
|
outputData: span.output || {},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if (span?.children && Array.isArray(span.children)) collectBlocks(span.children)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
collectBlocks(traceSpans)
|
||||||
|
|
||||||
const [updatedLog] = await db
|
const [updatedLog] = await db
|
||||||
.update(workflowExecutionLogs)
|
.update(workflowExecutionLogs)
|
||||||
.set({
|
.set({
|
||||||
@@ -145,8 +184,10 @@ export class ExecutionLogger implements IExecutionLoggerService {
|
|||||||
totalDurationMs,
|
totalDurationMs,
|
||||||
files: executionFiles.length > 0 ? executionFiles : null,
|
files: executionFiles.length > 0 ? executionFiles : null,
|
||||||
executionData: {
|
executionData: {
|
||||||
|
...existingExecutionData,
|
||||||
traceSpans,
|
traceSpans,
|
||||||
finalOutput,
|
finalOutput,
|
||||||
|
blockExecutions: blockExecutions,
|
||||||
tokenBreakdown: {
|
tokenBreakdown: {
|
||||||
prompt: costSummary.totalPromptTokens,
|
prompt: costSummary.totalPromptTokens,
|
||||||
completion: costSummary.totalCompletionTokens,
|
completion: costSummary.totalCompletionTokens,
|
||||||
|
|||||||
@@ -21,6 +21,9 @@ export interface SessionStartParams {
|
|||||||
workspaceId?: string
|
workspaceId?: string
|
||||||
variables?: Record<string, string>
|
variables?: Record<string, string>
|
||||||
triggerData?: Record<string, unknown>
|
triggerData?: Record<string, unknown>
|
||||||
|
initialInput?: Record<string, unknown>
|
||||||
|
startBlockId?: string
|
||||||
|
executionType?: 'api' | 'webhook' | 'schedule' | 'manual' | 'chat'
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface SessionCompleteParams {
|
export interface SessionCompleteParams {
|
||||||
@@ -61,7 +64,7 @@ export class LoggingSession {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async start(params: SessionStartParams = {}): Promise<void> {
|
async start(params: SessionStartParams = {}): Promise<void> {
|
||||||
const { userId, workspaceId, variables, triggerData } = params
|
const { userId, workspaceId, variables, triggerData, initialInput, startBlockId, executionType } = params
|
||||||
|
|
||||||
try {
|
try {
|
||||||
this.trigger = createTriggerObject(this.triggerType, triggerData)
|
this.trigger = createTriggerObject(this.triggerType, triggerData)
|
||||||
@@ -80,6 +83,9 @@ export class LoggingSession {
|
|||||||
trigger: this.trigger,
|
trigger: this.trigger,
|
||||||
environment: this.environment,
|
environment: this.environment,
|
||||||
workflowState: this.workflowState,
|
workflowState: this.workflowState,
|
||||||
|
initialInput,
|
||||||
|
startedFromBlockId: startBlockId,
|
||||||
|
executionType: executionType || this.triggerType,
|
||||||
})
|
})
|
||||||
|
|
||||||
if (this.requestId) {
|
if (this.requestId) {
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import type {
|
|||||||
} from '@/lib/logs/types'
|
} from '@/lib/logs/types'
|
||||||
import { db } from '@/db'
|
import { db } from '@/db'
|
||||||
import { workflowExecutionSnapshots } from '@/db/schema'
|
import { workflowExecutionSnapshots } from '@/db/schema'
|
||||||
|
import { filterEdgesForTriggers } from '@/lib/workflows/trigger-rules'
|
||||||
|
|
||||||
const logger = createLogger('SnapshotService')
|
const logger = createLogger('SnapshotService')
|
||||||
|
|
||||||
@@ -27,8 +28,27 @@ export class SnapshotService implements ISnapshotService {
|
|||||||
workflowId: string,
|
workflowId: string,
|
||||||
state: WorkflowState
|
state: WorkflowState
|
||||||
): Promise<SnapshotCreationResult> {
|
): Promise<SnapshotCreationResult> {
|
||||||
|
// Ensure consistency: apply the same trigger-edge filtering used by the editor/execution
|
||||||
|
const filteredState = filterEdgesForTriggers(state)
|
||||||
|
|
||||||
// Hash the position-less state for deduplication (functional equivalence)
|
// Hash the position-less state for deduplication (functional equivalence)
|
||||||
const stateHash = this.computeStateHash(state)
|
const stateHash = this.computeStateHash(filteredState)
|
||||||
|
|
||||||
|
// Log a concise preview of the state being considered for snapshot
|
||||||
|
try {
|
||||||
|
logger.info('📸 Preparing workflow snapshot', {
|
||||||
|
workflowId,
|
||||||
|
stateHash,
|
||||||
|
blocks: Object.entries(filteredState.blocks || {}).map(([id, b]: [string, any]) => ({
|
||||||
|
id,
|
||||||
|
type: (b as any)?.type,
|
||||||
|
name: (b as any)?.name,
|
||||||
|
triggerMode: (b as any)?.triggerMode === true,
|
||||||
|
enabled: (b as any)?.enabled !== false,
|
||||||
|
})),
|
||||||
|
edgesCount: (filteredState.edges || []).length,
|
||||||
|
})
|
||||||
|
} catch {}
|
||||||
|
|
||||||
const existingSnapshot = await this.getSnapshotByHash(workflowId, stateHash)
|
const existingSnapshot = await this.getSnapshotByHash(workflowId, stateHash)
|
||||||
if (existingSnapshot) {
|
if (existingSnapshot) {
|
||||||
@@ -45,7 +65,7 @@ export class SnapshotService implements ISnapshotService {
|
|||||||
id: uuidv4(),
|
id: uuidv4(),
|
||||||
workflowId,
|
workflowId,
|
||||||
stateHash,
|
stateHash,
|
||||||
stateData: state, // Full state with positions, subblock values, etc.
|
stateData: filteredState, // Full state with positions, subblock values, etc., after consistent filtering
|
||||||
}
|
}
|
||||||
|
|
||||||
const [newSnapshot] = await db
|
const [newSnapshot] = await db
|
||||||
@@ -53,8 +73,24 @@ export class SnapshotService implements ISnapshotService {
|
|||||||
.values(snapshotData)
|
.values(snapshotData)
|
||||||
.returning()
|
.returning()
|
||||||
|
|
||||||
|
logger.info('✅ Saved workflow snapshot', {
|
||||||
|
workflowId,
|
||||||
|
snapshotId: newSnapshot.id,
|
||||||
|
stateHash,
|
||||||
|
blocksCount: Object.keys(filteredState.blocks || {}).length,
|
||||||
|
edgesCount: (filteredState.edges || []).length,
|
||||||
|
})
|
||||||
|
|
||||||
|
// Emit the exact state saved (debug level to avoid log noise); redact sensitive values if needed
|
||||||
|
try {
|
||||||
|
// Lazy import to avoid cycles
|
||||||
|
const utils = await import('@/lib/utils')
|
||||||
|
const redactedState = utils.redactApiKeys(newSnapshot.stateData as any)
|
||||||
|
logger.debug('🧩 Snapshot state data (exact):', redactedState)
|
||||||
|
} catch {}
|
||||||
|
|
||||||
logger.debug(`Created new snapshot for workflow ${workflowId} with hash ${stateHash}`)
|
logger.debug(`Created new snapshot for workflow ${workflowId} with hash ${stateHash}`)
|
||||||
logger.debug(`Stored full state with ${Object.keys(state.blocks || {}).length} blocks`)
|
logger.debug(`Stored full state with ${Object.keys(filteredState.blocks || {}).length} blocks`)
|
||||||
return {
|
return {
|
||||||
snapshot: {
|
snapshot: {
|
||||||
...newSnapshot,
|
...newSnapshot,
|
||||||
|
|||||||
@@ -109,6 +109,13 @@ export interface WorkflowExecutionLog {
|
|||||||
error: string
|
error: string
|
||||||
stackTrace?: string
|
stackTrace?: string
|
||||||
}
|
}
|
||||||
|
// Newly added: persist the original triggering input (starter/chat/api/webhook)
|
||||||
|
initialInput?: Record<string, unknown>
|
||||||
|
// Newly added: where execution began and type metadata
|
||||||
|
startedFromBlockId?: string
|
||||||
|
executionType?: 'api' | 'webhook' | 'schedule' | 'manual' | 'chat'
|
||||||
|
// Optional precomputed block execution summaries
|
||||||
|
blockExecutions?: any[]
|
||||||
}
|
}
|
||||||
// Top-level cost information
|
// Top-level cost information
|
||||||
cost?: {
|
cost?: {
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ import { SIM_AGENT_API_URL_DEFAULT } from '@/lib/sim-agent'
|
|||||||
|
|
||||||
const logger = createLogger('SimAgentClient')
|
const logger = createLogger('SimAgentClient')
|
||||||
|
|
||||||
// Base URL for the sim-agent service
|
|
||||||
const SIM_AGENT_BASE_URL = env.SIM_AGENT_API_URL || SIM_AGENT_API_URL_DEFAULT
|
const SIM_AGENT_BASE_URL = env.SIM_AGENT_API_URL || SIM_AGENT_API_URL_DEFAULT
|
||||||
|
|
||||||
export interface SimAgentRequest {
|
export interface SimAgentRequest {
|
||||||
@@ -45,7 +44,6 @@ class SimAgentClient {
|
|||||||
try {
|
try {
|
||||||
const url = `${this.baseUrl}${endpoint}`
|
const url = `${this.baseUrl}${endpoint}`
|
||||||
|
|
||||||
// Use provided API key or try to get it from environment
|
|
||||||
const requestHeaders: Record<string, string> = {
|
const requestHeaders: Record<string, string> = {
|
||||||
'Content-Type': 'application/json',
|
'Content-Type': 'application/json',
|
||||||
...headers,
|
...headers,
|
||||||
|
|||||||
43
apps/sim/lib/workflows/trigger-rules.ts
Normal file
43
apps/sim/lib/workflows/trigger-rules.ts
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
import { getBlock } from '@/blocks'
|
||||||
|
import type { WorkflowState } from '@/stores/workflows/workflow/types'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decide whether incoming edges should be blocked for a target block.
|
||||||
|
* - Block if the block is a pure trigger category (webhook, etc.)
|
||||||
|
* - Block if the block is currently in triggerMode
|
||||||
|
* - Block if the block is the starter block
|
||||||
|
*/
|
||||||
|
export function shouldBlockIncomingEdgesForTarget(blockType: string, triggerMode: boolean | undefined): boolean {
|
||||||
|
// Starter blocks should never have incoming edges
|
||||||
|
if (blockType === 'starter') return true
|
||||||
|
|
||||||
|
// Runtime toggle
|
||||||
|
if (triggerMode === true) return true
|
||||||
|
|
||||||
|
// Pure trigger categories
|
||||||
|
try {
|
||||||
|
const config = getBlock(blockType)
|
||||||
|
if (config?.category === 'triggers') return true
|
||||||
|
} catch {}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return a copy of state with edges to trigger-like targets removed.
|
||||||
|
*/
|
||||||
|
export function filterEdgesForTriggers(state: WorkflowState): WorkflowState {
|
||||||
|
const blocks = state.blocks || {}
|
||||||
|
const edges = state.edges || []
|
||||||
|
|
||||||
|
const filteredEdges = edges.filter((edge) => {
|
||||||
|
const target = blocks[edge.target]
|
||||||
|
if (!target) return false // Drop dangling edges defensively
|
||||||
|
return !shouldBlockIncomingEdgesForTarget(target.type, target.triggerMode)
|
||||||
|
})
|
||||||
|
|
||||||
|
return {
|
||||||
|
...state,
|
||||||
|
edges: filteredEdges,
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -69,19 +69,18 @@
|
|||||||
"@react-email/components": "^0.0.34",
|
"@react-email/components": "^0.0.34",
|
||||||
"@sentry/nextjs": "^9.15.0",
|
"@sentry/nextjs": "^9.15.0",
|
||||||
"@trigger.dev/sdk": "4.0.1",
|
"@trigger.dev/sdk": "4.0.1",
|
||||||
"@types/pg": "8.15.5",
|
|
||||||
"@types/three": "0.177.0",
|
"@types/three": "0.177.0",
|
||||||
"@vercel/og": "^0.6.5",
|
"@vercel/og": "^0.6.5",
|
||||||
"@vercel/speed-insights": "^1.2.0",
|
"@vercel/speed-insights": "^1.2.0",
|
||||||
"ai": "^4.3.2",
|
"ai": "^4.3.2",
|
||||||
"better-auth": "^1.2.9",
|
"better-auth": "^1.2.9",
|
||||||
"browser-image-compression": "^2.0.2",
|
"browser-image-compression": "^2.0.2",
|
||||||
|
"cheerio": "1.1.2",
|
||||||
"class-variance-authority": "^0.7.1",
|
"class-variance-authority": "^0.7.1",
|
||||||
"clsx": "^2.1.1",
|
"clsx": "^2.1.1",
|
||||||
"cmdk": "^1.0.0",
|
"cmdk": "^1.0.0",
|
||||||
"croner": "^9.0.0",
|
"croner": "^9.0.0",
|
||||||
"csv-parse": "^5.6.0",
|
"csv-parse": "^5.6.0",
|
||||||
"csv-parser": "^3.2.0",
|
|
||||||
"dat.gui": "0.7.9",
|
"dat.gui": "0.7.9",
|
||||||
"date-fns": "4.1.0",
|
"date-fns": "4.1.0",
|
||||||
"drizzle-orm": "^0.41.0",
|
"drizzle-orm": "^0.41.0",
|
||||||
@@ -90,6 +89,7 @@
|
|||||||
"geist": "1.4.2",
|
"geist": "1.4.2",
|
||||||
"groq-sdk": "^0.15.0",
|
"groq-sdk": "^0.15.0",
|
||||||
"html-to-text": "^9.0.5",
|
"html-to-text": "^9.0.5",
|
||||||
|
"iconv-lite": "0.7.0",
|
||||||
"input-otp": "^1.4.2",
|
"input-otp": "^1.4.2",
|
||||||
"ioredis": "^5.6.0",
|
"ioredis": "^5.6.0",
|
||||||
"jose": "6.0.11",
|
"jose": "6.0.11",
|
||||||
@@ -99,12 +99,13 @@
|
|||||||
"lucide-react": "^0.479.0",
|
"lucide-react": "^0.479.0",
|
||||||
"mammoth": "^1.9.0",
|
"mammoth": "^1.9.0",
|
||||||
"mysql2": "3.14.3",
|
"mysql2": "3.14.3",
|
||||||
"next": "^15.3.2",
|
"next": "^15.4.1",
|
||||||
"next-runtime-env": "3.3.0",
|
"next-runtime-env": "3.3.0",
|
||||||
"next-themes": "^0.4.6",
|
"next-themes": "^0.4.6",
|
||||||
|
"officeparser": "^5.2.0",
|
||||||
"openai": "^4.91.1",
|
"openai": "^4.91.1",
|
||||||
"pdf-parse": "^1.1.1",
|
"papaparse": "5.5.3",
|
||||||
"pg": "8.16.3",
|
"pdf-lib": "^1.17.1",
|
||||||
"postgres": "^3.4.5",
|
"postgres": "^3.4.5",
|
||||||
"prismjs": "^1.30.0",
|
"prismjs": "^1.30.0",
|
||||||
"react": "19.1.0",
|
"react": "19.1.0",
|
||||||
@@ -119,13 +120,14 @@
|
|||||||
"rehype-highlight": "7.0.2",
|
"rehype-highlight": "7.0.2",
|
||||||
"remark-gfm": "4.0.1",
|
"remark-gfm": "4.0.1",
|
||||||
"resend": "^4.1.2",
|
"resend": "^4.1.2",
|
||||||
|
"rtf-parser": "1.3.3",
|
||||||
|
"rtf-stream-parser": "3.8.0",
|
||||||
"socket.io": "^4.8.1",
|
"socket.io": "^4.8.1",
|
||||||
"stripe": "^17.7.0",
|
"stripe": "^17.7.0",
|
||||||
"tailwind-merge": "^2.6.0",
|
"tailwind-merge": "^2.6.0",
|
||||||
"tailwindcss-animate": "^1.0.7",
|
"tailwindcss-animate": "^1.0.7",
|
||||||
"three": "0.177.0",
|
"three": "0.177.0",
|
||||||
"uuid": "^11.1.0",
|
"uuid": "^11.1.0",
|
||||||
"word-extractor": "1.0.4",
|
|
||||||
"xlsx": "0.18.5",
|
"xlsx": "0.18.5",
|
||||||
"zod": "^3.24.2"
|
"zod": "^3.24.2"
|
||||||
},
|
},
|
||||||
@@ -136,10 +138,12 @@
|
|||||||
"@testing-library/user-event": "^14.6.1",
|
"@testing-library/user-event": "^14.6.1",
|
||||||
"@trigger.dev/build": "4.0.1",
|
"@trigger.dev/build": "4.0.1",
|
||||||
"@types/html-to-text": "^9.0.4",
|
"@types/html-to-text": "^9.0.4",
|
||||||
|
"@types/iconv-lite": "0.0.1",
|
||||||
"@types/js-yaml": "4.0.9",
|
"@types/js-yaml": "4.0.9",
|
||||||
"@types/jsdom": "21.1.7",
|
"@types/jsdom": "21.1.7",
|
||||||
"@types/lodash": "^4.17.16",
|
"@types/lodash": "^4.17.16",
|
||||||
"@types/node": "24.2.1",
|
"@types/node": "24.2.1",
|
||||||
|
"@types/papaparse": "5.3.16",
|
||||||
"@types/prismjs": "^1.26.5",
|
"@types/prismjs": "^1.26.5",
|
||||||
"@types/react": "^19",
|
"@types/react": "^19",
|
||||||
"@types/react-dom": "^19",
|
"@types/react-dom": "^19",
|
||||||
@@ -162,5 +166,9 @@
|
|||||||
"canvas",
|
"canvas",
|
||||||
"better-sqlite3",
|
"better-sqlite3",
|
||||||
"sharp"
|
"sharp"
|
||||||
]
|
],
|
||||||
|
"overrides": {
|
||||||
|
"next": "^15.4.1",
|
||||||
|
"@next/env": "^15.4.1"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import { createLogger } from '@/lib/logs/console/logger'
|
|||||||
import { loadWorkflowFromNormalizedTables } from '@/lib/workflows/db-helpers'
|
import { loadWorkflowFromNormalizedTables } from '@/lib/workflows/db-helpers'
|
||||||
import * as schema from '@/db/schema'
|
import * as schema from '@/db/schema'
|
||||||
import { workflow, workflowBlocks, workflowEdges, workflowSubflows } from '@/db/schema'
|
import { workflow, workflowBlocks, workflowEdges, workflowSubflows } from '@/db/schema'
|
||||||
|
import { shouldBlockIncomingEdgesForTarget } from '@/lib/workflows/trigger-rules'
|
||||||
|
|
||||||
const logger = createLogger('SocketDatabase')
|
const logger = createLogger('SocketDatabase')
|
||||||
|
|
||||||
@@ -597,7 +598,21 @@ async function handleBlockOperationTx(
|
|||||||
throw new Error(`Block ${payload.id} not found in workflow ${workflowId}`)
|
throw new Error(`Block ${payload.id} not found in workflow ${workflowId}`)
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.debug(`Updated block trigger mode: ${payload.id} -> ${payload.triggerMode}`)
|
// When enabling trigger mode, remove all incoming edges to this block at the database level
|
||||||
|
if (payload.triggerMode === true) {
|
||||||
|
const removed = await tx
|
||||||
|
.delete(workflowEdges)
|
||||||
|
.where(
|
||||||
|
and(eq(workflowEdges.workflowId, workflowId), eq(workflowEdges.targetBlockId, payload.id))
|
||||||
|
)
|
||||||
|
.returning({ id: workflowEdges.id })
|
||||||
|
|
||||||
|
logger.debug(
|
||||||
|
`Updated block trigger mode: ${payload.id} -> ${payload.triggerMode}. Removed ${removed.length} incoming edges for trigger mode.`
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
logger.debug(`Updated block trigger mode: ${payload.id} -> ${payload.triggerMode}`)
|
||||||
|
}
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -743,6 +758,24 @@ async function handleEdgeOperationTx(
|
|||||||
throw new Error('Missing required fields for add edge operation')
|
throw new Error('Missing required fields for add edge operation')
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Guard: do not allow incoming edges to trigger-like targets
|
||||||
|
const [targetBlock] = await tx
|
||||||
|
.select({ id: workflowBlocks.id, type: workflowBlocks.type, triggerMode: workflowBlocks.triggerMode })
|
||||||
|
.from(workflowBlocks)
|
||||||
|
.where(and(eq(workflowBlocks.workflowId, workflowId), eq(workflowBlocks.id, payload.target)))
|
||||||
|
.limit(1)
|
||||||
|
|
||||||
|
if (!targetBlock) {
|
||||||
|
throw new Error(`Target block ${payload.target} not found in workflow ${workflowId}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (shouldBlockIncomingEdgesForTarget(targetBlock.type as string, targetBlock.triggerMode as boolean)) {
|
||||||
|
logger.debug(
|
||||||
|
`Rejected edge add ${payload.id}: incoming edges not allowed to ${payload.target} (type=${targetBlock.type}, triggerMode=${Boolean(targetBlock.triggerMode)})`
|
||||||
|
)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
await tx.insert(workflowEdges).values({
|
await tx.insert(workflowEdges).values({
|
||||||
id: payload.id,
|
id: payload.id,
|
||||||
workflowId,
|
workflowId,
|
||||||
|
|||||||
24
apps/sim/stores/execution/debug-canvas/store.ts
Normal file
24
apps/sim/stores/execution/debug-canvas/store.ts
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
import { create } from 'zustand'
|
||||||
|
import type { WorkflowState } from '@/stores/workflows/workflow/types'
|
||||||
|
|
||||||
|
interface DebugCanvasState {
|
||||||
|
isActive: boolean
|
||||||
|
workflowState: WorkflowState | null
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DebugCanvasActions {
|
||||||
|
activate: (workflowState: WorkflowState) => void
|
||||||
|
deactivate: () => void
|
||||||
|
setWorkflowState: (workflowState: WorkflowState | null) => void
|
||||||
|
clear: () => void
|
||||||
|
}
|
||||||
|
|
||||||
|
export const useDebugCanvasStore = create<DebugCanvasState & DebugCanvasActions>()((set) => ({
|
||||||
|
isActive: false,
|
||||||
|
workflowState: null,
|
||||||
|
|
||||||
|
activate: (workflowState) => set({ isActive: true, workflowState }),
|
||||||
|
deactivate: () => set({ isActive: false, workflowState: null }),
|
||||||
|
setWorkflowState: (workflowState) => set({ workflowState }),
|
||||||
|
clear: () => set({ isActive: false, workflowState: null }),
|
||||||
|
}))
|
||||||
95
apps/sim/stores/execution/debug-snapshots/store.ts
Normal file
95
apps/sim/stores/execution/debug-snapshots/store.ts
Normal file
@@ -0,0 +1,95 @@
|
|||||||
|
import { create } from 'zustand'
|
||||||
|
import type { ExecutionContext } from '@/executor/types'
|
||||||
|
|
||||||
|
interface BlockSnapshot {
|
||||||
|
output: any
|
||||||
|
executed: boolean
|
||||||
|
executionTime?: number
|
||||||
|
}
|
||||||
|
|
||||||
|
interface SnapshotEntry {
|
||||||
|
blockSnapshots: Map<string, BlockSnapshot>
|
||||||
|
envVarValues?: Record<string, string>
|
||||||
|
workflowVariables?: Record<string, any>
|
||||||
|
pendingBlocks: string[]
|
||||||
|
createdAt: number
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DebugSnapshotState {
|
||||||
|
blockSnapshots: Map<string, BlockSnapshot>
|
||||||
|
envVarValues?: Record<string, string>
|
||||||
|
workflowVariables?: Record<string, any>
|
||||||
|
history: SnapshotEntry[]
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DebugSnapshotActions {
|
||||||
|
captureFromContext: (ctx: ExecutionContext) => void
|
||||||
|
pushFromContext: (ctx: ExecutionContext, pendingBlocks: string[]) => void
|
||||||
|
stepBack: () => SnapshotEntry | null
|
||||||
|
clear: () => void
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildBlockSnapshots(ctx: ExecutionContext): Map<string, BlockSnapshot> {
|
||||||
|
const next = new Map<string, BlockSnapshot>()
|
||||||
|
try {
|
||||||
|
ctx.blockStates.forEach((state, key) => {
|
||||||
|
next.set(String(key), {
|
||||||
|
output: state?.output ?? {},
|
||||||
|
executed: !!state?.executed,
|
||||||
|
executionTime: state?.executionTime,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
} catch {}
|
||||||
|
return next
|
||||||
|
}
|
||||||
|
|
||||||
|
export const useDebugSnapshotStore = create<DebugSnapshotState & DebugSnapshotActions>()(
|
||||||
|
(set, get) => ({
|
||||||
|
blockSnapshots: new Map<string, BlockSnapshot>(),
|
||||||
|
envVarValues: undefined,
|
||||||
|
workflowVariables: undefined,
|
||||||
|
history: [],
|
||||||
|
|
||||||
|
captureFromContext: (ctx: ExecutionContext) => {
|
||||||
|
const next = buildBlockSnapshots(ctx)
|
||||||
|
set({
|
||||||
|
blockSnapshots: next,
|
||||||
|
envVarValues: ctx.environmentVariables || undefined,
|
||||||
|
workflowVariables: ctx.workflowVariables || undefined,
|
||||||
|
})
|
||||||
|
},
|
||||||
|
|
||||||
|
pushFromContext: (ctx: ExecutionContext, pendingBlocks: string[]) => {
|
||||||
|
const entry: SnapshotEntry = {
|
||||||
|
blockSnapshots: buildBlockSnapshots(ctx),
|
||||||
|
envVarValues: ctx.environmentVariables || undefined,
|
||||||
|
workflowVariables: ctx.workflowVariables || undefined,
|
||||||
|
pendingBlocks: [...pendingBlocks],
|
||||||
|
createdAt: Date.now(),
|
||||||
|
}
|
||||||
|
set((state) => ({ history: [...state.history, entry] }))
|
||||||
|
},
|
||||||
|
|
||||||
|
stepBack: () => {
|
||||||
|
const { history } = get()
|
||||||
|
if (history.length <= 1) return null
|
||||||
|
const nextHistory = history.slice(0, -1)
|
||||||
|
const prev = nextHistory[nextHistory.length - 1]
|
||||||
|
set({
|
||||||
|
history: nextHistory,
|
||||||
|
blockSnapshots: prev.blockSnapshots,
|
||||||
|
envVarValues: prev.envVarValues,
|
||||||
|
workflowVariables: prev.workflowVariables,
|
||||||
|
})
|
||||||
|
return prev
|
||||||
|
},
|
||||||
|
|
||||||
|
clear: () =>
|
||||||
|
set({
|
||||||
|
blockSnapshots: new Map(),
|
||||||
|
envVarValues: undefined,
|
||||||
|
workflowVariables: undefined,
|
||||||
|
history: [],
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
)
|
||||||
@@ -61,5 +61,20 @@ export const useExecutionStore = create<ExecutionState & ExecutionActions>()((se
|
|||||||
setExecutor: (executor) => set({ executor }),
|
setExecutor: (executor) => set({ executor }),
|
||||||
setDebugContext: (debugContext) => set({ debugContext }),
|
setDebugContext: (debugContext) => set({ debugContext }),
|
||||||
setAutoPanDisabled: (disabled) => set({ autoPanDisabled: disabled }),
|
setAutoPanDisabled: (disabled) => set({ autoPanDisabled: disabled }),
|
||||||
|
setPanelFocusedBlockId: (id) => set({ panelFocusedBlockId: id }),
|
||||||
|
setExecutingBlockIds: (ids) => set({ executingBlockIds: new Set(ids) }),
|
||||||
|
setBreakpointId: (id) => set({ breakpointId: id }),
|
||||||
|
|
||||||
|
setStartPositions: (ids) => set({ startPositionIds: new Set(Array.from(ids).slice(0, 1)) }),
|
||||||
|
toggleStartPosition: (id) => {
|
||||||
|
set((state) => {
|
||||||
|
const isActive = state.startPositionIds.has(id)
|
||||||
|
// Enforce single selection
|
||||||
|
const next = isActive ? new Set<string>() : new Set<string>([id])
|
||||||
|
return { startPositionIds: next }
|
||||||
|
})
|
||||||
|
},
|
||||||
|
clearStartPositions: () => set({ startPositionIds: new Set() }),
|
||||||
|
|
||||||
reset: () => set(initialState),
|
reset: () => set(initialState),
|
||||||
}))
|
}))
|
||||||
|
|||||||
@@ -9,6 +9,10 @@ export interface ExecutionState {
|
|||||||
executor: Executor | null
|
executor: Executor | null
|
||||||
debugContext: ExecutionContext | null
|
debugContext: ExecutionContext | null
|
||||||
autoPanDisabled: boolean
|
autoPanDisabled: boolean
|
||||||
|
panelFocusedBlockId?: string | null
|
||||||
|
executingBlockIds: Set<string>
|
||||||
|
breakpointId: string | null
|
||||||
|
startPositionIds: Set<string>
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ExecutionActions {
|
export interface ExecutionActions {
|
||||||
@@ -19,6 +23,12 @@ export interface ExecutionActions {
|
|||||||
setExecutor: (executor: Executor | null) => void
|
setExecutor: (executor: Executor | null) => void
|
||||||
setDebugContext: (context: ExecutionContext | null) => void
|
setDebugContext: (context: ExecutionContext | null) => void
|
||||||
setAutoPanDisabled: (disabled: boolean) => void
|
setAutoPanDisabled: (disabled: boolean) => void
|
||||||
|
setPanelFocusedBlockId: (id: string | null) => void
|
||||||
|
setExecutingBlockIds: (ids: Set<string>) => void
|
||||||
|
setBreakpointId: (id: string | null) => void
|
||||||
|
setStartPositions: (ids: Set<string>) => void
|
||||||
|
toggleStartPosition: (id: string) => void
|
||||||
|
clearStartPositions: () => void
|
||||||
reset: () => void
|
reset: () => void
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -30,6 +40,10 @@ export const initialState: ExecutionState = {
|
|||||||
executor: null,
|
executor: null,
|
||||||
debugContext: null,
|
debugContext: null,
|
||||||
autoPanDisabled: false,
|
autoPanDisabled: false,
|
||||||
|
panelFocusedBlockId: null,
|
||||||
|
executingBlockIds: new Set(),
|
||||||
|
breakpointId: null,
|
||||||
|
startPositionIds: new Set(),
|
||||||
}
|
}
|
||||||
|
|
||||||
// Types for panning functionality
|
// Types for panning functionality
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
export type PanelTab = 'console' | 'variables' | 'chat' | 'copilot'
|
export type PanelTab = 'console' | 'variables' | 'chat' | 'copilot' | 'debug'
|
||||||
|
|
||||||
export interface PanelStore {
|
export interface PanelStore {
|
||||||
isOpen: boolean
|
isOpen: boolean
|
||||||
|
|||||||
@@ -70,7 +70,7 @@ export const deleteTool: ToolConfig<MySQLDeleteParams, MySQLResponse> = {
|
|||||||
database: params.database,
|
database: params.database,
|
||||||
username: params.username,
|
username: params.username,
|
||||||
password: params.password,
|
password: params.password,
|
||||||
ssl: params.ssl || 'preferred',
|
ssl: params.ssl || 'required',
|
||||||
table: params.table,
|
table: params.table,
|
||||||
where: params.where,
|
where: params.where,
|
||||||
}),
|
}),
|
||||||
|
|||||||
@@ -64,7 +64,7 @@ export const executeTool: ToolConfig<MySQLExecuteParams, MySQLResponse> = {
|
|||||||
database: params.database,
|
database: params.database,
|
||||||
username: params.username,
|
username: params.username,
|
||||||
password: params.password,
|
password: params.password,
|
||||||
ssl: params.ssl || 'preferred',
|
ssl: params.ssl || 'required',
|
||||||
query: params.query,
|
query: params.query,
|
||||||
}),
|
}),
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -70,7 +70,7 @@ export const insertTool: ToolConfig<MySQLInsertParams, MySQLResponse> = {
|
|||||||
database: params.database,
|
database: params.database,
|
||||||
username: params.username,
|
username: params.username,
|
||||||
password: params.password,
|
password: params.password,
|
||||||
ssl: params.ssl || 'preferred',
|
ssl: params.ssl || 'required',
|
||||||
table: params.table,
|
table: params.table,
|
||||||
data: params.data,
|
data: params.data,
|
||||||
}),
|
}),
|
||||||
|
|||||||
@@ -64,7 +64,7 @@ export const queryTool: ToolConfig<MySQLQueryParams, MySQLResponse> = {
|
|||||||
database: params.database,
|
database: params.database,
|
||||||
username: params.username,
|
username: params.username,
|
||||||
password: params.password,
|
password: params.password,
|
||||||
ssl: params.ssl || 'preferred',
|
ssl: params.ssl || 'required',
|
||||||
query: params.query,
|
query: params.query,
|
||||||
}),
|
}),
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -76,7 +76,7 @@ export const updateTool: ToolConfig<MySQLUpdateParams, MySQLResponse> = {
|
|||||||
database: params.database,
|
database: params.database,
|
||||||
username: params.username,
|
username: params.username,
|
||||||
password: params.password,
|
password: params.password,
|
||||||
ssl: params.ssl || 'preferred',
|
ssl: params.ssl || 'required',
|
||||||
table: params.table,
|
table: params.table,
|
||||||
data: params.data,
|
data: params.data,
|
||||||
where: params.where,
|
where: params.where,
|
||||||
|
|||||||
10
package.json
10
package.json
@@ -27,24 +27,22 @@
|
|||||||
},
|
},
|
||||||
"overrides": {
|
"overrides": {
|
||||||
"react": "19.1.0",
|
"react": "19.1.0",
|
||||||
"react-dom": "19.1.0"
|
"react-dom": "19.1.0",
|
||||||
|
"next": "^15.4.1",
|
||||||
|
"@next/env": "^15.4.1"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@linear/sdk": "40.0.0",
|
"@linear/sdk": "40.0.0",
|
||||||
"@t3-oss/env-nextjs": "0.13.4",
|
"@t3-oss/env-nextjs": "0.13.4",
|
||||||
"@vercel/analytics": "1.5.0",
|
"@vercel/analytics": "1.5.0",
|
||||||
"drizzle-orm": "0.44.5",
|
|
||||||
"geist": "^1.4.2",
|
"geist": "^1.4.2",
|
||||||
"pg": "8.16.3",
|
|
||||||
"react-colorful": "5.6.1",
|
"react-colorful": "5.6.1",
|
||||||
"remark-gfm": "4.0.1",
|
"remark-gfm": "4.0.1",
|
||||||
"socket.io-client": "4.8.1"
|
"socket.io-client": "4.8.1"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@biomejs/biome": "2.0.0-beta.5",
|
"@biomejs/biome": "2.0.0-beta.5",
|
||||||
"@next/env": "^15.3.2",
|
"@next/env": "^15.4.1",
|
||||||
"@types/word-extractor": "1.0.6",
|
|
||||||
"dotenv-cli": "^8.0.0",
|
|
||||||
"husky": "9.1.7",
|
"husky": "9.1.7",
|
||||||
"lint-staged": "16.0.0",
|
"lint-staged": "16.0.0",
|
||||||
"turbo": "2.5.6"
|
"turbo": "2.5.6"
|
||||||
|
|||||||
Reference in New Issue
Block a user