github/sim
1
1
Fork 0
mirror of https://github.com/simstudioai/sim.git synced 2026-04-28 03:00:29 -04:00
Code Issues Packages Projects Releases Wiki Activity

cleanup code

Browse Source
This commit is contained in:
Vikhyath Mondreti
2026-01-15 19:49:52 -08:00
parent b464d70cda
commit d748a82645
27 changed files with 748 additions and 533 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
apps/sim/app/api/copilot/execute-tool/route.ts
View File

@@ -14,8 +14,7 @@ import {
import { generateRequestId } from '@/lib/core/utils/request'
import { getEffectiveDecryptedEnv } from '@/lib/environment/utils'
import { refreshTokenIfNeeded } from '@/app/api/auth/oauth/utils'
import { REFERENCE } from '@/executor/constants'
import { createEnvVarPattern } from '@/executor/utils/reference-validation'
import { resolveEnvVarReferences } from '@/executor/utils/reference-validation'
import { executeTool } from '@/tools'
import { getTool, resolveToolId } from '@/tools/utils'
@@ -28,45 +27,6 @@ const ExecuteToolSchema = z.object({
workflowId: z.string().optional(),
})
/**
* Resolves all {{ENV_VAR}} references in a value recursively
* Works with strings, arrays, and objects
*/
function resolveEnvVarReferences(value: any, envVars: Record<string, string>): any {
if (typeof value === 'string') {
// Check for exact match: entire string is "{{VAR_NAME}}"
const exactMatchPattern = new RegExp(
`^\\${REFERENCE.ENV_VAR_START}([^}]+)\\${REFERENCE.ENV_VAR_END}$`
)
const exactMatch = exactMatchPattern.exec(value)
if (exactMatch) {
const envVarName = exactMatch[1].trim()
return envVars[envVarName] ?? value
}
// Check for embedded references: "prefix {{VAR}} suffix"
const envVarPattern = createEnvVarPattern()
return value.replace(envVarPattern, (match, varName) => {
const trimmedName = varName.trim()
return envVars[trimmedName] ?? match
})
}
if (Array.isArray(value)) {
return value.map((item) => resolveEnvVarReferences(item, envVars))
}
if (value !== null && typeof value === 'object') {
const resolved: Record<string, any> = {}
for (const [key, val] of Object.entries(value)) {
resolved[key] = resolveEnvVarReferences(val, envVars)
}
return resolved
}
return value
}
export async function POST(req: NextRequest) {
const tracker = createRequestTracker()
@@ -145,7 +105,17 @@ export async function POST(req: NextRequest) {
// Build execution params starting with LLM-provided arguments
// Resolve all {{ENV_VAR}} references in the arguments
const executionParams: Record<string, any> = resolveEnvVarReferences(toolArgs, decryptedEnvVars)
const executionParams: Record<string, any> = resolveEnvVarReferences(
toolArgs,
decryptedEnvVars,
{
resolveExactMatch: true,
allowEmbedded: true,
trimKeys: true,
onMissing: 'keep',
deep: true,
}
) as Record<string, any>
logger.info(`[${tracker.requestId}] Resolved env var references in arguments`, {
toolName,

23
apps/sim/app/api/function/execute/route.ts
View File

@@ -9,6 +9,7 @@ import { escapeRegExp, normalizeName, REFERENCE } from '@/executor/constants'
import {
createEnvVarPattern,
createWorkflowVariablePattern,
resolveEnvVarReferences,
} from '@/executor/utils/reference-validation'
export const dynamic = 'force-dynamic'
export const runtime = 'nodejs'
@@ -479,9 +480,29 @@ function resolveEnvironmentVariables(
const replacements: Array<{ match: string; index: number; varName: string; varValue: string }> =
[]
const resolverVars: Record<string, string> = {}
Object.entries(params).forEach(([key, value]) => {
if (value) {
resolverVars[key] = String(value)
}
})
Object.entries(envVars).forEach(([key, value]) => {
if (value) {
resolverVars[key] = value
}
})
while ((match = regex.exec(code)) !== null) {
const varName = match[1].trim()
const varValue = envVars[varName] || params[varName] || ''
const resolved = resolveEnvVarReferences(match[0], resolverVars, {
allowEmbedded: true,
resolveExactMatch: true,
trimKeys: true,
onMissing: 'empty',
deep: false,
})
const varValue =
typeof resolved === 'string' ? resolved : resolved == null ? '' : String(resolved)
replacements.push({
match: match[0],
index: match.index,

30
apps/sim/app/api/mcp/servers/test-connection/route.ts
View File

@@ -5,8 +5,7 @@ import { McpClient } from '@/lib/mcp/client'
import { getParsedBody, withMcpAuth } from '@/lib/mcp/middleware'
import type { McpServerConfig, McpTransport } from '@/lib/mcp/types'
import { createMcpErrorResponse, createMcpSuccessResponse } from '@/lib/mcp/utils'
import { REFERENCE } from '@/executor/constants'
import { createEnvVarPattern } from '@/executor/utils/reference-validation'
import { resolveEnvVarReferences } from '@/executor/utils/reference-validation'
const logger = createLogger('McpServerTestAPI')
@@ -24,22 +23,23 @@ function isUrlBasedTransport(transport: McpTransport): boolean {
* Resolve environment variables in strings
*/
function resolveEnvVars(value: string, envVars: Record<string, string>): string {
const envVarPattern = createEnvVarPattern()
const envMatches = value.match(envVarPattern)
if (!envMatches) return value
const missingVars: string[] = []
const resolvedValue = resolveEnvVarReferences(value, envVars, {
allowEmbedded: true,
resolveExactMatch: true,
trimKeys: true,
onMissing: 'keep',
deep: false,
missingKeys: missingVars,
}) as string
let resolvedValue = value
for (const match of envMatches) {
const envKey = match.slice(REFERENCE.ENV_VAR_START.length, -REFERENCE.ENV_VAR_END.length).trim()
const envValue = envVars[envKey]
if (envValue === undefined) {
if (missingVars.length > 0) {
const uniqueMissing = Array.from(new Set(missingVars))
uniqueMissing.forEach((envKey) => {
logger.warn(`Environment variable "${envKey}" not found in MCP server test`)
continue
}
resolvedValue = resolvedValue.replace(match, envValue)
})
}
return resolvedValue
}

25
apps/sim/app/api/schedules/execute/route.test.ts
View File

@@ -6,6 +6,11 @@
import type { NextRequest } from 'next/server'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
// Mock the preflight module before any imports to avoid cascade of db/schema imports
vi.mock('@/lib/workflows/executor/preflight', () => ({
preflightWorkflowEnvVars: vi.fn().mockResolvedValue(undefined),
}))
function createMockRequest(): NextRequest {
const mockHeaders = new Map([
['authorization', 'Bearer test-cron-secret'],
@@ -93,6 +98,11 @@ describe('Scheduled Workflow Execution API Route', () => {
nextRunAt: 'nextRunAt',
lastQueuedAt: 'lastQueuedAt',
},
workflow: {
id: 'id',
userId: 'userId',
workspaceId: 'workspaceId',
},
}
})
@@ -170,6 +180,11 @@ describe('Scheduled Workflow Execution API Route', () => {
nextRunAt: 'nextRunAt',
lastQueuedAt: 'lastQueuedAt',
},
workflow: {
id: 'id',
userId: 'userId',
workspaceId: 'workspaceId',
},
}
})
@@ -229,6 +244,11 @@ describe('Scheduled Workflow Execution API Route', () => {
nextRunAt: 'nextRunAt',
lastQueuedAt: 'lastQueuedAt',
},
workflow: {
id: 'id',
userId: 'userId',
workspaceId: 'workspaceId',
},
}
})
@@ -311,6 +331,11 @@ describe('Scheduled Workflow Execution API Route', () => {
nextRunAt: 'nextRunAt',
lastQueuedAt: 'lastQueuedAt',
},
workflow: {
id: 'id',
userId: 'userId',
workspaceId: 'workspaceId',
},
}
})

36
apps/sim/app/api/schedules/execute/route.ts
View File

@@ -1,4 +1,4 @@
import { db, workflowSchedule } from '@sim/db'
import { db, workflow, workflowSchedule } from '@sim/db'
import { createLogger } from '@sim/logger'
import { tasks } from '@trigger.dev/sdk'
import { and, eq, isNull, lt, lte, not, or } from 'drizzle-orm'
@@ -6,6 +6,7 @@ import { type NextRequest, NextResponse } from 'next/server'
import { verifyCronAuth } from '@/lib/auth/internal'
import { isTriggerDevEnabled } from '@/lib/core/config/feature-flags'
import { generateRequestId } from '@/lib/core/utils/request'
import { preflightWorkflowEnvVars } from '@/lib/workflows/executor/preflight'
import { executeScheduleJob } from '@/background/schedule-execution'
export const dynamic = 'force-dynamic'
@@ -68,6 +69,39 @@ export async function GET(request: NextRequest) {
failedCount: schedule.failedCount || 0,
now: queueTime.toISOString(),
scheduledFor: schedule.nextRunAt?.toISOString(),
preflighted: true,
}
const [workflowRecord] = await db
.select({ userId: workflow.userId, workspaceId: workflow.workspaceId })
.from(workflow)
.where(eq(workflow.id, schedule.workflowId))
.limit(1)
if (!workflowRecord?.userId || !workflowRecord.workspaceId) {
logger.warn(
`[${requestId}] Missing workflow metadata for preflight. Skipping Trigger.dev enqueue.`,
{ workflowId: schedule.workflowId }
)
await executeScheduleJob({ ...payload, preflighted: false })
return null
}
try {
await preflightWorkflowEnvVars({
workflowId: schedule.workflowId,
workspaceId: workflowRecord.workspaceId,
envUserId: workflowRecord.userId,
requestId,
useDraftState: false,
})
} catch (error) {
logger.warn(
`[${requestId}] Env preflight failed. Skipping Trigger.dev enqueue for workflow ${schedule.workflowId}`,
{ error: error instanceof Error ? error.message : String(error) }
)
await executeScheduleJob({ ...payload, preflighted: false })
return null
}
const handle = await tasks.trigger('schedule-execution', payload)

7
apps/sim/app/api/workflows/[id]/execute/route.ts
View File

@@ -110,6 +110,7 @@ type AsyncExecutionParams = {
userId: string
input: any
triggerType: CoreTriggerType
preflighted?: boolean
}
/**
@@ -132,6 +133,7 @@ async function handleAsyncExecution(params: AsyncExecutionParams): Promise<NextR
userId,
input,
triggerType,
preflighted: params.preflighted,
}
try {
@@ -264,6 +266,7 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
requestId
)
const shouldPreflightEnvVars = isAsyncMode && isTriggerDevEnabled
const preprocessResult = await preprocessExecution({
workflowId,
userId,
@@ -272,6 +275,9 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
requestId,
checkDeployment: !shouldUseDraftState,
loggingSession,
preflightEnvVars: shouldPreflightEnvVars,
useDraftState: shouldUseDraftState,
envUserId: isClientSession ? userId : undefined,
})
if (!preprocessResult.success) {
@@ -303,6 +309,7 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
userId: actorUserId,
input,
triggerType: loggingTriggerType,
preflighted: shouldPreflightEnvVars,
})
}

69
apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/file-selector/file-selector-input.tsx
View File

@@ -4,15 +4,19 @@ import { useMemo } from 'react'
import { useParams } from 'next/navigation'
import { Tooltip } from '@/components/emcn'
import { getProviderIdFromServiceId } from '@/lib/oauth'
import { buildCanonicalIndex, resolveDependencyValue } from '@/lib/workflows/subblocks/visibility'
import { SelectorCombobox } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/selector-combobox/selector-combobox'
import { useDependsOnGate } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-depends-on-gate'
import { useForeignCredential } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-foreign-credential'
import { useSubBlockValue } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-sub-block-value'
import { getBlock } from '@/blocks/registry'
import type { SubBlockConfig } from '@/blocks/types'
import { isDependency } from '@/blocks/utils'
import { resolveSelectorForSubBlock, type SelectorResolution } from '@/hooks/selectors/resolution'
import { useCollaborativeWorkflow } from '@/hooks/use-collaborative-workflow'
import { useWorkflowRegistry } from '@/stores/workflows/registry/store'
import { useSubBlockStore } from '@/stores/workflows/subblock/store'
import { useWorkflowStore } from '@/stores/workflows/workflow/store'
interface FileSelectorInputProps {
blockId: string
@@ -42,21 +46,59 @@ export function FileSelectorInput({
previewContextValues,
})
const [connectedCredentialFromStore] = useSubBlockValue(blockId, 'credential')
const [domainValueFromStore] = useSubBlockValue(blockId, 'domain')
const [projectIdValueFromStore] = useSubBlockValue(blockId, 'projectId')
const [planIdValueFromStore] = useSubBlockValue(blockId, 'planId')
const [teamIdValueFromStore] = useSubBlockValue(blockId, 'teamId')
const [siteIdValueFromStore] = useSubBlockValue(blockId, 'siteId')
const [collectionIdValueFromStore] = useSubBlockValue(blockId, 'collectionId')
const blockState = useWorkflowStore((state) => state.blocks[blockId])
const blockConfig = blockState?.type ? getBlock(blockState.type) : null
const canonicalIndex = useMemo(
() => buildCanonicalIndex(blockConfig?.subBlocks || []),
[blockConfig?.subBlocks]
)
const canonicalModeOverrides = blockState?.data?.canonicalModes
const connectedCredential = previewContextValues?.credential ?? connectedCredentialFromStore
const blockValues = useSubBlockStore((state) => {
if (!activeWorkflowId) return {}
const workflowValues = state.workflowValues[activeWorkflowId] || {}
return (workflowValues as Record<string, Record<string, unknown>>)[blockId] || {}
})
const [domainValueFromStore] = useSubBlockValue(blockId, 'domain')
const connectedCredential = previewContextValues?.credential ?? blockValues.credential
const domainValue = previewContextValues?.domain ?? domainValueFromStore
const projectIdValue = previewContextValues?.projectId ?? projectIdValueFromStore
const planIdValue = previewContextValues?.planId ?? planIdValueFromStore
const teamIdValue = previewContextValues?.teamId ?? teamIdValueFromStore
const siteIdValue = previewContextValues?.siteId ?? siteIdValueFromStore
const collectionIdValue = previewContextValues?.collectionId ?? collectionIdValueFromStore
const teamIdValue = useMemo(
() =>
previewContextValues?.teamId ??
resolveDependencyValue('teamId', blockValues, canonicalIndex, canonicalModeOverrides),
[previewContextValues?.teamId, blockValues, canonicalIndex, canonicalModeOverrides]
)
const siteIdValue = useMemo(
() =>
previewContextValues?.siteId ??
resolveDependencyValue('siteId', blockValues, canonicalIndex, canonicalModeOverrides),
[previewContextValues?.siteId, blockValues, canonicalIndex, canonicalModeOverrides]
)
const collectionIdValue = useMemo(
() =>
previewContextValues?.collectionId ??
resolveDependencyValue('collectionId', blockValues, canonicalIndex, canonicalModeOverrides),
[previewContextValues?.collectionId, blockValues, canonicalIndex, canonicalModeOverrides]
)
const projectIdValue = useMemo(
() =>
previewContextValues?.projectId ??
resolveDependencyValue('projectId', blockValues, canonicalIndex, canonicalModeOverrides),
[previewContextValues?.projectId, blockValues, canonicalIndex, canonicalModeOverrides]
)
const planIdValue = useMemo(
() =>
previewContextValues?.planId ??
resolveDependencyValue('planId', blockValues, canonicalIndex, canonicalModeOverrides),
[previewContextValues?.planId, blockValues, canonicalIndex, canonicalModeOverrides]
)
const normalizedCredentialId =
typeof connectedCredential === 'string'
@@ -65,7 +107,6 @@ export function FileSelectorInput({
? ((connectedCredential as Record<string, any>).id ?? '')
: ''
// Derive provider from serviceId using OAuth config (same pattern as credential-selector)
const serviceId = subBlock.serviceId || ''
const effectiveProviderId = useMemo(() => getProviderIdFromServiceId(serviceId), [serviceId])

41
apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/project-selector/project-selector-input.tsx
View File

@@ -4,14 +4,17 @@ import { useEffect, useMemo, useState } from 'react'
import { useParams } from 'next/navigation'
import { Tooltip } from '@/components/emcn'
import { getProviderIdFromServiceId } from '@/lib/oauth'
import { buildCanonicalIndex, resolveDependencyValue } from '@/lib/workflows/subblocks/visibility'
import { SelectorCombobox } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/selector-combobox/selector-combobox'
import { useDependsOnGate } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-depends-on-gate'
import { useForeignCredential } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-foreign-credential'
import { useSubBlockValue } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-sub-block-value'
import { getBlock } from '@/blocks/registry'
import type { SubBlockConfig } from '@/blocks/types'
import { resolveSelectorForSubBlock } from '@/hooks/selectors/resolution'
import { useCollaborativeWorkflow } from '@/hooks/use-collaborative-workflow'
import { useWorkflowRegistry } from '@/stores/workflows/registry/store'
import { useSubBlockStore } from '@/stores/workflows/subblock/store'
import { useWorkflowStore } from '@/stores/workflows/workflow/store'
interface ProjectSelectorInputProps {
blockId: string
@@ -32,21 +35,36 @@ export function ProjectSelectorInput({
previewValue,
previewContextValues,
}: ProjectSelectorInputProps) {
const { collaborativeSetSubblockValue } = useCollaborativeWorkflow()
const params = useParams()
const activeWorkflowId = useWorkflowRegistry((s) => s.activeWorkflowId) as string | null
const [selectedProjectId, setSelectedProjectId] = useState<string>('')
// Use the proper hook to get the current value and setter
const [storeValue] = useSubBlockValue(blockId, subBlock.id)
const [connectedCredentialFromStore] = useSubBlockValue(blockId, 'credential')
const [linearTeamIdFromStore] = useSubBlockValue(blockId, 'teamId')
const [jiraDomainFromStore] = useSubBlockValue(blockId, 'domain')
// Use previewContextValues if provided (for tools inside agent blocks), otherwise use store values
const connectedCredential = previewContextValues?.credential ?? connectedCredentialFromStore
const linearTeamId = previewContextValues?.teamId ?? linearTeamIdFromStore
const blockState = useWorkflowStore((state) => state.blocks[blockId])
const blockConfig = blockState?.type ? getBlock(blockState.type) : null
const canonicalIndex = useMemo(
() => buildCanonicalIndex(blockConfig?.subBlocks || []),
[blockConfig?.subBlocks]
)
const canonicalModeOverrides = blockState?.data?.canonicalModes
const blockValues = useSubBlockStore((state) => {
if (!activeWorkflowId) return {}
const workflowValues = state.workflowValues[activeWorkflowId] || {}
return (workflowValues as Record<string, Record<string, unknown>>)[blockId] || {}
})
const connectedCredential = previewContextValues?.credential ?? blockValues.credential
const jiraDomain = previewContextValues?.domain ?? jiraDomainFromStore
// Derive provider from serviceId using OAuth config
const linearTeamId = useMemo(
() =>
previewContextValues?.teamId ??
resolveDependencyValue('teamId', blockValues, canonicalIndex, canonicalModeOverrides),
[previewContextValues?.teamId, blockValues, canonicalIndex, canonicalModeOverrides]
)
const serviceId = subBlock.serviceId || ''
const effectiveProviderId = useMemo(() => getProviderIdFromServiceId(serviceId), [serviceId])
@@ -54,7 +72,6 @@ export function ProjectSelectorInput({
effectiveProviderId,
(connectedCredential as string) || ''
)
const activeWorkflowId = useWorkflowRegistry((s) => s.activeWorkflowId) as string | null
const workflowIdFromUrl = (params?.workflowId as string) || activeWorkflowId || ''
const { finalDisabled } = useDependsOnGate(blockId, subBlock, {
disabled,
@@ -62,12 +79,8 @@ export function ProjectSelectorInput({
previewContextValues,
})
// Jira/Discord upstream fields - use values from previewContextValues or store
const domain = (jiraDomain as string) || ''
// Verify Jira credential belongs to current user; if not, treat as absent
// Get the current value from the store or prop value if in preview mode
useEffect(() => {
if (isPreview && previewValue !== undefined) {
setSelectedProjectId(previewValue)

17
apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-depends-on-gate.ts
View File

@@ -1,7 +1,11 @@
'use client'
import { useMemo } from 'react'
import { buildCanonicalIndex, resolveDependencyValue } from '@/lib/workflows/subblocks/visibility'
import {
buildCanonicalIndex,
isNonEmptyValue,
resolveDependencyValue,
} from '@/lib/workflows/subblocks/visibility'
import { getBlock } from '@/blocks/registry'
import type { SubBlockConfig } from '@/blocks/types'
import { useWorkflowRegistry } from '@/stores/workflows/registry/store'
@@ -135,21 +139,14 @@ export function useDependsOnGate(
return map
})
const isValueSatisfied = (value: unknown): boolean => {
if (value === null || value === undefined) return false
if (typeof value === 'string') return value.trim().length > 0
if (Array.isArray(value)) return value.length > 0
return value !== ''
}
const depsSatisfied = useMemo(() => {
// Check all fields (AND logic) - all must be satisfied
const allSatisfied =
allFields.length === 0 || allFields.every((key) => isValueSatisfied(dependencyValuesMap[key]))
allFields.length === 0 || allFields.every((key) => isNonEmptyValue(dependencyValuesMap[key]))
// Check any fields (OR logic) - at least one must be satisfied
const anySatisfied =
anyFields.length === 0 || anyFields.some((key) => isValueSatisfied(dependencyValuesMap[key]))
anyFields.length === 0 || anyFields.some((key) => isNonEmptyValue(dependencyValuesMap[key]))
return allSatisfied && anySatisfied
}, [allFields, anyFields, dependencyValuesMap])

73
apps/sim/background/schedule-execution.ts
View File

@@ -5,7 +5,6 @@ import { Cron } from 'croner'
import { eq } from 'drizzle-orm'
import { v4 as uuidv4 } from 'uuid'
import type { ZodRecord, ZodString } from 'zod'
import { decryptSecret } from '@/lib/core/security/encryption'
import { getPersonalAndWorkspaceEnv } from '@/lib/environment/utils'
import { preprocessExecution } from '@/lib/execution/preprocessing'
import { LoggingSession } from '@/lib/logs/execution/logging-session'
@@ -22,12 +21,9 @@ import {
getScheduleTimeValues,
getSubBlockValue,
} from '@/lib/workflows/schedules/utils'
import { REFERENCE } from '@/executor/constants'
import { ExecutionSnapshot } from '@/executor/execution/snapshot'
import type { ExecutionMetadata } from '@/executor/execution/types'
import type { ExecutionResult } from '@/executor/types'
import { createEnvVarPattern } from '@/executor/utils/reference-validation'
import { mergeSubblockState } from '@/stores/workflows/server-utils'
import { MAX_CONSECUTIVE_FAILURES } from '@/triggers/constants'
const logger = createLogger('TriggerScheduleExecution')
@@ -119,68 +115,6 @@ async function determineNextRunAfterError(
return new Date(now.getTime() + 24 * 60 * 60 * 1000)
}
async function ensureBlockVariablesResolvable(
blocks: Record<string, BlockState>,
variables: Record<string, string>,
requestId: string
) {
await Promise.all(
Object.values(blocks).map(async (block) => {
const subBlocks = block.subBlocks ?? {}
await Promise.all(
Object.values(subBlocks).map(async (subBlock) => {
const value = subBlock.value
if (
typeof value !== 'string' ||
!value.includes(REFERENCE.ENV_VAR_START) ||
!value.includes(REFERENCE.ENV_VAR_END)
) {
return
}
const envVarPattern = createEnvVarPattern()
const matches = value.match(envVarPattern)
if (!matches) {
return
}
for (const match of matches) {
const varName = match.slice(
REFERENCE.ENV_VAR_START.length,
-REFERENCE.ENV_VAR_END.length
)
const encryptedValue = variables[varName]
if (!encryptedValue) {
throw new Error(`Environment variable "${varName}" was not found`)
}
try {
await decryptSecret(encryptedValue)
} catch (error) {
logger.error(`[${requestId}] Error decrypting value for variable "${varName}"`, error)
const message = error instanceof Error ? error.message : 'Unknown error'
throw new Error(`Failed to decrypt environment variable "${varName}": ${message}`)
}
}
})
)
})
)
}
async function ensureEnvVarsDecryptable(variables: Record<string, string>, requestId: string) {
for (const [key, encryptedValue] of Object.entries(variables)) {
try {
await decryptSecret(encryptedValue)
} catch (error) {
logger.error(`[${requestId}] Failed to decrypt environment variable "${key}"`, error)
const message = error instanceof Error ? error.message : 'Unknown error'
throw new Error(`Failed to decrypt environment variable "${key}": ${message}`)
}
}
}
async function runWorkflowExecution({
payload,
workflowRecord,
@@ -217,8 +151,6 @@ async function runWorkflowExecution({
}
}
const mergedStates = mergeSubblockState(blocks)
const workspaceId = workflowRecord.workspaceId
if (!workspaceId) {
throw new Error(`Workflow ${payload.workflowId} has no associated workspace`)
@@ -236,9 +168,6 @@ async function runWorkflowExecution({
...workspaceEncrypted,
})
await ensureBlockVariablesResolvable(mergedStates, variables, requestId)
await ensureEnvVarsDecryptable(variables, requestId)
const input = {
_context: {
workflowId: payload.workflowId,
@@ -348,6 +277,7 @@ export type ScheduleExecutionPayload = {
failedCount?: number
now: string
scheduledFor?: string
preflighted?: boolean
}
function calculateNextRunTime(
@@ -407,6 +337,7 @@ export async function executeScheduleJob(payload: ScheduleExecutionPayload) {
checkRateLimit: true,
checkDeployment: true,
loggingSession,
preflightEnvVars: !payload.preflighted,
})
if (!preprocessResult.success) {

15
apps/sim/background/webhook-execution.ts
View File

@@ -20,8 +20,6 @@ import { getWorkflowById } from '@/lib/workflows/utils'
import { ExecutionSnapshot } from '@/executor/execution/snapshot'
import type { ExecutionMetadata } from '@/executor/execution/types'
import type { ExecutionResult } from '@/executor/types'
import { Serializer } from '@/serializer'
import { mergeSubblockState } from '@/stores/workflows/server-utils'
import { getTrigger, isTriggerValid } from '@/triggers'
const logger = createLogger('TriggerWebhookExecution')
@@ -171,19 +169,6 @@ async function executeWebhookJobInternal(
}
const workflowVariables = (wfRows[0]?.variables as Record<string, any>) || {}
// Merge subblock states (matching workflow-execution pattern)
const mergedStates = mergeSubblockState(blocks)
// Create serialized workflow
const serializer = new Serializer()
const serializedWorkflow = serializer.serializeWorkflow(
mergedStates,
edges,
loops || {},
parallels || {},
true // Enable validation during execution
)
// Handle special Airtable case
if (payload.provider === 'airtable') {
logger.info(`[${requestId}] Processing Airtable webhook via fetchAndProcessAirtablePayloads`)

2
apps/sim/background/workflow-execution.ts
View File

@@ -20,6 +20,7 @@ export type WorkflowExecutionPayload = {
input?: any
triggerType?: CoreTriggerType
metadata?: Record<string, any>
preflighted?: boolean
}
/**
@@ -51,6 +52,7 @@ export async function executeWorkflowJob(payload: WorkflowExecutionPayload) {
checkRateLimit: true,
checkDeployment: true,
loggingSession: loggingSession,
preflightEnvVars: !payload.preflighted,
})
if (!preprocessResult.success) {

79
apps/sim/executor/utils/reference-validation.ts
View File

@@ -19,6 +19,85 @@ export function createEnvVarPattern(): RegExp {
return new RegExp(`\\${REFERENCE.ENV_VAR_START}([^}]+)\\${REFERENCE.ENV_VAR_END}`, 'g')
}
export interface EnvVarResolveOptions {
allowEmbedded?: boolean
resolveExactMatch?: boolean
trimKeys?: boolean
onMissing?: 'keep' | 'throw' | 'empty'
deep?: boolean
missingKeys?: string[]
}
/**
* Resolve {{ENV_VAR}} references in values using provided env vars.
*/
export function resolveEnvVarReferences(
value: unknown,
envVars: Record<string, string>,
options: EnvVarResolveOptions = {}
): unknown {
const {
allowEmbedded = true,
resolveExactMatch = true,
trimKeys = false,
onMissing = 'keep',
deep = true,
} = options
if (typeof value === 'string') {
if (resolveExactMatch) {
const exactMatchPattern = new RegExp(
`^\\${REFERENCE.ENV_VAR_START}([^}]+)\\${REFERENCE.ENV_VAR_END}$`
)
const exactMatch = exactMatchPattern.exec(value)
if (exactMatch) {
const envKey = trimKeys ? exactMatch[1].trim() : exactMatch[1]
const envValue = envVars[envKey]
if (envValue !== undefined) return envValue
if (options.missingKeys) options.missingKeys.push(envKey)
if (onMissing === 'throw') {
throw new Error(`Environment variable "${envKey}" was not found`)
}
if (onMissing === 'empty') {
return ''
}
return value
}
}
if (!allowEmbedded) return value
const envVarPattern = createEnvVarPattern()
return value.replace(envVarPattern, (match, varName) => {
const envKey = trimKeys ? String(varName).trim() : String(varName)
const envValue = envVars[envKey]
if (envValue !== undefined) return envValue
if (options.missingKeys) options.missingKeys.push(envKey)
if (onMissing === 'throw') {
throw new Error(`Environment variable "${envKey}" was not found`)
}
if (onMissing === 'empty') {
return ''
}
return match
})
}
if (deep && Array.isArray(value)) {
return value.map((item) => resolveEnvVarReferences(item, envVars, options))
}
if (deep && value !== null && typeof value === 'object') {
const resolved: Record<string, any> = {}
for (const [key, val] of Object.entries(value)) {
resolved[key] = resolveEnvVarReferences(val, envVars, options)
}
return resolved
}
return value
}
/**
* Creates a regex pattern for matching workflow variables <variable.name>
* Captures the variable name (after "variable.") in group 1

15
apps/sim/lib/core/utils/response-format.ts
View File

@@ -53,14 +53,25 @@ export function extractFieldsFromSchema(schema: any): Field[] {
* Helper function to safely parse response format
* Handles both string and object formats
*/
export function parseResponseFormatSafely(responseFormatValue: any, blockId: string): any {
export function parseResponseFormatSafely(
responseFormatValue: any,
blockId: string,
options?: { allowReferences?: boolean }
): any {
if (!responseFormatValue) {
return null
}
const allowReferences = options?.allowReferences ?? false
try {
if (typeof responseFormatValue === 'string') {
return JSON.parse(responseFormatValue)
const trimmedValue = responseFormatValue.trim()
if (trimmedValue === '') return null
if (allowReferences && trimmedValue.startsWith('<') && trimmedValue.includes('>')) {
return trimmedValue
}
return JSON.parse(trimmedValue)
}
return responseFormatValue
} catch (error) {

86
apps/sim/lib/environment/utils.ts
View File

@@ -3,6 +3,9 @@ import { environment, workspaceEnvironment } from '@sim/db/schema'
import { createLogger } from '@sim/logger'
import { eq } from 'drizzle-orm'
import { decryptSecret } from '@/lib/core/security/encryption'
import { REFERENCE } from '@/executor/constants'
import { createEnvVarPattern } from '@/executor/utils/reference-validation'
import type { BlockState } from '@/stores/workflows/workflow/types'
const logger = createLogger('EnvironmentUtils')
@@ -107,3 +110,86 @@ export async function getEffectiveDecryptedEnv(
)
return { ...personalDecrypted, ...workspaceDecrypted }
}
/**
* Ensure all environment variables can be decrypted.
*/
export async function ensureEnvVarsDecryptable(
variables: Record<string, string>,
options: { requestId?: string } = {}
): Promise<void> {
const requestId = options.requestId
for (const [key, encryptedValue] of Object.entries(variables)) {
try {
await decryptSecret(encryptedValue)
} catch (error) {
const message = error instanceof Error ? error.message : 'Unknown error'
if (requestId) {
logger.error(`[${requestId}] Failed to decrypt environment variable "${key}"`, error)
} else {
logger.error(`Failed to decrypt environment variable "${key}"`, error)
}
throw new Error(`Failed to decrypt environment variable "${key}": ${message}`)
}
}
}
/**
* Ensure all {{ENV_VAR}} references in block subblocks resolve to decryptable values.
*/
export async function ensureBlockEnvVarsResolvable(
blocks: Record<string, BlockState>,
variables: Record<string, string>,
options: { requestId?: string } = {}
): Promise<void> {
const requestId = options.requestId
await Promise.all(
Object.values(blocks).map(async (block) => {
const subBlocks = block.subBlocks ?? {}
await Promise.all(
Object.values(subBlocks).map(async (subBlock) => {
const value = subBlock.value
if (
typeof value !== 'string' ||
!value.includes(REFERENCE.ENV_VAR_START) ||
!value.includes(REFERENCE.ENV_VAR_END)
) {
return
}
const envVarPattern = createEnvVarPattern()
const matches = value.match(envVarPattern)
if (!matches) {
return
}
for (const match of matches) {
const varName = match.slice(
REFERENCE.ENV_VAR_START.length,
-REFERENCE.ENV_VAR_END.length
)
const encryptedValue = variables[varName]
if (!encryptedValue) {
throw new Error(`Environment variable "${varName}" was not found`)
}
try {
await decryptSecret(encryptedValue)
} catch (error) {
const message = error instanceof Error ? error.message : 'Unknown error'
if (requestId) {
logger.error(
`[${requestId}] Error decrypting value for variable "${varName}"`,
error
)
} else {
logger.error(`Error decrypting value for variable "${varName}"`, error)
}
throw new Error(`Failed to decrypt environment variable "${varName}": ${message}`)
}
}
})
)
})
)
}

46
apps/sim/lib/execution/preprocessing.ts
View File

@@ -6,6 +6,7 @@ import { checkServerSideUsageLimits } from '@/lib/billing/calculations/usage-mon
import { getHighestPrioritySubscription } from '@/lib/billing/core/subscription'
import { RateLimiter } from '@/lib/core/rate-limiter/rate-limiter'
import { LoggingSession } from '@/lib/logs/execution/logging-session'
import { preflightWorkflowEnvVars } from '@/lib/workflows/executor/preflight'
import { getWorkspaceBilledAccountUserId } from '@/lib/workspaces/utils'
import type { CoreTriggerType } from '@/stores/logs/filters/types'
@@ -117,11 +118,14 @@ export interface PreprocessExecutionOptions {
checkRateLimit?: boolean // Default: false for manual/chat, true for others
checkDeployment?: boolean // Default: true for non-manual triggers
skipUsageLimits?: boolean // Default: false (only use for test mode)
preflightEnvVars?: boolean // Default: false
// Context information
workspaceId?: string // If known, used for billing resolution
loggingSession?: LoggingSession // If provided, will be used for error logging
isResumeContext?: boolean // If true, allows fallback billing on resolution failure (for paused workflow resumes)
useDraftState?: boolean // If true, use draft workflow state for preflight
envUserId?: string // Optional override for env var resolution user
}
/**
@@ -159,9 +163,12 @@ export async function preprocessExecution(
checkRateLimit = triggerType !== 'manual' && triggerType !== 'chat',
checkDeployment = triggerType !== 'manual',
skipUsageLimits = false,
preflightEnvVars = false,
workspaceId: providedWorkspaceId,
loggingSession: providedLoggingSession,
isResumeContext = false,
useDraftState = false,
envUserId,
} = options
logger.info(`[${requestId}] Starting execution preprocessing`, {
@@ -476,6 +483,45 @@ export async function preprocessExecution(
}
// ========== SUCCESS: All Checks Passed ==========
if (preflightEnvVars) {
try {
const resolvedEnvUserId = envUserId || workflowRecord.userId || userId
await preflightWorkflowEnvVars({
workflowId,
workspaceId,
envUserId: resolvedEnvUserId,
requestId,
useDraftState,
})
} catch (error) {
const message = error instanceof Error ? error.message : 'Env var preflight failed'
logger.warn(`[${requestId}] Env var preflight failed`, {
workflowId,
message,
})
await logPreprocessingError({
workflowId,
executionId,
triggerType,
requestId,
userId: actorUserId,
workspaceId,
errorMessage: message,
loggingSession: providedLoggingSession,
})
return {
success: false,
error: {
message,
statusCode: 400,
logCreated: true,
},
}
}
}
logger.info(`[${requestId}] All preprocessing checks passed`, {
workflowId,
actorUserId,

35
apps/sim/lib/mcp/service.ts
View File

@@ -25,8 +25,7 @@ import type {
McpTransport,
} from '@/lib/mcp/types'
import { MCP_CONSTANTS } from '@/lib/mcp/utils'
import { REFERENCE } from '@/executor/constants'
import { createEnvVarPattern } from '@/executor/utils/reference-validation'
import { resolveEnvVarReferences } from '@/executor/utils/reference-validation'
const logger = createLogger('McpService')
@@ -51,31 +50,21 @@ class McpService {
* Resolve environment variables in strings
*/
private resolveEnvVars(value: string, envVars: Record<string, string>): string {
const envVarPattern = createEnvVarPattern()
const envMatches = value.match(envVarPattern)
if (!envMatches) return value
let resolvedValue = value
const missingVars: string[] = []
for (const match of envMatches) {
const envKey = match
.slice(REFERENCE.ENV_VAR_START.length, -REFERENCE.ENV_VAR_END.length)
.trim()
const envValue = envVars[envKey]
if (envValue === undefined) {
missingVars.push(envKey)
continue
}
resolvedValue = resolvedValue.replace(match, envValue)
}
const resolvedValue = resolveEnvVarReferences(value, envVars, {
allowEmbedded: true,
resolveExactMatch: true,
trimKeys: true,
onMissing: 'keep',
deep: false,
missingKeys: missingVars,
}) as string
if (missingVars.length > 0) {
const uniqueMissing = Array.from(new Set(missingVars))
throw new Error(
`Missing required environment variable${missingVars.length > 1 ? 's' : ''}: ${missingVars.join(', ')}. ` +
`Please set ${missingVars.length > 1 ? 'these variables' : 'this variable'} in your workspace or personal environment settings.`
`Missing required environment variable${uniqueMissing.length > 1 ? 's' : ''}: ${uniqueMissing.join(', ')}. ` +
`Please set ${uniqueMissing.length > 1 ? 'these variables' : 'this variable'} in your workspace or personal environment settings.`
)
}

24
apps/sim/lib/webhooks/processor.ts
View File

@@ -16,8 +16,7 @@ import {
verifyProviderWebhook,
} from '@/lib/webhooks/utils.server'
import { executeWebhookJob } from '@/background/webhook-execution'
import { REFERENCE } from '@/executor/constants'
import { createEnvVarPattern } from '@/executor/utils/reference-validation'
import { resolveEnvVarReferences } from '@/executor/utils/reference-validation'
const logger = createLogger('WebhookProcessor')
@@ -353,19 +352,13 @@ export async function findAllWebhooksForPath(
* @returns String with all {{VARIABLE}} references replaced
*/
function resolveEnvVars(value: string, envVars: Record<string, string>): string {
const envVarPattern = createEnvVarPattern()
const envMatches = value.match(envVarPattern)
if (!envMatches) return value
let resolvedValue = value
for (const match of envMatches) {
const envKey = match.slice(REFERENCE.ENV_VAR_START.length, -REFERENCE.ENV_VAR_END.length).trim()
const envValue = envVars[envKey]
if (envValue !== undefined) {
resolvedValue = resolvedValue.replaceAll(match, envValue)
}
}
return resolvedValue
return resolveEnvVarReferences(value, envVars, {
allowEmbedded: true,
resolveExactMatch: true,
trimKeys: true,
onMissing: 'keep',
deep: false,
}) as string
}
/**
@@ -750,6 +743,7 @@ export async function checkWebhookPreprocessing(
checkRateLimit: true,
checkDeployment: true,
workspaceId: foundWorkflow.workspaceId,
preflightEnvVars: isTriggerDevEnabled,
})
if (!preprocessResult.success) {

18
apps/sim/lib/workflows/credentials/credential-extractor.ts
View File

@@ -1,23 +1,17 @@
import {
buildCanonicalIndex,
buildSubBlockValues,
evaluateSubBlockCondition,
hasAdvancedValues,
isSubBlockFeatureEnabled,
isSubBlockVisibleForMode,
type SubBlockCondition,
} from '@/lib/workflows/subblocks/visibility'
import { getBlock } from '@/blocks/registry'
import type { SubBlockConfig } from '@/blocks/types'
import { AuthMode } from '@/blocks/types'
import type { BlockState, SubBlockState, WorkflowState } from '@/stores/workflows/workflow/types'
/** Condition type for SubBlock visibility - mirrors the inline type from blocks/types.ts */
interface SubBlockCondition {
field: string
value: string | number | boolean | Array<string | number | boolean> | undefined
not?: boolean
and?: SubBlockCondition
}
// Credential types based on actual patterns in the codebase
export enum CredentialType {
OAUTH = 'oauth',
@@ -126,13 +120,7 @@ export function extractRequiredCredentials(
function isSubBlockVisible(block: BlockState, subBlockConfig: SubBlockConfig): boolean {
if (!isSubBlockFeatureEnabled(subBlockConfig)) return false
const values = Object.entries(block?.subBlocks || {}).reduce<Record<string, unknown>>(
(acc, [key, subBlock]) => {
acc[key] = subBlock?.value
return acc
},
{}
)
const values = buildSubBlockValues(block?.subBlocks || {})
const blockConfig = getBlock(block.type)
const blockSubBlocks = blockConfig?.subBlocks || []
const canonicalIndex = buildCanonicalIndex(blockSubBlocks)

56
apps/sim/lib/workflows/executor/execution-core.ts
View File

@@ -6,6 +6,7 @@
import { createLogger } from '@sim/logger'
import type { Edge } from 'reactflow'
import { z } from 'zod'
import { parseResponseFormatSafely } from '@/lib/core/utils/response-format'
import { getPersonalAndWorkspaceEnv } from '@/lib/environment/utils'
import { clearExecutionCancellation } from '@/lib/execution/cancellation'
import type { LoggingSession } from '@/lib/logs/execution/logging-session'
@@ -17,7 +18,6 @@ import {
import { TriggerUtils } from '@/lib/workflows/triggers/triggers'
import { updateWorkflowRunCounts } from '@/lib/workflows/utils'
import { Executor } from '@/executor'
import { REFERENCE } from '@/executor/constants'
import type { ExecutionSnapshot } from '@/executor/execution/snapshot'
import type {
ContextExtensions,
@@ -25,7 +25,7 @@ import type {
IterationContext,
} from '@/executor/execution/types'
import type { ExecutionResult, NormalizedBlockOutput } from '@/executor/types'
import { createEnvVarPattern } from '@/executor/utils/reference-validation'
import { resolveEnvVarReferences } from '@/executor/utils/reference-validation'
import { Serializer } from '@/serializer'
import { mergeSubblockState } from '@/stores/workflows/server-utils'
@@ -203,25 +203,13 @@ export async function executeWorkflowCore(
(subAcc, [key, subBlock]) => {
let value = subBlock.value
if (
typeof value === 'string' &&
value.includes(REFERENCE.ENV_VAR_START) &&
value.includes(REFERENCE.ENV_VAR_END)
) {
const envVarPattern = createEnvVarPattern()
const matches = value.match(envVarPattern)
if (matches) {
for (const match of matches) {
const varName = match.slice(
REFERENCE.ENV_VAR_START.length,
-REFERENCE.ENV_VAR_END.length
)
const decryptedValue = decryptedEnvVars[varName]
if (decryptedValue !== undefined) {
value = (value as string).replace(match, decryptedValue)
}
}
}
if (typeof value === 'string') {
value = resolveEnvVarReferences(value, decryptedEnvVars, {
resolveExactMatch: false,
trimKeys: false,
onMissing: 'keep',
deep: false,
}) as string
}
subAcc[key] = value
@@ -237,26 +225,16 @@ export async function executeWorkflowCore(
// Process response format
const processedBlockStates = Object.entries(currentBlockStates).reduce(
(acc, [blockId, blockState]) => {
if (blockState.responseFormat && typeof blockState.responseFormat === 'string') {
const responseFormatValue = blockState.responseFormat.trim()
if (responseFormatValue && !responseFormatValue.startsWith(REFERENCE.START)) {
try {
acc[blockId] = {
...blockState,
responseFormat: JSON.parse(responseFormatValue),
}
} catch {
acc[blockId] = {
...blockState,
responseFormat: undefined,
}
}
} else {
acc[blockId] = blockState
}
} else {
const responseFormatValue = blockState.responseFormat
if (responseFormatValue === undefined || responseFormatValue === null) {
acc[blockId] = blockState
return acc
}
const responseFormat = parseResponseFormatSafely(responseFormatValue, blockId, {
allowReferences: true,
})
acc[blockId] = { ...blockState, responseFormat: responseFormat ?? undefined }
return acc
},
{} as Record<string, Record<string, any>>

56
apps/sim/lib/workflows/executor/preflight.ts Normal file
View File

@@ -0,0 +1,56 @@
import { createLogger } from '@sim/logger'
import {
ensureBlockEnvVarsResolvable,
ensureEnvVarsDecryptable,
getPersonalAndWorkspaceEnv,
} from '@/lib/environment/utils'
import {
loadDeployedWorkflowState,
loadWorkflowFromNormalizedTables,
} from '@/lib/workflows/persistence/utils'
import { mergeSubblockState } from '@/stores/workflows/server-utils'
const logger = createLogger('ExecutionPreflight')
export interface EnvVarPreflightOptions {
workflowId: string
workspaceId: string
envUserId: string
requestId?: string
useDraftState?: boolean
}
/**
* Preflight env var checks to avoid scheduling executions that will fail.
*/
export async function preflightWorkflowEnvVars({
workflowId,
workspaceId,
envUserId,
requestId,
useDraftState = false,
}: EnvVarPreflightOptions): Promise<void> {
const workflowData = useDraftState
? await loadWorkflowFromNormalizedTables(workflowId)
: await loadDeployedWorkflowState(workflowId)
if (!workflowData) {
throw new Error('Workflow state not found')
}
const mergedStates = mergeSubblockState(workflowData.blocks)
const { personalEncrypted, workspaceEncrypted } = await getPersonalAndWorkspaceEnv(
envUserId,
workspaceId
)
const variables = { ...personalEncrypted, ...workspaceEncrypted }
await ensureBlockEnvVarsResolvable(mergedStates, variables, { requestId })
await ensureEnvVarsDecryptable(variables, { requestId })
if (requestId) {
logger.debug(`[${requestId}] Env var preflight passed`, { workflowId })
} else {
logger.debug('Env var preflight passed', { workflowId })
}
}

12
apps/sim/lib/workflows/subblocks/visibility.ts
View File

@@ -31,6 +31,18 @@ export interface CanonicalValueSelection {
advancedSourceId?: string
}
/**
* Build a flat map of subblock values keyed by subblock id.
*/
export function buildSubBlockValues(
subBlocks: Record<string, { value?: unknown } | null | undefined>
): Record<string, unknown> {
return Object.entries(subBlocks).reduce<Record<string, unknown>>((acc, [key, subBlock]) => {
acc[key] = subBlock?.value
return acc
}, {})
}
/**
* Build canonical group indices for a block's subblocks.
*/

167
apps/sim/serializer/index.test.ts
View File

@@ -515,79 +515,21 @@ describe('Serializer', () => {
})
})
/**
* Advanced mode field filtering tests
*/
describe('advanced mode field filtering', () => {
it.concurrent('should prefer advanced canonical values when present', () => {
describe('canonical mode field selection', () => {
it.concurrent('should use advanced value when canonicalModes specifies advanced', () => {
const serializer = new Serializer()
const advancedModeBlock: any = {
const block: any = {
id: 'slack-1',
type: 'slack',
name: 'Test Slack Block',
position: { x: 0, y: 0 },
advancedMode: true, // Advanced mode enabled
subBlocks: {
channel: { value: 'general' }, // basic mode field
manualChannel: { value: 'C1234567890' }, // advanced mode field
text: { value: 'Hello world' }, // both mode field
username: { value: 'bot' }, // both mode field
data: {
canonicalModes: { channel: 'advanced' },
},
outputs: {},
enabled: true,
}
const serialized = serializer.serializeWorkflow({ 'slack-1': advancedModeBlock }, [], {})
const slackBlock = serialized.blocks.find((b) => b.id === 'slack-1')
expect(slackBlock).toBeDefined()
expect(slackBlock?.config.params.channel).toBe('C1234567890')
expect(slackBlock?.config.params.manualChannel).toBeUndefined()
expect(slackBlock?.config.params.text).toBe('Hello world')
expect(slackBlock?.config.params.username).toBe('bot')
})
it.concurrent('should persist advanced canonical values even in basic mode', () => {
const serializer = new Serializer()
const basicModeBlock: any = {
id: 'slack-1',
type: 'slack',
name: 'Test Slack Block',
position: { x: 0, y: 0 },
advancedMode: false, // Basic mode enabled
subBlocks: {
channel: { value: 'general' }, // basic mode field
manualChannel: { value: 'C1234567890' }, // advanced mode field
text: { value: 'Hello world' }, // both mode field
username: { value: 'bot' }, // both mode field
},
outputs: {},
enabled: true,
}
const serialized = serializer.serializeWorkflow({ 'slack-1': basicModeBlock }, [], {})
const slackBlock = serialized.blocks.find((b) => b.id === 'slack-1')
expect(slackBlock).toBeDefined()
expect(slackBlock?.config.params.channel).toBe('C1234567890')
expect(slackBlock?.config.params.manualChannel).toBeUndefined()
expect(slackBlock?.config.params.text).toBe('Hello world')
expect(slackBlock?.config.params.username).toBe('bot')
})
it.concurrent('should keep advanced canonical values when mode is undefined', () => {
const serializer = new Serializer()
const defaultModeBlock: any = {
id: 'slack-1',
type: 'slack',
name: 'Test Slack Block',
position: { x: 0, y: 0 },
subBlocks: {
operation: { value: 'send' },
destinationType: { value: 'channel' },
channel: { value: 'general' },
manualChannel: { value: 'C1234567890' },
text: { value: 'Hello world' },
@@ -597,17 +539,106 @@ describe('Serializer', () => {
enabled: true,
}
const serialized = serializer.serializeWorkflow({ 'slack-1': defaultModeBlock }, [], {})
const serialized = serializer.serializeWorkflow({ 'slack-1': block }, [], {})
const slackBlock = serialized.blocks.find((b) => b.id === 'slack-1')
expect(slackBlock).toBeDefined()
expect(slackBlock).toBeDefined()
expect(slackBlock?.config.params.channel).toBe('C1234567890')
expect(slackBlock?.config.params.manualChannel).toBeUndefined()
expect(slackBlock?.config.params.text).toBe('Hello world')
expect(slackBlock?.config.params.username).toBe('bot')
})
it.concurrent('should use basic value when canonicalModes specifies basic', () => {
const serializer = new Serializer()
const block: any = {
id: 'slack-1',
type: 'slack',
name: 'Test Slack Block',
position: { x: 0, y: 0 },
data: {
canonicalModes: { channel: 'basic' },
},
subBlocks: {
operation: { value: 'send' },
destinationType: { value: 'channel' },
channel: { value: 'general' },
manualChannel: { value: 'C1234567890' },
text: { value: 'Hello world' },
username: { value: 'bot' },
},
outputs: {},
enabled: true,
}
const serialized = serializer.serializeWorkflow({ 'slack-1': block }, [], {})
const slackBlock = serialized.blocks.find((b) => b.id === 'slack-1')
expect(slackBlock).toBeDefined()
expect(slackBlock?.config.params.channel).toBe('general')
expect(slackBlock?.config.params.manualChannel).toBeUndefined()
expect(slackBlock?.config.params.text).toBe('Hello world')
expect(slackBlock?.config.params.username).toBe('bot')
})
it.concurrent('should fall back to legacy advancedMode when canonicalModes not set', () => {
const serializer = new Serializer()
const block: any = {
id: 'slack-1',
type: 'slack',
name: 'Test Slack Block',
position: { x: 0, y: 0 },
advancedMode: true,
subBlocks: {
operation: { value: 'send' },
destinationType: { value: 'channel' },
channel: { value: 'general' },
manualChannel: { value: 'C1234567890' },
text: { value: 'Hello world' },
username: { value: 'bot' },
},
outputs: {},
enabled: true,
}
const serialized = serializer.serializeWorkflow({ 'slack-1': block }, [], {})
const slackBlock = serialized.blocks.find((b) => b.id === 'slack-1')
expect(slackBlock).toBeDefined()
expect(slackBlock?.config.params.channel).toBe('C1234567890')
expect(slackBlock?.config.params.manualChannel).toBeUndefined()
})
it.concurrent('should use basic value by default when no mode specified', () => {
const serializer = new Serializer()
const block: any = {
id: 'slack-1',
type: 'slack',
name: 'Test Slack Block',
position: { x: 0, y: 0 },
subBlocks: {
operation: { value: 'send' },
destinationType: { value: 'channel' },
channel: { value: 'general' },
manualChannel: { value: 'C1234567890' },
text: { value: 'Hello world' },
username: { value: 'bot' },
},
outputs: {},
enabled: true,
}
const serialized = serializer.serializeWorkflow({ 'slack-1': block }, [], {})
const slackBlock = serialized.blocks.find((b) => b.id === 'slack-1')
expect(slackBlock).toBeDefined()
expect(slackBlock?.config.params.channel).toBe('general')
expect(slackBlock?.config.params.manualChannel).toBeUndefined()
})
it.concurrent('should preserve advanced-only values when present in basic mode', () => {
const serializer = new Serializer()

229
apps/sim/serializer/index.ts
View File

@@ -1,18 +1,17 @@
import { createLogger } from '@sim/logger'
import type { Edge } from 'reactflow'
import { parseResponseFormatSafely } from '@/lib/core/utils/response-format'
import { BlockPathCalculator } from '@/lib/workflows/blocks/block-path-calculator'
import {
buildCanonicalIndex,
buildSubBlockValues,
evaluateSubBlockCondition,
getCanonicalValues,
isNonEmptyValue,
isSubBlockFeatureEnabled,
isSubBlockVisibleForMode,
resolveCanonicalMode,
} from '@/lib/workflows/subblocks/visibility'
import { getBlock } from '@/blocks'
import type { SubBlockConfig } from '@/blocks/types'
import { REFERENCE } from '@/executor/constants'
import type { SerializedBlock, SerializedWorkflow } from '@/serializer/types'
import type { BlockState, Loop, Parallel } from '@/stores/workflows/workflow/types'
import { generateLoopBlocks, generateParallelBlocks } from '@/stores/workflows/workflow/utils'
@@ -44,8 +43,7 @@ function shouldSerializeSubBlock(
displayAdvancedOptions: boolean,
isTriggerContext: boolean,
isTriggerCategory: boolean,
canonicalIndex: ReturnType<typeof buildCanonicalIndex>,
canonicalModeOverrides?: Record<string, 'basic' | 'advanced'>
canonicalIndex: ReturnType<typeof buildCanonicalIndex>
): boolean {
if (!isSubBlockFeatureEnabled(subBlockConfig)) return false
@@ -55,18 +53,15 @@ function shouldSerializeSubBlock(
return false
}
const visibleByMode = isSubBlockVisibleForMode(
subBlockConfig,
displayAdvancedOptions,
canonicalIndex,
values,
canonicalModeOverrides
)
const isCanonicalMember = Boolean(canonicalIndex.canonicalIdBySubBlockId[subBlockConfig.id])
if (isCanonicalMember) {
return evaluateSubBlockCondition(subBlockConfig.condition, values)
}
if (!visibleByMode) {
if (subBlockConfig.mode === 'advanced' && isNonEmptyValue(values[subBlockConfig.id])) {
return true
}
if (subBlockConfig.mode === 'advanced' && !displayAdvancedOptions) {
return isNonEmptyValue(values[subBlockConfig.id])
}
if (subBlockConfig.mode === 'basic' && displayAdvancedOptions) {
return false
}
@@ -224,16 +219,12 @@ export class Serializer {
// Extract parameters from UI state
const params = this.extractParams(block)
try {
const isTriggerCategory = blockConfig.category === 'triggers'
if (block.triggerMode === true || isTriggerCategory) {
params.triggerMode = true
}
if (block.advancedMode === true) {
params.advancedMode = true
}
} catch (_) {
// no-op: conservative, avoid blocking serialization if blockConfig is unexpected
const isTriggerCategory = blockConfig.category === 'triggers'
if (block.triggerMode === true || isTriggerCategory) {
params.triggerMode = true
}
if (block.advancedMode === true) {
params.advancedMode = true
}
// Validate required fields that only users can provide (before execution starts)
@@ -254,16 +245,7 @@ export class Serializer {
// For non-custom tools, we determine the tool ID
const nonCustomTools = tools.filter((tool: any) => tool.type !== 'custom-tool')
if (nonCustomTools.length > 0) {
try {
toolId = blockConfig.tools.config?.tool
? blockConfig.tools.config.tool(params)
: blockConfig.tools.access[0]
} catch (error) {
logger.warn('Tool selection failed during serialization, using default:', {
error: error instanceof Error ? error.message : String(error),
})
toolId = blockConfig.tools.access[0]
}
toolId = this.selectToolId(blockConfig, params)
}
} catch (error) {
logger.error('Error processing tools in agent block:', { error })
@@ -272,16 +254,7 @@ export class Serializer {
}
} else {
// For non-agent blocks, get tool ID from block config as usual
try {
toolId = blockConfig.tools.config?.tool
? blockConfig.tools.config.tool(params)
: blockConfig.tools.access[0]
} catch (error) {
logger.warn('Tool selection failed during serialization, using default:', {
error: error instanceof Error ? error.message : String(error),
})
toolId = blockConfig.tools.access[0]
}
toolId = this.selectToolId(blockConfig, params)
}
// Get inputs from block config
@@ -305,7 +278,10 @@ export class Serializer {
// Include response format fields if available
...(params.responseFormat
? {
responseFormat: this.parseResponseFormatSafely(params.responseFormat),
responseFormat:
parseResponseFormatSafely(params.responseFormat, block.id, {
allowReferences: true,
}) ?? undefined,
}
: {}),
},
@@ -320,52 +296,9 @@ export class Serializer {
}
}
private parseResponseFormatSafely(responseFormat: any): any {
if (!responseFormat) {
return undefined
}
// If already an object, return as-is
if (typeof responseFormat === 'object' && responseFormat !== null) {
return responseFormat
}
// Handle string values
if (typeof responseFormat === 'string') {
const trimmedValue = responseFormat.trim()
// Check for variable references like <start.input>
if (trimmedValue.startsWith(REFERENCE.START) && trimmedValue.includes(REFERENCE.END)) {
// Keep variable references as-is
return trimmedValue
}
if (trimmedValue === '') {
return undefined
}
// Try to parse as JSON
try {
return JSON.parse(trimmedValue)
} catch (error) {
// If parsing fails, return undefined to avoid crashes
// This allows the workflow to continue without structured response format
logger.warn('Failed to parse response format as JSON in serializer, using undefined:', {
value: trimmedValue,
error: error instanceof Error ? error.message : String(error),
})
return undefined
}
}
// For any other type, return undefined
return undefined
}
private extractParams(block: BlockState): Record<string, any> {
// Special handling for subflow blocks (loops, parallels, etc.)
if (block.type === 'loop' || block.type === 'parallel') {
return {} // Loop and parallel blocks don't have traditional params
return {}
}
const blockConfig = getBlock(block.type)
@@ -374,53 +307,42 @@ export class Serializer {
}
const params: Record<string, any> = {}
const displayAdvancedOptions = block.advancedMode ?? false
const legacyAdvancedMode = block.advancedMode ?? false
const canonicalModeOverrides = block.data?.canonicalModes
const isStarterBlock = block.type === 'starter'
const isAgentBlock = block.type === 'agent'
const isTriggerContext = block.triggerMode ?? false
const isTriggerCategory = blockConfig.category === 'triggers'
const canonicalIndex = buildCanonicalIndex(blockConfig.subBlocks)
const canonicalModeOverrides = block.data?.canonicalModes
const allValues = buildSubBlockValues(block.subBlocks)
// First pass: collect ALL raw values for condition evaluation
const allValues: Record<string, any> = {}
Object.entries(block.subBlocks).forEach(([id, subBlock]) => {
allValues[id] = subBlock.value
})
// Second pass: filter by mode and conditions
Object.entries(block.subBlocks).forEach(([id, subBlock]) => {
const matchingConfigs = blockConfig.subBlocks.filter((config) => config.id === id)
// Include field if it matches current mode OR if it's the starter inputFormat with values
const hasStarterInputFormatValues =
isStarterBlock &&
id === 'inputFormat' &&
Array.isArray(subBlock.value) &&
subBlock.value.length > 0
// Include legacy agent block fields (systemPrompt, userPrompt, memories) even if not in current config
// This ensures backward compatibility with old workflows that were exported before the messages array migration
const isLegacyAgentField =
isAgentBlock && ['systemPrompt', 'userPrompt', 'memories'].includes(id)
const anyConditionMet =
matchingConfigs.length === 0
? true
: matchingConfigs.some((config) =>
shouldSerializeSubBlock(
config,
allValues,
displayAdvancedOptions,
isTriggerContext,
isTriggerCategory,
canonicalIndex,
canonicalModeOverrides
)
)
const shouldInclude =
matchingConfigs.length === 0 ||
matchingConfigs.some((config) =>
shouldSerializeSubBlock(
config,
allValues,
legacyAdvancedMode,
isTriggerContext,
isTriggerCategory,
canonicalIndex
)
)
if (
(matchingConfigs.length > 0 && anyConditionMet) ||
(matchingConfigs.length > 0 && shouldInclude) ||
hasStarterInputFormatValues ||
isLegacyAgentField
) {
@@ -428,56 +350,38 @@ export class Serializer {
}
})
// Then check for any subBlocks with default values
blockConfig.subBlocks.forEach((subBlockConfig) => {
const id = subBlockConfig.id
if (
(params[id] === null || params[id] === undefined) &&
params[id] == null &&
subBlockConfig.value &&
shouldSerializeSubBlock(
subBlockConfig,
allValues,
displayAdvancedOptions,
legacyAdvancedMode,
isTriggerContext,
isTriggerCategory,
canonicalIndex,
canonicalModeOverrides
canonicalIndex
)
) {
// If the value is absent and there's a default value function, use it
params[id] = subBlockConfig.value(params)
}
})
// Finally, consolidate canonical parameters (e.g., selector and manual ID into a single param)
Object.values(canonicalIndex.groupsById).forEach((group) => {
const { basicValue, advancedValue } = getCanonicalValues(group, params)
const hasBasic = isNonEmptyValue(basicValue)
const hasAdvanced = isNonEmptyValue(advancedValue)
const basicRaw = group.basicId ? params[group.basicId] : undefined
const advancedRawValues = group.advancedIds.map((id) => params[id])
const preferredMode = resolveCanonicalMode(group, params, canonicalModeOverrides)
let chosen: unknown
if (hasAdvanced && hasBasic) {
chosen = preferredMode === 'advanced' ? advancedValue : basicValue
} else if (hasAdvanced) {
chosen = advancedValue
} else if (hasBasic) {
chosen = basicValue
} else if (
basicRaw === null &&
advancedRawValues.every((value) => value === null || value === undefined)
) {
chosen = null
}
const pairMode =
canonicalModeOverrides?.[group.canonicalId] ?? (legacyAdvancedMode ? 'advanced' : 'basic')
const chosen = pairMode === 'advanced' ? advancedValue : basicValue
const sourceIds = [group.basicId, ...group.advancedIds].filter(Boolean) as string[]
sourceIds.forEach((id) => {
if (id !== group.canonicalId) delete params[id]
})
if (chosen !== undefined) params[group.canonicalId] = chosen
else delete params[group.canonicalId]
if (chosen !== undefined) {
params[group.canonicalId] = chosen
}
})
return params
@@ -513,17 +417,7 @@ export class Serializer {
}
// Determine the current tool ID using the same logic as the serializer
let currentToolId = ''
try {
currentToolId = blockConfig.tools.config?.tool
? blockConfig.tools.config.tool(params)
: blockConfig.tools.access[0]
} catch (error) {
logger.warn('Tool selection failed during validation, using default:', {
error: error instanceof Error ? error.message : String(error),
})
currentToolId = blockConfig.tools.access[0]
}
const currentToolId = this.selectToolId(blockConfig, params)
// Get the specific tool to validate against
const currentTool = getTool(currentToolId)
@@ -531,13 +425,11 @@ export class Serializer {
return // Tool not found, skip validation
}
// Check required user-only parameters for the current tool
const missingFields: string[] = []
const displayAdvancedOptions = block.advancedMode ?? false
const isTriggerContext = block.triggerMode ?? false
const isTriggerCategory = blockConfig.category === 'triggers'
const canonicalIndex = buildCanonicalIndex(blockConfig.subBlocks || [])
const canonicalModeOverrides = block.data?.canonicalModes
// Iterate through the tool's parameters, not the block's subBlocks
Object.entries(currentTool.params || {}).forEach(([paramId, paramConfig]) => {
@@ -554,8 +446,7 @@ export class Serializer {
displayAdvancedOptions,
isTriggerContext,
isTriggerCategory,
canonicalIndex,
canonicalModeOverrides
canonicalIndex
)
const isRequired = (() => {
@@ -581,8 +472,7 @@ export class Serializer {
displayAdvancedOptions,
isTriggerContext,
isTriggerCategory,
canonicalIndex,
canonicalModeOverrides
canonicalIndex
)
)
const displayName = activeConfig?.title || paramId
@@ -637,6 +527,19 @@ export class Serializer {
return accessibleMap
}
private selectToolId(blockConfig: any, params: Record<string, any>): string {
try {
return blockConfig.tools.config?.tool
? blockConfig.tools.config.tool(params)
: blockConfig.tools.access[0]
} catch (error) {
logger.warn('Tool selection failed during serialization, using default:', {
error: error instanceof Error ? error.message : String(error),
})
return blockConfig.tools.access[0]
}
}
deserializeWorkflow(workflow: SerializedWorkflow): {
blocks: Record<string, BlockState>
edges: Edge[]

46
apps/sim/serializer/tests/serializer.extended.test.ts
View File

@@ -147,20 +147,19 @@ const { mockBlockConfigs, createMockGetBlock, slackWithCanonicalParam } = vi.hoi
config: { tool: () => 'slack_send_message' },
},
subBlocks: [
{ id: 'channel', type: 'dropdown', label: 'Channel', mode: 'basic' },
{
id: 'channel',
type: 'dropdown',
label: 'Channel',
mode: 'basic',
canonicalParamId: 'channel',
},
{
id: 'manualChannel',
type: 'short-input',
label: 'Channel ID',
mode: 'advanced',
canonicalParamId: 'targetChannel',
},
{
id: 'channelSelector',
type: 'dropdown',
label: 'Channel Selector',
mode: 'basic',
canonicalParamId: 'targetChannel',
canonicalParamId: 'channel',
},
{ id: 'text', type: 'long-input', label: 'Message' },
{ id: 'username', type: 'short-input', label: 'Username', mode: 'both' },
@@ -656,16 +655,18 @@ describe('Serializer Extended Tests', () => {
})
describe('canonical parameter handling', () => {
it('should consolidate basic/advanced mode fields into canonical param in advanced mode', () => {
it('should use advanced value when canonicalModes specifies advanced', () => {
const serializer = new Serializer()
const block: BlockState = {
id: 'slack-1',
type: 'slack',
name: 'Slack',
position: { x: 0, y: 0 },
advancedMode: true,
data: {
canonicalModes: { channel: 'advanced' },
},
subBlocks: {
channelSelector: { id: 'channelSelector', type: 'dropdown', value: 'general' },
channel: { id: 'channel', type: 'channel-selector', value: 'general' },
manualChannel: { id: 'manualChannel', type: 'short-input', value: 'C12345' },
text: { id: 'text', type: 'long-input', value: 'Hello' },
},
@@ -676,22 +677,23 @@ describe('Serializer Extended Tests', () => {
const serialized = serializer.serializeWorkflow({ 'slack-1': block }, [], {})
const slackBlock = serialized.blocks.find((b) => b.id === 'slack-1')
expect(slackBlock?.config.params.targetChannel).toBe('C12345')
expect(slackBlock?.config.params.channelSelector).toBeUndefined()
expect(slackBlock?.config.params.channel).toBe('C12345')
expect(slackBlock?.config.params.manualChannel).toBeUndefined()
})
it('should consolidate to basic value when in basic mode', () => {
it('should use basic value when canonicalModes specifies basic', () => {
const serializer = new Serializer()
const block: BlockState = {
id: 'slack-1',
type: 'slack',
name: 'Slack',
position: { x: 0, y: 0 },
advancedMode: false,
data: {
canonicalModes: { channel: 'basic' },
},
subBlocks: {
channelSelector: { id: 'channelSelector', type: 'dropdown', value: 'general' },
manualChannel: { id: 'manualChannel', type: 'short-input', value: '' },
channel: { id: 'channel', type: 'channel-selector', value: 'general' },
manualChannel: { id: 'manualChannel', type: 'short-input', value: 'C12345' },
text: { id: 'text', type: 'long-input', value: 'Hello' },
},
outputs: {},
@@ -701,7 +703,7 @@ describe('Serializer Extended Tests', () => {
const serialized = serializer.serializeWorkflow({ 'slack-1': block }, [], {})
const slackBlock = serialized.blocks.find((b) => b.id === 'slack-1')
expect(slackBlock?.config.params.targetChannel).toBe('general')
expect(slackBlock?.config.params.channel).toBe('general')
})
it('should handle missing canonical param values', () => {
@@ -711,9 +713,8 @@ describe('Serializer Extended Tests', () => {
type: 'slack',
name: 'Slack',
position: { x: 0, y: 0 },
advancedMode: false,
subBlocks: {
channelSelector: { id: 'channelSelector', type: 'dropdown', value: null },
channel: { id: 'channel', type: 'channel-selector', value: null },
manualChannel: { id: 'manualChannel', type: 'short-input', value: null },
text: { id: 'text', type: 'long-input', value: 'Hello' },
},
@@ -724,8 +725,7 @@ describe('Serializer Extended Tests', () => {
const serialized = serializer.serializeWorkflow({ 'slack-1': block }, [], {})
const slackBlock = serialized.blocks.find((b) => b.id === 'slack-1')
// When both values are null, the canonical param is set to null (preserving the null value)
expect(slackBlock?.config.params.targetChannel).toBeNull()
expect(slackBlock?.config.params.channel).toBeNull()
})
})

4
apps/sim/stores/workflows/workflow/types.ts
View File

@@ -83,6 +83,10 @@ export interface BlockState {
enabled: boolean
horizontalHandles?: boolean
height?: number
/**
* @deprecated Use `data.canonicalModes` for per-canonical-pair mode control instead.
* This field is kept for backward compatibility with older workflows.
*/
advancedMode?: boolean
triggerMode?: boolean
data?: BlockData

16
packages/testing/src/mocks/blocks.mock.ts
View File

@@ -169,8 +169,20 @@ export const mockBlockConfigs: Record<string, any> = {
config: { tool: () => 'slack_send_message' },
},
subBlocks: [
{ id: 'channel', type: 'dropdown', title: 'Channel', mode: 'basic' },
{ id: 'manualChannel', type: 'short-input', title: 'Channel ID', mode: 'advanced' },
{
id: 'channel',
type: 'dropdown',
title: 'Channel',
mode: 'basic',
canonicalParamId: 'channel',
},
{
id: 'manualChannel',
type: 'short-input',
title: 'Channel ID',
mode: 'advanced',
canonicalParamId: 'channel',
},
{ id: 'text', type: 'long-input', title: 'Message' },
{ id: 'username', type: 'short-input', title: 'Username', mode: 'both' },
],