improvement(executor): redesign executor + add start block (#1790)

* fix(billing): should allow restoring subscription (#1728)

* fix(already-cancelled-sub): UI should allow restoring subscription

* restore functionality fixed

* fix

* improvement(start): revert to start block

* make it work with start block

* fix start block persistence

* cleanup triggers

* debounce status checks

* update docs

* improvement(start): revert to start block

* make it work with start block

* fix start block persistence

* cleanup triggers

* debounce status checks

* update docs

* SSE v0.1

* v0.2

* v0.3

* v0.4

* v0.5

* v0.6

* broken checkpoint

* Executor progress - everything preliminarily tested except while loops and triggers

* Executor fixes

* Fix var typing

* Implement while loop execution

* Loop and parallel result agg

* Refactor v1 - loops work

* Fix var resolution in for each loop

* Fix while loop condition and variable resolution

* Fix loop iteration counts

* Fix loop badges

* Clean logs

* Fix variable references from start block

* Fix condition block

* Fix conditional convergence

* Dont execute orphaned nodse

* Code cleanup 1 and error surfacing

* compile time try catch

* Some fixes

* Fix error throwing

* Sentinels v1

* Fix multiple start and end nodes in loop

* Edge restoration

* Fix reachable nodes execution

* Parallel subflows

* Fix loop/parallel sentinel convergence

* Loops and parallels orchestrator

* Split executor

* Variable resolution split

* Dag phase

* Refactor

* Refactor

* Refactor 3

* Lint + refactor

* Lint + cleanup + refactor

* Readability

* Initial logs

* Fix trace spans

* Console pills for iters

* Add input/output pills

* Checkpoint

* remove unused code

* THIS IS THE COMMIT THAT CAN BREAK A LOT OF THINGS

* ANOTHER BIG REFACTOR

* Lint + fix tests

* Fix webhook

* Remove comment

* Merge stash

* Fix triggers?

* Stuff

* Fix error port

* Lint

* Consolidate state

* Clean up some var resolution

* Remove some var resolution logs

* Fix chat

* Fix chat triggers

* Fix chat trigger fully

* Snapshot refactor

* Fix mcp and custom tools

* Lint

* Fix parallel default count and trace span overlay

* Agent purple

* Fix test

* Fix test

---------

Co-authored-by: Waleed <walif6@gmail.com>
Co-authored-by: Vikhyath Mondreti <vikhyathvikku@gmail.com>
Co-authored-by: Vikhyath Mondreti <vikhyath@simstudio.ai>
This commit is contained in:
Siddharth Ganesan
2025-11-02 12:21:16 -08:00
committed by GitHub
parent 7d67ae397d
commit 3bf00cbd2a
137 changed files with 8552 additions and 20440 deletions

View File

@@ -0,0 +1,319 @@
import { createLogger } from '@/lib/logs/console/logger'
import { BlockType, REFERENCE } from '@/executor/consts'
import type { ExecutionContext } from '@/executor/types'
import type { SerializedBlock, SerializedWorkflow } from '@/serializer/types'
import type { ExecutionState, LoopScope } from '../execution/state'
import { BlockResolver } from './resolvers/block'
import { EnvResolver } from './resolvers/env'
import { LoopResolver } from './resolvers/loop'
import { ParallelResolver } from './resolvers/parallel'
import type { ResolutionContext, Resolver } from './resolvers/reference'
import { WorkflowResolver } from './resolvers/workflow'
const logger = createLogger('VariableResolver')
const INVALID_REFERENCE_CHARS = /[+*/=<>!]/
function isLikelyReferenceSegment(segment: string): boolean {
if (!segment.startsWith(REFERENCE.START) || !segment.endsWith(REFERENCE.END)) {
return false
}
const inner = segment.slice(1, -1)
// Starts with space - not a reference
if (inner.startsWith(' ')) {
return false
}
// Contains only comparison operators or has operators with spaces
if (inner.match(/^\s*[<>=!]+\s*$/) || inner.match(/\s[<>=!]+\s/)) {
return false
}
// Starts with comparison operator followed by space
if (inner.match(/^[<>=!]+\s/)) {
return false
}
// For dotted references (like <block.field>)
if (inner.includes('.')) {
const dotIndex = inner.indexOf('.')
const beforeDot = inner.substring(0, dotIndex)
const afterDot = inner.substring(dotIndex + 1)
// No spaces after dot
if (afterDot.includes(' ')) {
return false
}
// No invalid chars in either part
if (INVALID_REFERENCE_CHARS.test(beforeDot) || INVALID_REFERENCE_CHARS.test(afterDot)) {
return false
}
} else if (INVALID_REFERENCE_CHARS.test(inner) || inner.match(/^\d/) || inner.match(/\s\d/)) {
// No invalid chars, doesn't start with digit, no space before digit
return false
}
return true
}
export class VariableResolver {
private resolvers: Resolver[]
private blockResolver: BlockResolver
constructor(
private workflow: SerializedWorkflow,
private workflowVariables: Record<string, any>,
private state: ExecutionState
) {
this.blockResolver = new BlockResolver(workflow)
this.resolvers = [
new LoopResolver(workflow),
new ParallelResolver(workflow),
new WorkflowResolver(workflowVariables),
new EnvResolver(),
this.blockResolver,
]
}
resolveInputs(
ctx: ExecutionContext,
currentNodeId: string,
params: Record<string, any>,
block?: SerializedBlock
): Record<string, any> {
if (!params) {
return {}
}
const resolved: Record<string, any> = {}
const isConditionBlock = block?.metadata?.id === BlockType.CONDITION
if (isConditionBlock && typeof params.conditions === 'string') {
try {
const parsed = JSON.parse(params.conditions)
if (Array.isArray(parsed)) {
resolved.conditions = parsed.map((cond: any) => ({
...cond,
value:
typeof cond.value === 'string'
? this.resolveTemplateWithoutConditionFormatting(ctx, currentNodeId, cond.value)
: cond.value,
}))
} else {
resolved.conditions = this.resolveValue(
ctx,
currentNodeId,
params.conditions,
undefined,
block
)
}
} catch (parseError) {
logger.warn('Failed to parse conditions JSON, falling back to normal resolution', {
error: parseError,
conditions: params.conditions,
})
resolved.conditions = this.resolveValue(
ctx,
currentNodeId,
params.conditions,
undefined,
block
)
}
}
for (const [key, value] of Object.entries(params)) {
if (isConditionBlock && key === 'conditions') {
continue
}
resolved[key] = this.resolveValue(ctx, currentNodeId, value, undefined, block)
}
return resolved
}
resolveSingleReference(
ctx: ExecutionContext,
currentNodeId: string,
reference: string,
loopScope?: LoopScope
): any {
return this.resolveValue(ctx, currentNodeId, reference, loopScope)
}
private resolveValue(
ctx: ExecutionContext,
currentNodeId: string,
value: any,
loopScope?: LoopScope,
block?: SerializedBlock
): any {
if (value === null || value === undefined) {
return value
}
if (Array.isArray(value)) {
return value.map((v) => this.resolveValue(ctx, currentNodeId, v, loopScope, block))
}
if (typeof value === 'object') {
return Object.entries(value).reduce(
(acc, [key, val]) => ({
...acc,
[key]: this.resolveValue(ctx, currentNodeId, val, loopScope, block),
}),
{}
)
}
if (typeof value === 'string') {
return this.resolveTemplate(ctx, currentNodeId, value, loopScope, block)
}
return value
}
private resolveTemplate(
ctx: ExecutionContext,
currentNodeId: string,
template: string,
loopScope?: LoopScope,
block?: SerializedBlock
): string {
let result = template
const resolutionContext: ResolutionContext = {
executionContext: ctx,
executionState: this.state,
currentNodeId,
loopScope,
}
const referenceRegex = new RegExp(
`${REFERENCE.START}([^${REFERENCE.END}]+)${REFERENCE.END}`,
'g'
)
let replacementError: Error | null = null
result = result.replace(referenceRegex, (match) => {
if (replacementError) return match
if (!isLikelyReferenceSegment(match)) {
return match
}
try {
const resolved = this.resolveReference(match, resolutionContext)
if (resolved === undefined) {
return match
}
const blockType = block?.metadata?.id
const isInTemplateLiteral =
blockType === BlockType.FUNCTION &&
template.includes('${') &&
template.includes('}') &&
template.includes('`')
return this.blockResolver.formatValueForBlock(resolved, blockType, isInTemplateLiteral)
} catch (error) {
replacementError = error instanceof Error ? error : new Error(String(error))
return match
}
})
if (replacementError !== null) {
throw replacementError
}
const envRegex = new RegExp(`${REFERENCE.ENV_VAR_START}([^}]+)${REFERENCE.ENV_VAR_END}`, 'g')
result = result.replace(envRegex, (match) => {
const resolved = this.resolveReference(match, resolutionContext)
return typeof resolved === 'string' ? resolved : match
})
return result
}
/**
* Resolves template string but without condition-specific formatting.
* Used when resolving condition values that are already parsed from JSON.
*/
private resolveTemplateWithoutConditionFormatting(
ctx: ExecutionContext,
currentNodeId: string,
template: string,
loopScope?: LoopScope
): string {
let result = template
const resolutionContext: ResolutionContext = {
executionContext: ctx,
executionState: this.state,
currentNodeId,
loopScope,
}
const referenceRegex = new RegExp(
`${REFERENCE.START}([^${REFERENCE.END}]+)${REFERENCE.END}`,
'g'
)
let replacementError: Error | null = null
result = result.replace(referenceRegex, (match) => {
if (replacementError) return match
if (!isLikelyReferenceSegment(match)) {
return match
}
try {
const resolved = this.resolveReference(match, resolutionContext)
if (resolved === undefined) {
return match
}
// Format value for JavaScript evaluation
// Strings need to be quoted, objects need JSON.stringify
if (typeof resolved === 'string') {
// Escape backslashes first, then single quotes, then wrap in single quotes
const escaped = resolved.replace(/\\/g, '\\\\').replace(/'/g, "\\'")
return `'${escaped}'`
}
if (typeof resolved === 'object' && resolved !== null) {
return JSON.stringify(resolved)
}
// For numbers, booleans, null, undefined - use as-is
return String(resolved)
} catch (error) {
replacementError = error instanceof Error ? error : new Error(String(error))
return match
}
})
if (replacementError !== null) {
throw replacementError
}
const envRegex = new RegExp(`${REFERENCE.ENV_VAR_START}([^}]+)${REFERENCE.ENV_VAR_END}`, 'g')
result = result.replace(envRegex, (match) => {
const resolved = this.resolveReference(match, resolutionContext)
return typeof resolved === 'string' ? resolved : match
})
return result
}
private resolveReference(reference: string, context: ResolutionContext): any {
for (const resolver of this.resolvers) {
if (resolver.canResolve(reference)) {
const result = resolver.resolve(reference, context)
logger.debug('Reference resolved', {
reference,
resolver: resolver.constructor.name,
result,
})
return result
}
}
logger.warn('No resolver found for reference', { reference })
return undefined
}
}

View File

@@ -0,0 +1,257 @@
import { createLogger } from '@/lib/logs/console/logger'
import { isReference, parseReferencePath, SPECIAL_REFERENCE_PREFIXES } from '@/executor/consts'
import type { SerializedWorkflow } from '@/serializer/types'
import { normalizeBlockName } from '@/stores/workflows/utils'
import type { ResolutionContext, Resolver } from './reference'
const logger = createLogger('BlockResolver')
export class BlockResolver implements Resolver {
private blockByNormalizedName: Map<string, string>
constructor(private workflow: SerializedWorkflow) {
this.blockByNormalizedName = new Map()
for (const block of workflow.blocks) {
this.blockByNormalizedName.set(block.id, block.id)
if (block.metadata?.name) {
const normalized = normalizeBlockName(block.metadata.name)
this.blockByNormalizedName.set(normalized, block.id)
}
}
}
canResolve(reference: string): boolean {
if (!isReference(reference)) {
return false
}
const parts = parseReferencePath(reference)
if (parts.length === 0) {
return false
}
const [type] = parts
return !SPECIAL_REFERENCE_PREFIXES.includes(type as any)
}
resolve(reference: string, context: ResolutionContext): any {
const parts = parseReferencePath(reference)
if (parts.length === 0) {
return undefined
}
const [blockName, ...pathParts] = parts
logger.debug('Resolving block reference', {
reference,
blockName,
pathParts,
})
const blockId = this.findBlockIdByName(blockName)
if (!blockId) {
logger.error('Block not found by name', { blockName, reference })
throw new Error(`Block "${blockName}" not found`)
}
const output = this.getBlockOutput(blockId, context)
logger.debug('Block output retrieved', {
blockName,
blockId,
hasOutput: !!output,
outputKeys: output ? Object.keys(output) : [],
})
if (!output) {
throw new Error(`No state found for block "${blockName}"`)
}
if (pathParts.length === 0) {
return output
}
const result = this.navigatePath(output, pathParts)
if (result === undefined) {
const availableKeys = output && typeof output === 'object' ? Object.keys(output) : []
throw new Error(
`No value found at path "${pathParts.join('.')}" in block "${blockName}". Available fields: ${availableKeys.join(', ')}`
)
}
logger.debug('Navigated path result', {
blockName,
pathParts,
result,
})
return result
}
private getBlockOutput(blockId: string, context: ResolutionContext): any {
const stateOutput = context.executionState.getBlockOutput(blockId)
if (stateOutput !== undefined) {
return stateOutput
}
const contextState = context.executionContext.blockStates?.get(blockId)
if (contextState?.output) {
return contextState.output
}
return undefined
}
private findBlockIdByName(name: string): string | undefined {
if (this.blockByNormalizedName.has(name)) {
return this.blockByNormalizedName.get(name)
}
const normalized = normalizeBlockName(name)
return this.blockByNormalizedName.get(normalized)
}
private navigatePath(obj: any, path: string[]): any {
let current = obj
for (const part of path) {
if (current === null || current === undefined) {
return undefined
}
const arrayMatch = part.match(/^([^[]+)\[(\d+)\](.*)$/)
if (arrayMatch) {
current = this.resolvePartWithIndices(current, part, '', 'block')
} else if (/^\d+$/.test(part)) {
const index = Number.parseInt(part, 10)
current = Array.isArray(current) ? current[index] : undefined
} else {
current = current[part]
}
}
return current
}
private resolvePartWithIndices(
base: any,
part: string,
fullPath: string,
sourceName: string
): any {
let value = base
const propMatch = part.match(/^([^[]+)/)
let rest = part
if (propMatch) {
const prop = propMatch[1]
value = value[prop]
rest = part.slice(prop.length)
if (value === undefined) {
throw new Error(`No value found at path "${fullPath}" in block "${sourceName}".`)
}
}
const indexRe = /^\[(\d+)\]/
while (rest.length > 0) {
const m = rest.match(indexRe)
if (!m) {
throw new Error(`Invalid path "${part}" in "${fullPath}" for block "${sourceName}".`)
}
const idx = Number.parseInt(m[1], 10)
if (!Array.isArray(value)) {
throw new Error(`Invalid path "${part}" in "${fullPath}" for block "${sourceName}".`)
}
if (idx < 0 || idx >= value.length) {
throw new Error(
`Array index ${idx} out of bounds (length: ${value.length}) in path "${part}"`
)
}
value = value[idx]
rest = rest.slice(m[0].length)
}
return value
}
formatValueForBlock(
value: any,
blockType: string | undefined,
isInTemplateLiteral = false
): string {
if (blockType === 'condition') {
return this.stringifyForCondition(value)
}
if (blockType === 'function') {
return this.formatValueForCodeContext(value, isInTemplateLiteral)
}
if (blockType === 'response') {
if (typeof value === 'string') {
return JSON.stringify(value)
}
return value
}
if (typeof value === 'object' && value !== null) {
return JSON.stringify(value)
}
return String(value)
}
private stringifyForCondition(value: any): string {
if (typeof value === 'string') {
const sanitized = value
.replace(/\\/g, '\\\\')
.replace(/"/g, '\\"')
.replace(/\n/g, '\\n')
.replace(/\r/g, '\\r')
return `"${sanitized}"`
}
if (value === null) {
return 'null'
}
if (value === undefined) {
return 'undefined'
}
if (typeof value === 'object') {
return JSON.stringify(value)
}
return String(value)
}
private formatValueForCodeContext(value: any, isInTemplateLiteral: boolean): string {
if (isInTemplateLiteral) {
if (typeof value === 'string') {
return value
}
if (typeof value === 'object' && value !== null) {
return JSON.stringify(value)
}
return String(value)
}
if (typeof value === 'string') {
return JSON.stringify(value)
}
if (typeof value === 'object' && value !== null) {
return JSON.stringify(value)
}
if (value === undefined) {
return 'undefined'
}
if (value === null) {
return 'null'
}
return String(value)
}
tryParseJSON(value: any): any {
if (typeof value !== 'string') {
return value
}
const trimmed = value.trim()
if (trimmed.length > 0 && (trimmed.startsWith('{') || trimmed.startsWith('['))) {
try {
return JSON.parse(trimmed)
} catch {
return value
}
}
return value
}
}

View File

@@ -0,0 +1,22 @@
import { createLogger } from '@/lib/logs/console/logger'
import { extractEnvVarName, isEnvVarReference } from '@/executor/consts'
import type { ResolutionContext, Resolver } from './reference'
const logger = createLogger('EnvResolver')
export class EnvResolver implements Resolver {
canResolve(reference: string): boolean {
return isEnvVarReference(reference)
}
resolve(reference: string, context: ResolutionContext): any {
const varName = extractEnvVarName(reference)
const value = context.executionContext.environmentVariables?.[varName]
if (value === undefined) {
logger.debug('Environment variable not found, returning original reference', { varName })
return reference
}
return value
}
}

View File

@@ -0,0 +1,73 @@
import { createLogger } from '@/lib/logs/console/logger'
import { isReference, parseReferencePath, REFERENCE } from '@/executor/consts'
import { extractBaseBlockId } from '@/executor/utils/subflow-utils'
import type { SerializedWorkflow } from '@/serializer/types'
import type { ResolutionContext, Resolver } from './reference'
const logger = createLogger('LoopResolver')
export class LoopResolver implements Resolver {
constructor(private workflow: SerializedWorkflow) {}
canResolve(reference: string): boolean {
if (!isReference(reference)) {
return false
}
const parts = parseReferencePath(reference)
if (parts.length === 0) {
return false
}
const [type] = parts
return type === REFERENCE.PREFIX.LOOP
}
resolve(reference: string, context: ResolutionContext): any {
const parts = parseReferencePath(reference)
if (parts.length < 2) {
logger.warn('Invalid loop reference - missing property', { reference })
return undefined
}
const [_, property] = parts
let loopScope = context.loopScope
if (!loopScope) {
const loopId = this.findLoopForBlock(context.currentNodeId)
if (!loopId) {
logger.debug('Block not in a loop', { nodeId: context.currentNodeId })
return undefined
}
loopScope = context.executionState.getLoopScope(loopId)
}
if (!loopScope) {
logger.warn('Loop scope not found', { reference })
return undefined
}
switch (property) {
case 'iteration':
case 'index':
return loopScope.iteration
case 'item':
case 'currentItem':
return loopScope.item
case 'items':
return loopScope.items
default:
logger.warn('Unknown loop property', { property })
return undefined
}
}
private findLoopForBlock(blockId: string): string | undefined {
const baseId = extractBaseBlockId(blockId)
for (const loopId of Object.keys(this.workflow.loops || {})) {
const loopConfig = this.workflow.loops[loopId]
if (loopConfig.nodes.includes(baseId)) {
return loopId
}
}
return undefined
}
}

View File

@@ -0,0 +1,100 @@
import { createLogger } from '@/lib/logs/console/logger'
import { isReference, parseReferencePath, REFERENCE } from '@/executor/consts'
import { extractBaseBlockId, extractBranchIndex } from '@/executor/utils/subflow-utils'
import type { SerializedWorkflow } from '@/serializer/types'
import type { ResolutionContext, Resolver } from './reference'
const logger = createLogger('ParallelResolver')
export class ParallelResolver implements Resolver {
constructor(private workflow: SerializedWorkflow) {}
canResolve(reference: string): boolean {
if (!isReference(reference)) {
return false
}
const parts = parseReferencePath(reference)
if (parts.length === 0) {
return false
}
const [type] = parts
return type === REFERENCE.PREFIX.PARALLEL
}
resolve(reference: string, context: ResolutionContext): any {
const parts = parseReferencePath(reference)
if (parts.length < 2) {
logger.warn('Invalid parallel reference - missing property', { reference })
return undefined
}
const [_, property] = parts
const parallelId = this.findParallelForBlock(context.currentNodeId)
if (!parallelId) {
logger.debug('Block not in a parallel', { nodeId: context.currentNodeId })
return undefined
}
const parallelConfig = this.workflow.parallels?.[parallelId]
if (!parallelConfig) {
logger.warn('Parallel config not found', { parallelId })
return undefined
}
const branchIndex = extractBranchIndex(context.currentNodeId)
if (branchIndex === null) {
logger.debug('Node ID does not have branch index', { nodeId: context.currentNodeId })
return undefined
}
const distributionItems = this.getDistributionItems(parallelConfig)
switch (property) {
case 'index':
return branchIndex
case 'currentItem':
if (Array.isArray(distributionItems)) {
return distributionItems[branchIndex]
}
if (typeof distributionItems === 'object' && distributionItems !== null) {
const keys = Object.keys(distributionItems)
const key = keys[branchIndex]
return key !== undefined ? distributionItems[key] : undefined
}
return undefined
case 'items':
return distributionItems
default:
logger.warn('Unknown parallel property', { property })
return undefined
}
}
private findParallelForBlock(blockId: string): string | undefined {
const baseId = extractBaseBlockId(blockId)
if (!this.workflow.parallels) {
return undefined
}
for (const parallelId of Object.keys(this.workflow.parallels)) {
const parallelConfig = this.workflow.parallels[parallelId]
if (parallelConfig?.nodes.includes(baseId)) {
return parallelId
}
}
return undefined
}
private getDistributionItems(parallelConfig: any): any {
let distributionItems = parallelConfig.distributionItems || parallelConfig.distribution || []
if (typeof distributionItems === 'string' && !distributionItems.startsWith('<')) {
try {
distributionItems = JSON.parse(distributionItems.replace(/'/g, '"'))
} catch (e) {
logger.error('Failed to parse distribution items', { distributionItems })
return []
}
}
return distributionItems
}
}

View File

@@ -0,0 +1,13 @@
import type { ExecutionContext } from '@/executor/types'
import type { ExecutionState, LoopScope } from '../../execution/state'
export interface ResolutionContext {
executionContext: ExecutionContext
executionState: ExecutionState
currentNodeId: string
loopScope?: LoopScope
}
export interface Resolver {
canResolve(reference: string): boolean
resolve(reference: string, context: ResolutionContext): any
}

View File

@@ -0,0 +1,49 @@
import { createLogger } from '@/lib/logs/console/logger'
import { isReference, parseReferencePath, REFERENCE } from '@/executor/consts'
import type { ResolutionContext, Resolver } from './reference'
const logger = createLogger('WorkflowResolver')
export class WorkflowResolver implements Resolver {
constructor(private workflowVariables: Record<string, any>) {}
canResolve(reference: string): boolean {
if (!isReference(reference)) {
return false
}
const parts = parseReferencePath(reference)
if (parts.length === 0) {
return false
}
const [type] = parts
return type === REFERENCE.PREFIX.VARIABLE
}
resolve(reference: string, context: ResolutionContext): any {
const parts = parseReferencePath(reference)
if (parts.length < 2) {
logger.warn('Invalid variable reference - missing variable name', { reference })
return undefined
}
const [_, variableName] = parts
if (context.executionContext.workflowVariables) {
for (const varObj of Object.values(context.executionContext.workflowVariables)) {
const v = varObj as any
if (v.name === variableName || v.id === variableName) {
return v.value
}
}
}
for (const varObj of Object.values(this.workflowVariables)) {
const v = varObj as any
if (v.name === variableName || v.id === variableName) {
return v.value
}
}
logger.debug('Workflow variable not found', { variableName })
return undefined
}
}