feat(executor): run from/until block (#3029)

* Run from block * Fixes * Fix * Fix * Minor improvements * Fix * Fix trace spans * Fix loop l ogs * Change ordering * Run u ntil block * Lint * Clean up * Fix * Allow run from block for triggers * Consolidation * Fix lint * Fix * Fix mock payload * Fix * Fix trigger clear snapshot * Fix loops and parallels * Fix * Cleanup * Fix test * Fix bugs * Catch error * Fix * Fix * I think it works?? * Fix * Fix * Add tests * Fix lint --------- Co-authored-by: Vikhyath Mondreti <vikhyath@simstudio.ai>
2026-02-15 08:55:05 -05:00 · 2026-01-28 12:53:23 -08:00
parent 72a2f79701
commit 655fe4f3b7
22 changed files with 3146 additions and 139 deletions
--- a/apps/sim/executor/execution/engine.ts
+++ b/apps/sim/executor/execution/engine.ts
@@ -26,6 +26,7 @@ export class ExecutionEngine {
  private allowResumeTriggers: boolean
  private cancelledFlag = false
  private errorFlag = false
+  private stoppedEarlyFlag = false
  private executionError: Error | null = null
  private lastCancellationCheck = 0
  private readonly useRedisCancellation: boolean
@@ -105,7 +106,7 @@ export class ExecutionEngine {
      this.initializeQueue(triggerBlockId)

      while (this.hasWork()) {
-        if ((await this.checkCancellation()) || this.errorFlag) {
+        if ((await this.checkCancellation()) || this.errorFlag || this.stoppedEarlyFlag) {
          break
        }
        await this.processQueue()
@@ -259,6 +260,16 @@ export class ExecutionEngine {
  }

  private initializeQueue(triggerBlockId?: string): void {
+    if (this.context.runFromBlockContext) {
+      const { startBlockId } = this.context.runFromBlockContext
+      logger.info('Initializing queue for run-from-block mode', {
+        startBlockId,
+        dirtySetSize: this.context.runFromBlockContext.dirtySet.size,
+      })
+      this.addToQueue(startBlockId)
+      return
+    }
+
    const pendingBlocks = this.context.metadata.pendingBlocks
    const remainingEdges = (this.context.metadata as any).remainingEdges

@@ -385,6 +396,17 @@ export class ExecutionEngine {
      this.finalOutput = output
    }

+    if (this.context.stopAfterBlockId === nodeId) {
+      // For loop/parallel sentinels, only stop if the subflow has fully exited (all iterations done)
+      // shouldContinue: true means more iterations, shouldExit: true means loop is done
+      const shouldContinueLoop = output.shouldContinue === true
+      if (!shouldContinueLoop) {
+        logger.info('Stopping execution after target block', { nodeId })
+        this.stoppedEarlyFlag = true
+        return
+      }
+    }
+
    const readyNodes = this.edgeManager.processOutgoingEdges(node, output, false)

    logger.info('Processing outgoing edges', {
--- a/apps/sim/executor/execution/executor.ts
+++ b/apps/sim/executor/execution/executor.ts
@@ -5,17 +5,31 @@ import { BlockExecutor } from '@/executor/execution/block-executor'
 import { EdgeManager } from '@/executor/execution/edge-manager'
 import { ExecutionEngine } from '@/executor/execution/engine'
 import { ExecutionState } from '@/executor/execution/state'
-import type { ContextExtensions, WorkflowInput } from '@/executor/execution/types'
+import type {
+  ContextExtensions,
+  SerializableExecutionState,
+  WorkflowInput,
+} from '@/executor/execution/types'
 import { createBlockHandlers } from '@/executor/handlers/registry'
 import { LoopOrchestrator } from '@/executor/orchestrators/loop'
 import { NodeExecutionOrchestrator } from '@/executor/orchestrators/node'
 import { ParallelOrchestrator } from '@/executor/orchestrators/parallel'
 import type { BlockState, ExecutionContext, ExecutionResult } from '@/executor/types'
+import {
+  computeExecutionSets,
+  type RunFromBlockContext,
+  resolveContainerToSentinelStart,
+  validateRunFromBlock,
+} from '@/executor/utils/run-from-block'
 import {
  buildResolutionFromBlock,
  buildStartBlockOutput,
  resolveExecutorStartBlock,
 } from '@/executor/utils/start-block'
+import {
+  extractLoopIdFromSentinel,
+  extractParallelIdFromSentinel,
+} from '@/executor/utils/subflow-utils'
 import { VariableResolver } from '@/executor/variables/resolver'
 import type { SerializedWorkflow } from '@/serializer/types'

@@ -48,7 +62,10 @@ export class DAGExecutor {

  async execute(workflowId: string, triggerBlockId?: string): Promise<ExecutionResult> {
    const savedIncomingEdges = this.contextExtensions.dagIncomingEdges
-    const dag = this.dagBuilder.build(this.workflow, triggerBlockId, savedIncomingEdges)
+    const dag = this.dagBuilder.build(this.workflow, {
+      triggerBlockId,
+      savedIncomingEdges,
+    })
    const { context, state } = this.createExecutionContext(workflowId, triggerBlockId)

    const resolver = new VariableResolver(this.workflow, this.workflowVariables, state)
@@ -89,17 +106,156 @@ export class DAGExecutor {
    }
  }

+  /**
+   * Execute from a specific block using cached outputs for upstream blocks.
+   */
+  async executeFromBlock(
+    workflowId: string,
+    startBlockId: string,
+    sourceSnapshot: SerializableExecutionState
+  ): Promise<ExecutionResult> {
+    // Build full DAG with all blocks to compute upstream set for snapshot filtering
+    // includeAllBlocks is needed because the startBlockId might be a trigger not reachable from the main trigger
+    const dag = this.dagBuilder.build(this.workflow, { includeAllBlocks: true })
+
+    const executedBlocks = new Set(sourceSnapshot.executedBlocks)
+    const validation = validateRunFromBlock(startBlockId, dag, executedBlocks)
+    if (!validation.valid) {
+      throw new Error(validation.error)
+    }
+
+    const { dirtySet, upstreamSet, reachableUpstreamSet } = computeExecutionSets(dag, startBlockId)
+    const effectiveStartBlockId = resolveContainerToSentinelStart(startBlockId, dag) ?? startBlockId
+
+    // Extract container IDs from sentinel IDs in reachable upstream set
+    // Use reachableUpstreamSet (not upstreamSet) to preserve sibling branch outputs
+    // Example: A->C, B->C where C references A.result || B.result
+    // When running from A, B's output should be preserved for C to reference
+    const reachableContainerIds = new Set<string>()
+    for (const nodeId of reachableUpstreamSet) {
+      const loopId = extractLoopIdFromSentinel(nodeId)
+      if (loopId) reachableContainerIds.add(loopId)
+      const parallelId = extractParallelIdFromSentinel(nodeId)
+      if (parallelId) reachableContainerIds.add(parallelId)
+    }
+
+    // Filter snapshot to include all blocks reachable from dirty blocks
+    // This preserves sibling branch outputs that dirty blocks may reference
+    const filteredBlockStates: Record<string, any> = {}
+    for (const [blockId, state] of Object.entries(sourceSnapshot.blockStates)) {
+      if (reachableUpstreamSet.has(blockId) || reachableContainerIds.has(blockId)) {
+        filteredBlockStates[blockId] = state
+      }
+    }
+    const filteredExecutedBlocks = sourceSnapshot.executedBlocks.filter(
+      (id) => reachableUpstreamSet.has(id) || reachableContainerIds.has(id)
+    )
+
+    // Filter loop/parallel executions to only include reachable containers
+    const filteredLoopExecutions: Record<string, any> = {}
+    if (sourceSnapshot.loopExecutions) {
+      for (const [loopId, execution] of Object.entries(sourceSnapshot.loopExecutions)) {
+        if (reachableContainerIds.has(loopId)) {
+          filteredLoopExecutions[loopId] = execution
+        }
+      }
+    }
+    const filteredParallelExecutions: Record<string, any> = {}
+    if (sourceSnapshot.parallelExecutions) {
+      for (const [parallelId, execution] of Object.entries(sourceSnapshot.parallelExecutions)) {
+        if (reachableContainerIds.has(parallelId)) {
+          filteredParallelExecutions[parallelId] = execution
+        }
+      }
+    }
+
+    const filteredSnapshot: SerializableExecutionState = {
+      ...sourceSnapshot,
+      blockStates: filteredBlockStates,
+      executedBlocks: filteredExecutedBlocks,
+      loopExecutions: filteredLoopExecutions,
+      parallelExecutions: filteredParallelExecutions,
+    }
+
+    logger.info('Executing from block', {
+      workflowId,
+      startBlockId,
+      effectiveStartBlockId,
+      dirtySetSize: dirtySet.size,
+      upstreamSetSize: upstreamSet.size,
+      reachableUpstreamSetSize: reachableUpstreamSet.size,
+    })
+
+    // Remove incoming edges from non-dirty sources so convergent blocks don't wait for cached upstream
+    for (const nodeId of dirtySet) {
+      const node = dag.nodes.get(nodeId)
+      if (!node) continue
+
+      const nonDirtyIncoming: string[] = []
+      for (const sourceId of node.incomingEdges) {
+        if (!dirtySet.has(sourceId)) {
+          nonDirtyIncoming.push(sourceId)
+        }
+      }
+
+      for (const sourceId of nonDirtyIncoming) {
+        node.incomingEdges.delete(sourceId)
+      }
+    }
+
+    const runFromBlockContext = { startBlockId: effectiveStartBlockId, dirtySet }
+    const { context, state } = this.createExecutionContext(workflowId, undefined, {
+      snapshotState: filteredSnapshot,
+      runFromBlockContext,
+    })
+
+    const resolver = new VariableResolver(this.workflow, this.workflowVariables, state)
+    const loopOrchestrator = new LoopOrchestrator(dag, state, resolver)
+    loopOrchestrator.setContextExtensions(this.contextExtensions)
+    const parallelOrchestrator = new ParallelOrchestrator(dag, state)
+    parallelOrchestrator.setResolver(resolver)
+    parallelOrchestrator.setContextExtensions(this.contextExtensions)
+    const allHandlers = createBlockHandlers()
+    const blockExecutor = new BlockExecutor(allHandlers, resolver, this.contextExtensions, state)
+    const edgeManager = new EdgeManager(dag)
+    loopOrchestrator.setEdgeManager(edgeManager)
+    const nodeOrchestrator = new NodeExecutionOrchestrator(
+      dag,
+      state,
+      blockExecutor,
+      loopOrchestrator,
+      parallelOrchestrator
+    )
+    const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
+
+    return await engine.run()
+  }
+
  private createExecutionContext(
    workflowId: string,
-    triggerBlockId?: string
+    triggerBlockId?: string,
+    overrides?: {
+      snapshotState?: SerializableExecutionState
+      runFromBlockContext?: RunFromBlockContext
+    }
  ): { context: ExecutionContext; state: ExecutionState } {
-    const snapshotState = this.contextExtensions.snapshotState
+    const snapshotState = overrides?.snapshotState ?? this.contextExtensions.snapshotState
    const blockStates = snapshotState?.blockStates
      ? new Map(Object.entries(snapshotState.blockStates))
      : new Map<string, BlockState>()
-    const executedBlocks = snapshotState?.executedBlocks
+    let executedBlocks = snapshotState?.executedBlocks
      ? new Set(snapshotState.executedBlocks)
      : new Set<string>()
+
+    if (overrides?.runFromBlockContext) {
+      const { dirtySet } = overrides.runFromBlockContext
+      executedBlocks = new Set([...executedBlocks].filter((id) => !dirtySet.has(id)))
+      logger.info('Cleared executed status for dirty blocks', {
+        dirtySetSize: dirtySet.size,
+        remainingExecutedBlocks: executedBlocks.size,
+      })
+    }
+
    const state = new ExecutionState(blockStates, executedBlocks)

    const context: ExecutionContext = {
@@ -109,7 +265,7 @@ export class DAGExecutor {
      userId: this.contextExtensions.userId,
      isDeployedContext: this.contextExtensions.isDeployedContext,
      blockStates: state.getBlockStates(),
-      blockLogs: snapshotState?.blockLogs ?? [],
+      blockLogs: overrides?.runFromBlockContext ? [] : (snapshotState?.blockLogs ?? []),
      metadata: {
        ...this.contextExtensions.metadata,
        startTime: new Date().toISOString(),
@@ -169,6 +325,8 @@ export class DAGExecutor {
      abortSignal: this.contextExtensions.abortSignal,
      includeFileBase64: this.contextExtensions.includeFileBase64,
      base64MaxBytes: this.contextExtensions.base64MaxBytes,
+      runFromBlockContext: overrides?.runFromBlockContext,
+      stopAfterBlockId: this.contextExtensions.stopAfterBlockId,
    }

    if (this.contextExtensions.resumeFromSnapshot) {
@@ -193,6 +351,15 @@ export class DAGExecutor {
        pendingBlocks: context.metadata.pendingBlocks,
        skipStarterBlockInit: true,
      })
+    } else if (overrides?.runFromBlockContext) {
+      // In run-from-block mode, initialize the start block only if it's a regular block
+      // Skip for sentinels/containers (loop/parallel) which aren't real blocks
+      const startBlockId = overrides.runFromBlockContext.startBlockId
+      const isRegularBlock = this.workflow.blocks.some((b) => b.id === startBlockId)
+
+      if (isRegularBlock) {
+        this.initializeStarterBlock(context, state, startBlockId)
+      }
    } else {
      this.initializeStarterBlock(context, state, triggerBlockId)
    }
--- a/apps/sim/executor/execution/types.ts
+++ b/apps/sim/executor/execution/types.ts
@@ -1,5 +1,6 @@
 import type { Edge } from 'reactflow'
 import type { BlockLog, BlockState, NormalizedBlockOutput } from '@/executor/types'
+import type { RunFromBlockContext } from '@/executor/utils/run-from-block'
 import type { SubflowType } from '@/stores/workflows/workflow/types'

 export interface ExecutionMetadata {
@@ -105,6 +106,17 @@ export interface ContextExtensions {
    output: { input?: any; output: NormalizedBlockOutput; executionTime: number },
    iterationContext?: IterationContext
  ) => Promise<void>
+
+  /**
+   * Run-from-block configuration. When provided, executor runs in partial
+   * execution mode starting from the specified block.
+   */
+  runFromBlockContext?: RunFromBlockContext
+
+  /**
+   * Stop execution after this block completes. Used for "run until block" feature.
+   */
+  stopAfterBlockId?: string
 }

 export interface WorkflowInput {