feat(tests): added testing package, overhauled tests (#2586)

* feat(tests): added testing package, overhauled tests

* fix build

packages/testing/src/assertions/execution.assertions.ts

@@ -0,0 +1,159 @@
+import { expect } from 'vitest'
+import type { ExecutionContext } from '../types'
+
+/**
+ * Asserts that a block was executed.
+ *
+ * @example
+ * ```ts
+ * expectBlockExecuted(ctx, 'block-1')
+ * ```
+ */
+export function expectBlockExecuted(ctx: ExecutionContext, blockId: string): void {
+  expect(ctx.executedBlocks.has(blockId), `Block "${blockId}" should have been executed`).toBe(true)
+}
+
+/**
+ * Asserts that a block was NOT executed.
+ *
+ * @example
+ * ```ts
+ * expectBlockNotExecuted(ctx, 'skipped-block')
+ * ```
+ */
+export function expectBlockNotExecuted(ctx: ExecutionContext, blockId: string): void {
+  expect(ctx.executedBlocks.has(blockId), `Block "${blockId}" should not have been executed`).toBe(
+    false
+  )
+}
+
+/**
+ * Asserts that blocks were executed in a specific order.
+ *
+ * @example
+ * ```ts
+ * expectExecutionOrder(executionLog, ['start', 'step1', 'step2', 'end'])
+ * ```
+ */
+export function expectExecutionOrder(executedBlocks: string[], expectedOrder: string[]): void {
+  const actualOrder = executedBlocks.filter((id) => expectedOrder.includes(id))
+  expect(actualOrder, 'Blocks should be executed in expected order').toEqual(expectedOrder)
+}
+
+/**
+ * Asserts that a block has a specific output state.
+ *
+ * @example
+ * ```ts
+ * expectBlockOutput(ctx, 'agent-1', { response: 'Hello' })
+ * ```
+ */
+export function expectBlockOutput(
+  ctx: ExecutionContext,
+  blockId: string,
+  expectedOutput: Record<string, any>
+): void {
+  const state = ctx.blockStates.get(blockId)
+  expect(state, `Block "${blockId}" should have state`).toBeDefined()
+  expect(state).toMatchObject(expectedOutput)
+}
+
+/**
+ * Asserts that execution has a specific number of logs.
+ *
+ * @example
+ * ```ts
+ * expectLogCount(ctx, 5)
+ * ```
+ */
+export function expectLogCount(ctx: ExecutionContext, expectedCount: number): void {
+  expect(ctx.blockLogs.length, `Should have ${expectedCount} logs`).toBe(expectedCount)
+}
+
+/**
+ * Asserts that a condition decision was made.
+ *
+ * @example
+ * ```ts
+ * expectConditionDecision(ctx, 'condition-1', true)
+ * ```
+ */
+export function expectConditionDecision(
+  ctx: ExecutionContext,
+  blockId: string,
+  expectedResult: boolean
+): void {
+  const decision = ctx.decisions.condition.get(blockId)
+  expect(decision, `Condition "${blockId}" should have a decision`).toBeDefined()
+  expect(decision).toBe(expectedResult)
+}
+
+/**
+ * Asserts that a loop was completed.
+ *
+ * @example
+ * ```ts
+ * expectLoopCompleted(ctx, 'loop-1')
+ * ```
+ */
+export function expectLoopCompleted(ctx: ExecutionContext, loopId: string): void {
+  expect(ctx.completedLoops.has(loopId), `Loop "${loopId}" should be completed`).toBe(true)
+}
+
+/**
+ * Asserts that a block is in the active execution path.
+ *
+ * @example
+ * ```ts
+ * expectInActivePath(ctx, 'current-block')
+ * ```
+ */
+export function expectInActivePath(ctx: ExecutionContext, blockId: string): void {
+  expect(ctx.activeExecutionPath.has(blockId), `Block "${blockId}" should be in active path`).toBe(
+    true
+  )
+}
+
+/**
+ * Asserts that execution was cancelled.
+ *
+ * @example
+ * ```ts
+ * expectExecutionCancelled(ctx)
+ * ```
+ */
+export function expectExecutionCancelled(ctx: ExecutionContext): void {
+  expect(ctx.abortSignal?.aborted, 'Execution should be cancelled').toBe(true)
+}
+
+/**
+ * Asserts that execution was NOT cancelled.
+ *
+ * @example
+ * ```ts
+ * expectExecutionNotCancelled(ctx)
+ * ```
+ */
+export function expectExecutionNotCancelled(ctx: ExecutionContext): void {
+  expect(ctx.abortSignal?.aborted ?? false, 'Execution should not be cancelled').toBe(false)
+}
+
+/**
+ * Asserts that execution has specific environment variables.
+ *
+ * @example
+ * ```ts
+ * expectEnvironmentVariables(ctx, { API_KEY: 'test', MODE: 'production' })
+ * ```
+ */
+export function expectEnvironmentVariables(
+  ctx: ExecutionContext,
+  expectedVars: Record<string, string>
+): void {
+  Object.entries(expectedVars).forEach(([key, value]) => {
+    expect(
+      ctx.environmentVariables[key],
+      `Environment variable "${key}" should be "${value}"`
+    ).toBe(value)
+  })
+}

packages/testing/src/assertions/index.ts

@@ -0,0 +1,69 @@
+/**
+ * Custom assertions for testing workflows and execution.
+ *
+ * These provide semantic, readable assertions for common test scenarios.
+ *
+ * @example
+ * ```ts
+ * import {
+ *   expectBlockExists,
+ *   expectEdgeConnects,
+ *   expectExecutionOrder,
+ * } from '@sim/testing/assertions'
+ *
+ * // Workflow assertions
+ * expectBlockExists(workflow.blocks, 'agent-1', 'agent')
+ * expectEdgeConnects(workflow.edges, 'start', 'agent-1')
+ *
+ * // Execution assertions
+ * expectBlockExecuted(ctx, 'agent-1')
+ * expectExecutionOrder(log, ['start', 'agent-1', 'end'])
+ * ```
+ */
+
+// Execution assertions
+export {
+  expectBlockExecuted,
+  expectBlockNotExecuted,
+  expectBlockOutput,
+  expectConditionDecision,
+  expectEnvironmentVariables,
+  expectExecutionCancelled,
+  expectExecutionNotCancelled,
+  expectExecutionOrder,
+  expectInActivePath,
+  expectLogCount,
+  expectLoopCompleted,
+} from './execution.assertions'
+// Permission assertions
+export {
+  expectApiKeyInvalid,
+  expectApiKeyValid,
+  expectPermissionAllowed,
+  expectPermissionDenied,
+  expectRoleCannotPerform,
+  expectRoleCanPerform,
+  expectSocketAccessDenied,
+  expectSocketAccessGranted,
+  expectUserHasNoPermission,
+  expectUserHasPermission,
+  expectWorkflowAccessDenied,
+  expectWorkflowAccessGranted,
+} from './permission.assertions'
+// Workflow assertions
+export {
+  expectBlockCount,
+  expectBlockDisabled,
+  expectBlockEnabled,
+  expectBlockExists,
+  expectBlockHasParent,
+  expectBlockNotExists,
+  expectBlockPosition,
+  expectEdgeConnects,
+  expectEdgeCount,
+  expectEmptyWorkflow,
+  expectLinearChain,
+  expectLoopExists,
+  expectNoEdgeBetween,
+  expectParallelExists,
+} from './workflow.assertions'

packages/testing/src/assertions/permission.assertions.ts

@@ -0,0 +1,144 @@
+import { expect } from 'vitest'
+import type { PermissionType } from '../factories/permission.factory'
+
+/**
+ * Asserts that a permission check result is allowed.
+ */
+export function expectPermissionAllowed(result: { allowed: boolean; reason?: string }): void {
+  expect(result.allowed).toBe(true)
+  expect(result.reason).toBeUndefined()
+}
+
+/**
+ * Asserts that a permission check result is denied with a specific reason pattern.
+ */
+export function expectPermissionDenied(
+  result: { allowed: boolean; reason?: string },
+  reasonPattern?: string | RegExp
+): void {
+  expect(result.allowed).toBe(false)
+  expect(result.reason).toBeDefined()
+  if (reasonPattern) {
+    if (typeof reasonPattern === 'string') {
+      expect(result.reason).toContain(reasonPattern)
+    } else {
+      expect(result.reason).toMatch(reasonPattern)
+    }
+  }
+}
+
+/**
+ * Asserts that a workflow validation result indicates success.
+ */
+export function expectWorkflowAccessGranted(result: {
+  error: { message: string; status: number } | null
+  session: unknown
+  workflow: unknown
+}): void {
+  expect(result.error).toBeNull()
+  expect(result.session).not.toBeNull()
+  expect(result.workflow).not.toBeNull()
+}
+
+/**
+ * Asserts that a workflow validation result indicates access denied.
+ */
+export function expectWorkflowAccessDenied(
+  result: {
+    error: { message: string; status: number } | null
+    session: unknown
+    workflow: unknown
+  },
+  expectedStatus: 401 | 403 | 404 = 403
+): void {
+  expect(result.error).not.toBeNull()
+  expect(result.error?.status).toBe(expectedStatus)
+  expect(result.session).toBeNull()
+  expect(result.workflow).toBeNull()
+}
+
+/**
+ * Asserts that a user has a specific permission level.
+ */
+export function expectUserHasPermission(
+  permissions: Array<{ userId: string; permissionType: PermissionType }>,
+  userId: string,
+  expectedPermission: PermissionType
+): void {
+  const userPermission = permissions.find((p) => p.userId === userId)
+  expect(userPermission).toBeDefined()
+  expect(userPermission?.permissionType).toBe(expectedPermission)
+}
+
+/**
+ * Asserts that a user has no permission.
+ */
+export function expectUserHasNoPermission(
+  permissions: Array<{ userId: string; permissionType: PermissionType }>,
+  userId: string
+): void {
+  const userPermission = permissions.find((p) => p.userId === userId)
+  expect(userPermission).toBeUndefined()
+}
+
+/**
+ * Asserts that a role can perform an operation.
+ */
+export function expectRoleCanPerform(
+  checkFn: (role: string, operation: string) => { allowed: boolean },
+  role: string,
+  operation: string
+): void {
+  const result = checkFn(role, operation)
+  expect(result.allowed).toBe(true)
+}
+
+/**
+ * Asserts that a role cannot perform an operation.
+ */
+export function expectRoleCannotPerform(
+  checkFn: (role: string, operation: string) => { allowed: boolean },
+  role: string,
+  operation: string
+): void {
+  const result = checkFn(role, operation)
+  expect(result.allowed).toBe(false)
+}
+
+/**
+ * Asserts socket workflow access is granted.
+ */
+export function expectSocketAccessGranted(result: {
+  hasAccess: boolean
+  role?: string
+  workspaceId?: string
+}): void {
+  expect(result.hasAccess).toBe(true)
+  expect(result.role).toBeDefined()
+}
+
+/**
+ * Asserts socket workflow access is denied.
+ */
+export function expectSocketAccessDenied(result: {
+  hasAccess: boolean
+  role?: string
+  workspaceId?: string
+}): void {
+  expect(result.hasAccess).toBe(false)
+  expect(result.role).toBeUndefined()
+}
+
+/**
+ * Asserts API key authentication succeeded.
+ */
+export function expectApiKeyValid(result: boolean): void {
+  expect(result).toBe(true)
+}
+
+/**
+ * Asserts API key authentication failed.
+ */
+export function expectApiKeyInvalid(result: boolean): void {
+  expect(result).toBe(false)
+}

packages/testing/src/assertions/workflow.assertions.ts

@@ -0,0 +1,244 @@
+import { expect } from 'vitest'
+import type { BlockState, Edge, WorkflowState } from '../types'
+
+/**
+ * Asserts that a block exists in the workflow.
+ *
+ * @example
+ * ```ts
+ * const workflow = createLinearWorkflow(3)
+ * expectBlockExists(workflow.blocks, 'block-0')
+ * expectBlockExists(workflow.blocks, 'block-0', 'starter')
+ * ```
+ */
+export function expectBlockExists(
+  blocks: Record<string, BlockState>,
+  blockId: string,
+  expectedType?: string
+): void {
+  expect(blocks[blockId], `Block "${blockId}" should exist`).toBeDefined()
+  expect(blocks[blockId].id).toBe(blockId)
+  if (expectedType) {
+    expect(blocks[blockId].type, `Block "${blockId}" should be type "${expectedType}"`).toBe(
+      expectedType
+    )
+  }
+}
+
+/**
+ * Asserts that a block does NOT exist in the workflow.
+ *
+ * @example
+ * ```ts
+ * expectBlockNotExists(workflow.blocks, 'deleted-block')
+ * ```
+ */
+export function expectBlockNotExists(blocks: Record<string, BlockState>, blockId: string): void {
+  expect(blocks[blockId], `Block "${blockId}" should not exist`).toBeUndefined()
+}
+
+/**
+ * Asserts that an edge connects two blocks.
+ *
+ * @example
+ * ```ts
+ * expectEdgeConnects(workflow.edges, 'block-0', 'block-1')
+ * ```
+ */
+export function expectEdgeConnects(edges: Edge[], sourceId: string, targetId: string): void {
+  const edge = edges.find((e) => e.source === sourceId && e.target === targetId)
+  expect(edge, `Edge from "${sourceId}" to "${targetId}" should exist`).toBeDefined()
+}
+
+/**
+ * Asserts that no edge connects two blocks.
+ *
+ * @example
+ * ```ts
+ * expectNoEdgeBetween(workflow.edges, 'block-1', 'block-0') // No reverse edge
+ * ```
+ */
+export function expectNoEdgeBetween(edges: Edge[], sourceId: string, targetId: string): void {
+  const edge = edges.find((e) => e.source === sourceId && e.target === targetId)
+  expect(edge, `Edge from "${sourceId}" to "${targetId}" should not exist`).toBeUndefined()
+}
+
+/**
+ * Asserts that a block has a specific parent.
+ *
+ * @example
+ * ```ts
+ * expectBlockHasParent(workflow.blocks, 'child-block', 'loop-1')
+ * ```
+ */
+export function expectBlockHasParent(
+  blocks: Record<string, BlockState>,
+  childId: string,
+  expectedParentId: string
+): void {
+  const block = blocks[childId]
+  expect(block, `Child block "${childId}" should exist`).toBeDefined()
+  expect(block.data?.parentId, `Block "${childId}" should have parent "${expectedParentId}"`).toBe(
+    expectedParentId
+  )
+}
+
+/**
+ * Asserts that a workflow has a specific number of blocks.
+ *
+ * @example
+ * ```ts
+ * expectBlockCount(workflow, 5)
+ * ```
+ */
+export function expectBlockCount(workflow: WorkflowState, expectedCount: number): void {
+  const actualCount = Object.keys(workflow.blocks).length
+  expect(actualCount, `Workflow should have ${expectedCount} blocks`).toBe(expectedCount)
+}
+
+/**
+ * Asserts that a workflow has a specific number of edges.
+ *
+ * @example
+ * ```ts
+ * expectEdgeCount(workflow, 4)
+ * ```
+ */
+export function expectEdgeCount(workflow: WorkflowState, expectedCount: number): void {
+  expect(workflow.edges.length, `Workflow should have ${expectedCount} edges`).toBe(expectedCount)
+}
+
+/**
+ * Asserts that a block is at a specific position.
+ *
+ * @example
+ * ```ts
+ * expectBlockPosition(workflow.blocks, 'block-1', { x: 200, y: 0 })
+ * ```
+ */
+export function expectBlockPosition(
+  blocks: Record<string, BlockState>,
+  blockId: string,
+  expectedPosition: { x: number; y: number }
+): void {
+  const block = blocks[blockId]
+  expect(block, `Block "${blockId}" should exist`).toBeDefined()
+  expect(block.position.x, `Block "${blockId}" x position`).toBeCloseTo(expectedPosition.x, 0)
+  expect(block.position.y, `Block "${blockId}" y position`).toBeCloseTo(expectedPosition.y, 0)
+}
+
+/**
+ * Asserts that a block is enabled.
+ *
+ * @example
+ * ```ts
+ * expectBlockEnabled(workflow.blocks, 'block-1')
+ * ```
+ */
+export function expectBlockEnabled(blocks: Record<string, BlockState>, blockId: string): void {
+  const block = blocks[blockId]
+  expect(block, `Block "${blockId}" should exist`).toBeDefined()
+  expect(block.enabled, `Block "${blockId}" should be enabled`).toBe(true)
+}
+
+/**
+ * Asserts that a block is disabled.
+ *
+ * @example
+ * ```ts
+ * expectBlockDisabled(workflow.blocks, 'disabled-block')
+ * ```
+ */
+export function expectBlockDisabled(blocks: Record<string, BlockState>, blockId: string): void {
+  const block = blocks[blockId]
+  expect(block, `Block "${blockId}" should exist`).toBeDefined()
+  expect(block.enabled, `Block "${blockId}" should be disabled`).toBe(false)
+}
+
+/**
+ * Asserts that a workflow has a loop with specific configuration.
+ *
+ * @example
+ * ```ts
+ * expectLoopExists(workflow, 'loop-1', { iterations: 5, loopType: 'for' })
+ * ```
+ */
+export function expectLoopExists(
+  workflow: WorkflowState,
+  loopId: string,
+  expectedConfig?: { iterations?: number; loopType?: string; nodes?: string[] }
+): void {
+  const loop = workflow.loops[loopId]
+  expect(loop, `Loop "${loopId}" should exist`).toBeDefined()
+
+  if (expectedConfig) {
+    if (expectedConfig.iterations !== undefined) {
+      expect(loop.iterations).toBe(expectedConfig.iterations)
+    }
+    if (expectedConfig.loopType !== undefined) {
+      expect(loop.loopType).toBe(expectedConfig.loopType)
+    }
+    if (expectedConfig.nodes !== undefined) {
+      expect(loop.nodes).toEqual(expectedConfig.nodes)
+    }
+  }
+}
+
+/**
+ * Asserts that a workflow has a parallel block with specific configuration.
+ *
+ * @example
+ * ```ts
+ * expectParallelExists(workflow, 'parallel-1', { count: 3 })
+ * ```
+ */
+export function expectParallelExists(
+  workflow: WorkflowState,
+  parallelId: string,
+  expectedConfig?: { count?: number; parallelType?: string; nodes?: string[] }
+): void {
+  const parallel = workflow.parallels[parallelId]
+  expect(parallel, `Parallel "${parallelId}" should exist`).toBeDefined()
+
+  if (expectedConfig) {
+    if (expectedConfig.count !== undefined) {
+      expect(parallel.count).toBe(expectedConfig.count)
+    }
+    if (expectedConfig.parallelType !== undefined) {
+      expect(parallel.parallelType).toBe(expectedConfig.parallelType)
+    }
+    if (expectedConfig.nodes !== undefined) {
+      expect(parallel.nodes).toEqual(expectedConfig.nodes)
+    }
+  }
+}
+
+/**
+ * Asserts that the workflow state is empty.
+ *
+ * @example
+ * ```ts
+ * const workflow = createWorkflowState()
+ * expectEmptyWorkflow(workflow)
+ * ```
+ */
+export function expectEmptyWorkflow(workflow: WorkflowState): void {
+  expect(Object.keys(workflow.blocks).length, 'Workflow should have no blocks').toBe(0)
+  expect(workflow.edges.length, 'Workflow should have no edges').toBe(0)
+  expect(Object.keys(workflow.loops).length, 'Workflow should have no loops').toBe(0)
+  expect(Object.keys(workflow.parallels).length, 'Workflow should have no parallels').toBe(0)
+}
+
+/**
+ * Asserts that blocks are connected in a linear chain.
+ *
+ * @example
+ * ```ts
+ * expectLinearChain(workflow.edges, ['start', 'step1', 'step2', 'end'])
+ * ```
+ */
+export function expectLinearChain(edges: Edge[], blockIds: string[]): void {
+  for (let i = 0; i < blockIds.length - 1; i++) {
+    expectEdgeConnects(edges, blockIds[i], blockIds[i + 1])
+  }
+}