improvement(serializer): canonical subblock, serialization cleanups, schedules/webhooks are deployment version friendly (#2848)

* hide form deployment tab from docs

* progress

* fix resolution

* cleanup code

* fix positioning

* cleanup dead sockets adv mode ops

* address greptile comments

* fix tests plus more simplification

* fix cleanup

* bring back advanced mode with specific definition

* revert feature flags

* improvement(subblock): ui

* resolver change to make all var references optional chaining

* fix(webhooks/schedules): deployment version friendly

* fix tests

* fix credential sets with new lifecycle

* prep merge

* add back migration

* fix display check for adv fields

* fix trigger vs block scoping

---------

Co-authored-by: Emir Karabeg <emirkarabeg@berkeley.edu>
This commit is contained in:
Vikhyath Mondreti
2026-01-16 15:23:43 -08:00
committed by GitHub
parent ce3ddb6ba0
commit 78e4ca9d45
70 changed files with 12806 additions and 1011 deletions

View File

@@ -1,16 +1,17 @@
import {
buildCanonicalIndex,
buildSubBlockValues,
evaluateSubBlockCondition,
hasAdvancedValues,
isSubBlockFeatureEnabled,
isSubBlockVisibleForMode,
type SubBlockCondition,
} from '@/lib/workflows/subblocks/visibility'
import { getBlock } from '@/blocks/registry'
import type { SubBlockConfig } from '@/blocks/types'
import { AuthMode } from '@/blocks/types'
import type { BlockState, SubBlockState, WorkflowState } from '@/stores/workflows/workflow/types'
/** Condition type for SubBlock visibility - mirrors the inline type from blocks/types.ts */
interface SubBlockCondition {
field: string
value: string | number | boolean | Array<string | number | boolean> | undefined
not?: boolean
and?: SubBlockCondition
}
// Credential types based on actual patterns in the codebase
export enum CredentialType {
OAUTH = 'oauth',
@@ -117,36 +118,32 @@ export function extractRequiredCredentials(
/** Helper to check visibility, respecting mode and conditions */
function isSubBlockVisible(block: BlockState, subBlockConfig: SubBlockConfig): boolean {
const mode = subBlockConfig.mode ?? 'both'
if (mode === 'trigger' && !block?.triggerMode) return false
if (mode === 'basic' && block?.advancedMode) return false
if (mode === 'advanced' && !block?.advancedMode) return false
if (!isSubBlockFeatureEnabled(subBlockConfig)) return false
if (!subBlockConfig.condition) return true
const values = buildSubBlockValues(block?.subBlocks || {})
const blockConfig = getBlock(block.type)
const blockSubBlocks = blockConfig?.subBlocks || []
const canonicalIndex = buildCanonicalIndex(blockSubBlocks)
const effectiveAdvanced =
(block?.advancedMode ?? false) || hasAdvancedValues(blockSubBlocks, values, canonicalIndex)
const canonicalModeOverrides = block.data?.canonicalModes
const condition =
typeof subBlockConfig.condition === 'function'
? subBlockConfig.condition()
: subBlockConfig.condition
if (subBlockConfig.mode === 'trigger' && !block?.triggerMode) return false
if (block?.triggerMode && subBlockConfig.mode && subBlockConfig.mode !== 'trigger') return false
const evaluate = (cond: SubBlockCondition): boolean => {
const currentValue = block?.subBlocks?.[cond.field]?.value
const expected = cond.value
let match =
expected === undefined
? true
: Array.isArray(expected)
? expected.includes(currentValue as string)
: currentValue === expected
if (cond.not) match = !match
if (cond.and) match = match && evaluate(cond.and)
return match
if (
!isSubBlockVisibleForMode(
subBlockConfig,
effectiveAdvanced,
canonicalIndex,
values,
canonicalModeOverrides
)
) {
return false
}
return evaluate(condition)
return evaluateSubBlockCondition(subBlockConfig.condition as SubBlockCondition, values)
}
// Sort: OAuth first, then secrets, alphabetically within each type

View File

@@ -6,6 +6,7 @@
import { createLogger } from '@sim/logger'
import type { Edge } from 'reactflow'
import { z } from 'zod'
import { parseResponseFormatSafely } from '@/lib/core/utils/response-format'
import { getPersonalAndWorkspaceEnv } from '@/lib/environment/utils'
import { clearExecutionCancellation } from '@/lib/execution/cancellation'
import type { LoggingSession } from '@/lib/logs/execution/logging-session'
@@ -17,7 +18,6 @@ import {
import { TriggerUtils } from '@/lib/workflows/triggers/triggers'
import { updateWorkflowRunCounts } from '@/lib/workflows/utils'
import { Executor } from '@/executor'
import { REFERENCE } from '@/executor/constants'
import type { ExecutionSnapshot } from '@/executor/execution/snapshot'
import type {
ContextExtensions,
@@ -25,7 +25,7 @@ import type {
IterationContext,
} from '@/executor/execution/types'
import type { ExecutionResult, NormalizedBlockOutput } from '@/executor/types'
import { createEnvVarPattern } from '@/executor/utils/reference-validation'
import { resolveEnvVarReferences } from '@/executor/utils/reference-validation'
import { Serializer } from '@/serializer'
import { mergeSubblockState } from '@/stores/workflows/server-utils'
@@ -203,25 +203,13 @@ export async function executeWorkflowCore(
(subAcc, [key, subBlock]) => {
let value = subBlock.value
if (
typeof value === 'string' &&
value.includes(REFERENCE.ENV_VAR_START) &&
value.includes(REFERENCE.ENV_VAR_END)
) {
const envVarPattern = createEnvVarPattern()
const matches = value.match(envVarPattern)
if (matches) {
for (const match of matches) {
const varName = match.slice(
REFERENCE.ENV_VAR_START.length,
-REFERENCE.ENV_VAR_END.length
)
const decryptedValue = decryptedEnvVars[varName]
if (decryptedValue !== undefined) {
value = (value as string).replace(match, decryptedValue)
}
}
}
if (typeof value === 'string') {
value = resolveEnvVarReferences(value, decryptedEnvVars, {
resolveExactMatch: false,
trimKeys: false,
onMissing: 'keep',
deep: false,
}) as string
}
subAcc[key] = value
@@ -237,26 +225,16 @@ export async function executeWorkflowCore(
// Process response format
const processedBlockStates = Object.entries(currentBlockStates).reduce(
(acc, [blockId, blockState]) => {
if (blockState.responseFormat && typeof blockState.responseFormat === 'string') {
const responseFormatValue = blockState.responseFormat.trim()
if (responseFormatValue && !responseFormatValue.startsWith(REFERENCE.START)) {
try {
acc[blockId] = {
...blockState,
responseFormat: JSON.parse(responseFormatValue),
}
} catch {
acc[blockId] = {
...blockState,
responseFormat: undefined,
}
}
} else {
acc[blockId] = blockState
}
} else {
const responseFormatValue = blockState.responseFormat
if (responseFormatValue === undefined || responseFormatValue === null) {
acc[blockId] = blockState
return acc
}
const responseFormat = parseResponseFormatSafely(responseFormatValue, blockId, {
allowReferences: true,
})
acc[blockId] = { ...blockState, responseFormat: responseFormat ?? undefined }
return acc
},
{} as Record<string, Record<string, any>>

View File

@@ -0,0 +1,51 @@
import { createLogger } from '@sim/logger'
import {
ensureBlockEnvVarsResolvable,
ensureEnvVarsDecryptable,
getPersonalAndWorkspaceEnv,
} from '@/lib/environment/utils'
import { loadDeployedWorkflowState } from '@/lib/workflows/persistence/utils'
import { mergeSubblockState } from '@/stores/workflows/server-utils'
const logger = createLogger('ExecutionPreflight')
export interface EnvVarPreflightOptions {
workflowId: string
workspaceId: string
envUserId: string
requestId?: string
}
/**
* Preflight env var checks to avoid scheduling executions that will fail.
* Always uses deployed workflow state since preflight is only done for async
* executions which always run on deployed state.
*/
export async function preflightWorkflowEnvVars({
workflowId,
workspaceId,
envUserId,
requestId,
}: EnvVarPreflightOptions): Promise<void> {
const workflowData = await loadDeployedWorkflowState(workflowId)
if (!workflowData) {
throw new Error('Workflow state not found')
}
const mergedStates = mergeSubblockState(workflowData.blocks)
const { personalEncrypted, workspaceEncrypted } = await getPersonalAndWorkspaceEnv(
envUserId,
workspaceId
)
const variables = { ...personalEncrypted, ...workspaceEncrypted }
await ensureBlockEnvVarsResolvable(mergedStates, variables, { requestId })
await ensureEnvVarsDecryptable(variables, { requestId })
if (requestId) {
logger.debug(`[${requestId}] Env var preflight passed`, { workflowId })
} else {
logger.debug('Env var preflight passed', { workflowId })
}
}

View File

@@ -495,6 +495,7 @@ export async function deployWorkflow(params: {
}): Promise<{
success: boolean
version?: number
deploymentVersionId?: string
deployedAt?: Date
currentState?: any
error?: string
@@ -533,6 +534,7 @@ export async function deployWorkflow(params: {
.where(eq(workflowDeploymentVersion.workflowId, workflowId))
const nextVersion = Number(maxVersion) + 1
const deploymentVersionId = uuidv4()
// Deactivate all existing versions
await tx
@@ -542,7 +544,7 @@ export async function deployWorkflow(params: {
// Create new deployment version
await tx.insert(workflowDeploymentVersion).values({
id: uuidv4(),
id: deploymentVersionId,
workflowId,
version: nextVersion,
state: currentState,
@@ -562,10 +564,10 @@ export async function deployWorkflow(params: {
// Note: Templates are NOT automatically updated on deployment
// Template updates must be done explicitly through the "Update Template" button
return nextVersion
return { version: nextVersion, deploymentVersionId }
})
logger.info(`Deployed workflow ${workflowId} as v${deployedVersion}`)
logger.info(`Deployed workflow ${workflowId} as v${deployedVersion.version}`)
if (workflowName) {
try {
@@ -582,7 +584,7 @@ export async function deployWorkflow(params: {
workflowName,
blocksCount: Object.keys(currentState.blocks).length,
edgesCount: currentState.edges.length,
version: deployedVersion,
version: deployedVersion.version,
loopsCount: Object.keys(currentState.loops).length,
parallelsCount: Object.keys(currentState.parallels).length,
blockTypes: JSON.stringify(blockTypeCounts),
@@ -594,7 +596,8 @@ export async function deployWorkflow(params: {
return {
success: true,
version: deployedVersion,
version: deployedVersion.version,
deploymentVersionId: deployedVersion.deploymentVersionId,
deployedAt: now,
currentState,
}

View File

@@ -35,11 +35,18 @@ vi.mock('@sim/db', () => ({
workflowSchedule: {
workflowId: 'workflow_id',
blockId: 'block_id',
deploymentVersionId: 'deployment_version_id',
},
}))
vi.mock('drizzle-orm', () => ({
eq: vi.fn((...args) => ({ type: 'eq', args })),
and: vi.fn((...args) => ({ type: 'and', args })),
sql: vi.fn((strings, ...values) => ({ type: 'sql', strings, values })),
}))
vi.mock('@/lib/webhooks/deploy', () => ({
cleanupWebhooksForWorkflow: vi.fn().mockResolvedValue(undefined),
}))
vi.mock('@sim/logger', () => loggerMock)

View File

@@ -1,7 +1,8 @@
import { workflowSchedule } from '@sim/db'
import { db, workflowSchedule } from '@sim/db'
import { createLogger } from '@sim/logger'
import { eq } from 'drizzle-orm'
import { and, eq } from 'drizzle-orm'
import type { DbOrTx } from '@/lib/db/types'
import { cleanupWebhooksForWorkflow } from '@/lib/webhooks/deploy'
import type { BlockState } from '@/lib/workflows/schedules/utils'
import { findScheduleBlocks, validateScheduleBlock } from '@/lib/workflows/schedules/validation'
@@ -26,7 +27,8 @@ export interface ScheduleDeployResult {
export async function createSchedulesForDeploy(
workflowId: string,
blocks: Record<string, BlockState>,
tx: DbOrTx
tx: DbOrTx,
deploymentVersionId?: string
): Promise<ScheduleDeployResult> {
const scheduleBlocks = findScheduleBlocks(blocks)
@@ -61,6 +63,7 @@ export async function createSchedulesForDeploy(
const values = {
id: scheduleId,
workflowId,
deploymentVersionId: deploymentVersionId || null,
blockId,
cronExpression: cronExpression!,
triggerType: 'schedule',
@@ -75,6 +78,7 @@ export async function createSchedulesForDeploy(
const setValues = {
blockId,
cronExpression: cronExpression!,
...(deploymentVersionId ? { deploymentVersionId } : {}),
updatedAt: now,
nextRunAt: nextRunAt!,
timezone: timezone!,
@@ -86,7 +90,11 @@ export async function createSchedulesForDeploy(
.insert(workflowSchedule)
.values(values)
.onConflictDoUpdate({
target: [workflowSchedule.workflowId, workflowSchedule.blockId],
target: [
workflowSchedule.workflowId,
workflowSchedule.blockId,
workflowSchedule.deploymentVersionId,
],
set: setValues,
})
@@ -109,8 +117,36 @@ export async function createSchedulesForDeploy(
* Delete all schedules for a workflow
* This should be called within a database transaction during undeploy
*/
export async function deleteSchedulesForWorkflow(workflowId: string, tx: DbOrTx): Promise<void> {
await tx.delete(workflowSchedule).where(eq(workflowSchedule.workflowId, workflowId))
export async function deleteSchedulesForWorkflow(
workflowId: string,
tx: DbOrTx,
deploymentVersionId?: string
): Promise<void> {
await tx
.delete(workflowSchedule)
.where(
deploymentVersionId
? and(
eq(workflowSchedule.workflowId, workflowId),
eq(workflowSchedule.deploymentVersionId, deploymentVersionId)
)
: eq(workflowSchedule.workflowId, workflowId)
)
logger.info(`Deleted all schedules for workflow ${workflowId}`)
logger.info(
deploymentVersionId
? `Deleted schedules for workflow ${workflowId} deployment ${deploymentVersionId}`
: `Deleted all schedules for workflow ${workflowId}`
)
}
export async function cleanupDeploymentVersion(params: {
workflowId: string
workflow: Record<string, unknown>
requestId: string
deploymentVersionId: string
}): Promise<void> {
const { workflowId, workflow, requestId, deploymentVersionId } = params
await cleanupWebhooksForWorkflow(workflowId, workflow, requestId, deploymentVersionId)
await deleteSchedulesForWorkflow(workflowId, db, deploymentVersionId)
}

View File

@@ -1,4 +1,5 @@
export {
cleanupDeploymentVersion,
createSchedulesForDeploy,
deleteSchedulesForWorkflow,
type ScheduleDeployResult,

View File

@@ -0,0 +1,269 @@
import { getEnv, isTruthy } from '@/lib/core/config/env'
import type { SubBlockConfig } from '@/blocks/types'
export type CanonicalMode = 'basic' | 'advanced'
export interface CanonicalGroup {
canonicalId: string
basicId?: string
advancedIds: string[]
}
export interface CanonicalIndex {
groupsById: Record<string, CanonicalGroup>
canonicalIdBySubBlockId: Record<string, string>
}
export interface SubBlockCondition {
field: string
value: string | number | boolean | Array<string | number | boolean> | undefined
not?: boolean
and?: SubBlockCondition
}
export interface CanonicalModeOverrides {
[canonicalId: string]: CanonicalMode | undefined
}
export interface CanonicalValueSelection {
basicValue: unknown
advancedValue: unknown
advancedSourceId?: string
}
/**
* Build a flat map of subblock values keyed by subblock id.
*/
export function buildSubBlockValues(
subBlocks: Record<string, { value?: unknown } | null | undefined>
): Record<string, unknown> {
return Object.entries(subBlocks).reduce<Record<string, unknown>>((acc, [key, subBlock]) => {
acc[key] = subBlock?.value
return acc
}, {})
}
/**
* Build canonical group indices for a block's subblocks.
*/
export function buildCanonicalIndex(subBlocks: SubBlockConfig[]): CanonicalIndex {
const groupsById: Record<string, CanonicalGroup> = {}
const canonicalIdBySubBlockId: Record<string, string> = {}
subBlocks.forEach((subBlock) => {
if (!subBlock.canonicalParamId) return
const canonicalId = subBlock.canonicalParamId
if (!groupsById[canonicalId]) {
groupsById[canonicalId] = { canonicalId, advancedIds: [] }
}
const group = groupsById[canonicalId]
if (subBlock.mode === 'advanced') {
group.advancedIds.push(subBlock.id)
} else {
group.basicId = subBlock.id
}
canonicalIdBySubBlockId[subBlock.id] = canonicalId
})
return { groupsById, canonicalIdBySubBlockId }
}
/**
* Resolve if a canonical group is a swap pair (basic + advanced).
*/
export function isCanonicalPair(group?: CanonicalGroup): boolean {
return Boolean(group?.basicId && group?.advancedIds?.length)
}
/**
* Determine the active mode for a canonical group.
*/
export function resolveCanonicalMode(
group: CanonicalGroup,
values: Record<string, unknown>,
overrides?: CanonicalModeOverrides
): CanonicalMode {
const override = overrides?.[group.canonicalId]
if (override === 'advanced' && group.advancedIds.length > 0) return 'advanced'
if (override === 'basic' && group.basicId) return 'basic'
const { basicValue, advancedValue } = getCanonicalValues(group, values)
const hasBasic = isNonEmptyValue(basicValue)
const hasAdvanced = isNonEmptyValue(advancedValue)
if (!group.basicId) return 'advanced'
if (!hasBasic && hasAdvanced) return 'advanced'
return 'basic'
}
/**
* Evaluate a subblock condition against a map of raw values.
*/
export function evaluateSubBlockCondition(
condition: SubBlockCondition | (() => SubBlockCondition) | undefined,
values: Record<string, unknown>
): boolean {
if (!condition) return true
const actual = typeof condition === 'function' ? condition() : condition
const fieldValue = values[actual.field]
const valueMatch = Array.isArray(actual.value)
? fieldValue != null &&
(actual.not
? !actual.value.includes(fieldValue as any)
: actual.value.includes(fieldValue as any))
: actual.not
? fieldValue !== actual.value
: fieldValue === actual.value
const andMatch = !actual.and
? true
: (() => {
const andFieldValue = values[actual.and!.field]
const andValueMatch = Array.isArray(actual.and!.value)
? andFieldValue != null &&
(actual.and!.not
? !actual.and!.value.includes(andFieldValue as any)
: actual.and!.value.includes(andFieldValue as any))
: actual.and!.not
? andFieldValue !== actual.and!.value
: andFieldValue === actual.and!.value
return andValueMatch
})()
return valueMatch && andMatch
}
/**
* Check if a value is considered set for advanced visibility/selection.
*/
export function isNonEmptyValue(value: unknown): boolean {
if (value === null || value === undefined) return false
if (typeof value === 'string') return value.trim().length > 0
if (Array.isArray(value)) return value.length > 0
return true
}
/**
* Resolve basic and advanced values for a canonical group.
*/
export function getCanonicalValues(
group: CanonicalGroup,
values: Record<string, unknown>
): CanonicalValueSelection {
const basicValue = group.basicId ? values[group.basicId] : undefined
let advancedValue: unknown
let advancedSourceId: string | undefined
group.advancedIds.forEach((advancedId) => {
if (advancedValue !== undefined) return
const candidate = values[advancedId]
if (isNonEmptyValue(candidate)) {
advancedValue = candidate
advancedSourceId = advancedId
}
})
return { basicValue, advancedValue, advancedSourceId }
}
/**
* Check if a block has any standalone advanced-only fields (not part of canonical pairs).
* These require the block-level advanced mode toggle to be visible.
*/
export function hasStandaloneAdvancedFields(
subBlocks: SubBlockConfig[],
canonicalIndex: CanonicalIndex
): boolean {
for (const subBlock of subBlocks) {
if (subBlock.mode !== 'advanced') continue
if (!canonicalIndex.canonicalIdBySubBlockId[subBlock.id]) return true
}
return false
}
/**
* Check if any advanced-only or canonical advanced values are present.
*/
export function hasAdvancedValues(
subBlocks: SubBlockConfig[],
values: Record<string, unknown>,
canonicalIndex: CanonicalIndex
): boolean {
const checkedCanonical = new Set<string>()
for (const subBlock of subBlocks) {
const canonicalId = canonicalIndex.canonicalIdBySubBlockId[subBlock.id]
if (canonicalId) {
const group = canonicalIndex.groupsById[canonicalId]
if (group && isCanonicalPair(group) && !checkedCanonical.has(canonicalId)) {
checkedCanonical.add(canonicalId)
const { advancedValue } = getCanonicalValues(group, values)
if (isNonEmptyValue(advancedValue)) return true
}
continue
}
if (subBlock.mode === 'advanced' && isNonEmptyValue(values[subBlock.id])) {
return true
}
}
return false
}
/**
* Determine whether a subblock is visible based on mode and canonical swaps.
*/
export function isSubBlockVisibleForMode(
subBlock: SubBlockConfig,
displayAdvancedOptions: boolean,
canonicalIndex: CanonicalIndex,
values: Record<string, unknown>,
overrides?: CanonicalModeOverrides
): boolean {
const canonicalId = canonicalIndex.canonicalIdBySubBlockId[subBlock.id]
const group = canonicalId ? canonicalIndex.groupsById[canonicalId] : undefined
if (group && isCanonicalPair(group)) {
const mode = resolveCanonicalMode(group, values, overrides)
if (mode === 'advanced') return group.advancedIds.includes(subBlock.id)
return group.basicId === subBlock.id
}
if (subBlock.mode === 'basic' && displayAdvancedOptions) return false
if (subBlock.mode === 'advanced' && !displayAdvancedOptions) return false
return true
}
/**
* Resolve the dependency value for a dependsOn key, honoring canonical swaps.
*/
export function resolveDependencyValue(
dependencyKey: string,
values: Record<string, unknown>,
canonicalIndex: CanonicalIndex,
overrides?: CanonicalModeOverrides
): unknown {
const canonicalId =
canonicalIndex.groupsById[dependencyKey]?.canonicalId ||
canonicalIndex.canonicalIdBySubBlockId[dependencyKey]
if (!canonicalId) {
return values[dependencyKey]
}
const group = canonicalIndex.groupsById[canonicalId]
if (!group) return values[dependencyKey]
const { basicValue, advancedValue } = getCanonicalValues(group, values)
const mode = resolveCanonicalMode(group, values, overrides)
if (mode === 'advanced') return advancedValue ?? basicValue
return basicValue ?? advancedValue
}
/**
* Check if a subblock is gated by a feature flag.
*/
export function isSubBlockFeatureEnabled(subBlock: SubBlockConfig): boolean {
if (!subBlock.requiresFeature) return true
return isTruthy(getEnv(subBlock.requiresFeature))
}