improvement(serializer): canonical subblock, serialization cleanups, schedules/webhooks are deployment version friendly (#2848)

* hide form deployment tab from docs

* progress

* fix resolution

* cleanup code

* fix positioning

* cleanup dead sockets adv mode ops

* address greptile comments

* fix tests plus more simplification

* fix cleanup

* bring back advanced mode with specific definition

* revert feature flags

* improvement(subblock): ui

* resolver change to make all var references optional chaining

* fix(webhooks/schedules): deployment version friendly

* fix tests

* fix credential sets with new lifecycle

* prep merge

* add back migration

* fix display check for adv fields

* fix trigger vs block scoping

---------

Co-authored-by: Emir Karabeg <emirkarabeg@berkeley.edu>

apps/sim/executor/utils/permission-check.ts

@@ -3,6 +3,7 @@ import { member, permissionGroup, permissionGroupMember } from '@sim/db/schema'
 import { createLogger } from '@sim/logger'
 import { and, eq } from 'drizzle-orm'
 import { isOrganizationOnEnterprisePlan } from '@/lib/billing'
+import { isAccessControlEnabled, isHosted } from '@/lib/core/config/feature-flags'
 import {
   type PermissionGroupConfig,
   parsePermissionGroupConfig,
@@ -52,6 +53,10 @@ export class InvitationsNotAllowedError extends Error {
 export async function getUserPermissionConfig(
   userId: string
 ): Promise<PermissionGroupConfig | null> {
+  if (!isHosted && !isAccessControlEnabled) {
+    return null
+  }
+
   const [membership] = await db
     .select({ organizationId: member.organizationId })
     .from(member)

apps/sim/executor/utils/reference-validation.ts

@@ -19,6 +19,85 @@ export function createEnvVarPattern(): RegExp {
   return new RegExp(`\\${REFERENCE.ENV_VAR_START}([^}]+)\\${REFERENCE.ENV_VAR_END}`, 'g')
 }
 
+export interface EnvVarResolveOptions {
+  allowEmbedded?: boolean
+  resolveExactMatch?: boolean
+  trimKeys?: boolean
+  onMissing?: 'keep' | 'throw' | 'empty'
+  deep?: boolean
+  missingKeys?: string[]
+}
+
+/**
+ * Resolve {{ENV_VAR}} references in values using provided env vars.
+ */
+export function resolveEnvVarReferences(
+  value: unknown,
+  envVars: Record<string, string>,
+  options: EnvVarResolveOptions = {}
+): unknown {
+  const {
+    allowEmbedded = true,
+    resolveExactMatch = true,
+    trimKeys = false,
+    onMissing = 'keep',
+    deep = true,
+  } = options
+
+  if (typeof value === 'string') {
+    if (resolveExactMatch) {
+      const exactMatchPattern = new RegExp(
+        `^\\${REFERENCE.ENV_VAR_START}([^}]+)\\${REFERENCE.ENV_VAR_END}$`
+      )
+      const exactMatch = exactMatchPattern.exec(value)
+      if (exactMatch) {
+        const envKey = trimKeys ? exactMatch[1].trim() : exactMatch[1]
+        const envValue = envVars[envKey]
+        if (envValue !== undefined) return envValue
+        if (options.missingKeys) options.missingKeys.push(envKey)
+        if (onMissing === 'throw') {
+          throw new Error(`Environment variable "${envKey}" was not found`)
+        }
+        if (onMissing === 'empty') {
+          return ''
+        }
+        return value
+      }
+    }
+
+    if (!allowEmbedded) return value
+
+    const envVarPattern = createEnvVarPattern()
+    return value.replace(envVarPattern, (match, varName) => {
+      const envKey = trimKeys ? String(varName).trim() : String(varName)
+      const envValue = envVars[envKey]
+      if (envValue !== undefined) return envValue
+      if (options.missingKeys) options.missingKeys.push(envKey)
+      if (onMissing === 'throw') {
+        throw new Error(`Environment variable "${envKey}" was not found`)
+      }
+      if (onMissing === 'empty') {
+        return ''
+      }
+      return match
+    })
+  }
+
+  if (deep && Array.isArray(value)) {
+    return value.map((item) => resolveEnvVarReferences(item, envVars, options))
+  }
+
+  if (deep && value !== null && typeof value === 'object') {
+    const resolved: Record<string, any> = {}
+    for (const [key, val] of Object.entries(value)) {
+      resolved[key] = resolveEnvVarReferences(val, envVars, options)
+    }
+    return resolved
+  }
+
+  return value
+}
+
 /**
  * Creates a regex pattern for matching workflow variables <variable.name>
  * Captures the variable name (after "variable.") in group 1

apps/sim/executor/variables/resolvers/block.test.ts

@@ -126,16 +126,14 @@ describe('BlockResolver', () => {
       expect(resolver.resolve('<source.items.1.id>', ctx)).toBe(2)
     })
 
-    it.concurrent('should throw error for non-existent path', () => {
+    it.concurrent('should return undefined for non-existent path', () => {
       const workflow = createTestWorkflow([{ id: 'source' }])
       const resolver = new BlockResolver(workflow)
       const ctx = createTestContext('current', {
         source: { existing: 'value' },
       })
 
-      expect(() => resolver.resolve('<source.nonexistent>', ctx)).toThrow(
-        /No value found at path "nonexistent" in block "source"/
-      )
+      expect(resolver.resolve('<source.nonexistent>', ctx)).toBeUndefined()
     })
 
     it.concurrent('should return undefined for non-existent block', () => {
@@ -971,19 +969,17 @@ describe('BlockResolver', () => {
         source: { value: undefined, other: 'exists' },
       })
 
-      expect(() => resolver.resolve('<source.value>', ctx)).toThrow()
+      expect(resolver.resolve('<source.value>', ctx)).toBeUndefined()
     })
 
-    it.concurrent('should handle deeply nested path errors', () => {
+    it.concurrent('should return undefined for deeply nested non-existent path', () => {
       const workflow = createTestWorkflow([{ id: 'source' }])
       const resolver = new BlockResolver(workflow)
       const ctx = createTestContext('current', {
         source: { level1: { level2: {} } },
       })
 
-      expect(() => resolver.resolve('<source.level1.level2.level3>', ctx)).toThrow(
-        /No value found at path "level1.level2.level3"/
-      )
+      expect(resolver.resolve('<source.level1.level2.level3>', ctx)).toBeUndefined()
     })
   })
 })

apps/sim/executor/variables/resolvers/block.ts

@@ -108,11 +108,7 @@ export class BlockResolver implements Resolver {
       }
     }
 
-    // If still undefined, throw error with original path
-    const availableKeys = output && typeof output === 'object' ? Object.keys(output) : []
-    throw new Error(
-      `No value found at path "${pathParts.join('.')}" in block "${blockName}". Available fields: ${availableKeys.join(', ')}`
-    )
+    return undefined
   }
 
   private getBlockOutput(blockId: string, context: ResolutionContext): any {