github/sim
1
1
Fork 0
mirror of https://github.com/simstudioai/sim.git synced 2026-04-28 03:00:29 -04:00
Code Issues Packages Projects Releases Wiki Activity

remove worker code

Browse Source
This commit is contained in:
Vikhyath Mondreti
2026-04-09 19:15:02 -07:00
parent c61cbb04a5
commit 33d1342452
63 changed files with 148 additions and 5158 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
README.md
View File

@@ -74,10 +74,6 @@ docker compose -f docker-compose.prod.yml up -d
Open [http://localhost:3000](http://localhost:3000)
#### Background worker note
The Docker Compose stack starts a dedicated worker container by default. If `REDIS_URL` is not configured, the worker will start, log that it is idle, and do no queue processing. This is expected. Queue-backed API, webhook, and schedule execution requires Redis; installs without Redis continue to use the inline execution path.
Sim also supports local models via [Ollama](https://ollama.ai) and [vLLM](https://docs.vllm.ai/) — see the [Docker self-hosting docs](https://docs.sim.ai/self-hosting/docker) for setup details.
### Self-hosted: Manual Setup
@@ -123,12 +119,10 @@ cd packages/db && bun run db:migrate
5. Start development servers:
```bash
bun run dev:full # Starts Next.js app, realtime socket server, and the BullMQ worker
bun run dev:full # Starts Next.js app and realtime socket server
```
If `REDIS_URL` is not configured, the worker will remain idle and execution continues inline.
Or run separately: `bun run dev` (Next.js), `cd apps/sim && bun run dev:sockets` (realtime), and `cd apps/sim && bun run worker` (BullMQ worker).
Or run separately: `bun run dev` (Next.js) and `cd apps/sim && bun run dev:sockets` (realtime).
## Copilot API Keys

64
apps/sim/app/api/jobs/[jobId]/route.test.ts
View File

@@ -6,16 +6,16 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'
const {
mockCheckHybridAuth,
mockGetDispatchJobRecord,
mockGetJobQueue,
mockVerifyWorkflowAccess,
mockGetWorkflowById,
mockGetJob,
} = vi.hoisted(() => ({
mockCheckHybridAuth: vi.fn(),
mockGetDispatchJobRecord: vi.fn(),
mockGetJobQueue: vi.fn(),
mockVerifyWorkflowAccess: vi.fn(),
mockGetWorkflowById: vi.fn(),
mockGetJob: vi.fn(),
}))
vi.mock('@sim/logger', () => ({
@@ -32,19 +32,9 @@ vi.mock('@/lib/auth/hybrid', () => ({
}))
vi.mock('@/lib/core/async-jobs', () => ({
JOB_STATUS: {
PENDING: 'pending',
PROCESSING: 'processing',
COMPLETED: 'completed',
FAILED: 'failed',
},
getJobQueue: mockGetJobQueue,
}))
vi.mock('@/lib/core/workspace-dispatch/store', () => ({
getDispatchJobRecord: mockGetDispatchJobRecord,
}))
vi.mock('@/lib/core/utils/request', () => ({
generateRequestId: vi.fn().mockReturnValue('request-1'),
}))
@@ -85,71 +75,51 @@ describe('GET /api/jobs/[jobId]', () => {
})
mockGetJobQueue.mockResolvedValue({
getJob: vi.fn().mockResolvedValue(null),
getJob: mockGetJob,
})
})
it('returns dispatcher-aware waiting status with metadata', async () => {
mockGetDispatchJobRecord.mockResolvedValue({
id: 'dispatch-1',
workspaceId: 'workspace-1',
lane: 'runtime',
queueName: 'workflow-execution',
bullmqJobName: 'workflow-execution',
bullmqPayload: {},
it('returns job status with metadata', async () => {
mockGetJob.mockResolvedValue({
id: 'job-1',
status: 'pending',
metadata: {
workflowId: 'workflow-1',
},
priority: 10,
status: 'waiting',
createdAt: 1000,
admittedAt: 2000,
})
const response = await GET(createMockRequest(), {
params: Promise.resolve({ jobId: 'dispatch-1' }),
params: Promise.resolve({ jobId: 'job-1' }),
})
const body = await response.json()
expect(response.status).toBe(200)
expect(body.status).toBe('waiting')
expect(body.metadata.queueName).toBe('workflow-execution')
expect(body.metadata.lane).toBe('runtime')
expect(body.metadata.workspaceId).toBe('workspace-1')
expect(body.status).toBe('pending')
expect(body.metadata.workflowId).toBe('workflow-1')
})
it('returns completed output from dispatch state', async () => {
mockGetDispatchJobRecord.mockResolvedValue({
id: 'dispatch-2',
workspaceId: 'workspace-1',
lane: 'interactive',
queueName: 'workflow-execution',
bullmqJobName: 'direct-workflow-execution',
bullmqPayload: {},
it('returns completed output from job', async () => {
mockGetJob.mockResolvedValue({
id: 'job-2',
status: 'completed',
metadata: {
workflowId: 'workflow-1',
},
priority: 1,
status: 'completed',
createdAt: 1000,
startedAt: 2000,
completedAt: 7000,
output: { success: true },
})
const response = await GET(createMockRequest(), {
params: Promise.resolve({ jobId: 'dispatch-2' }),
params: Promise.resolve({ jobId: 'job-2' }),
})
const body = await response.json()
expect(response.status).toBe(200)
expect(body.status).toBe('completed')
expect(body.output).toEqual({ success: true })
expect(body.metadata.duration).toBe(5000)
})
it('returns 404 when neither dispatch nor BullMQ job exists', async () => {
mockGetDispatchJobRecord.mockResolvedValue(null)
it('returns 404 when job does not exist', async () => {
mockGetJob.mockResolvedValue(null)
const response = await GET(createMockRequest(), {
params: Promise.resolve({ jobId: 'missing-job' }),

28
apps/sim/app/api/jobs/[jobId]/route.ts
View File

@@ -3,8 +3,6 @@ import { type NextRequest, NextResponse } from 'next/server'
import { checkHybridAuth } from '@/lib/auth/hybrid'
import { getJobQueue } from '@/lib/core/async-jobs'
import { generateRequestId } from '@/lib/core/utils/request'
import { presentDispatchOrJobStatus } from '@/lib/core/workspace-dispatch/status'
import { getDispatchJobRecord } from '@/lib/core/workspace-dispatch/store'
import { createErrorResponse } from '@/app/api/workflows/utils'
const logger = createLogger('TaskStatusAPI')
@@ -25,15 +23,14 @@ export async function GET(
const authenticatedUserId = authResult.userId
const dispatchJob = await getDispatchJobRecord(taskId)
const jobQueue = await getJobQueue()
const job = dispatchJob ? null : await jobQueue.getJob(taskId)
const job = await jobQueue.getJob(taskId)
if (!job && !dispatchJob) {
if (!job) {
return createErrorResponse('Task not found', 404)
}
const metadataToCheck = dispatchJob?.metadata ?? job?.metadata
const metadataToCheck = job.metadata
if (metadataToCheck?.workflowId) {
const { verifyWorkflowAccess } = await import('@/socket/middleware/permissions')
@@ -61,25 +58,22 @@ export async function GET(
return createErrorResponse('Access denied', 403)
}
const presented = presentDispatchOrJobStatus(dispatchJob, job)
const response: any = {
const response: Record<string, unknown> = {
success: true,
taskId,
status: presented.status,
metadata: presented.metadata,
status: job.status,
metadata: job.metadata,
}
if (presented.output !== undefined) response.output = presented.output
if (presented.error !== undefined) response.error = presented.error
if (presented.estimatedDuration !== undefined) {
response.estimatedDuration = presented.estimatedDuration
}
if (job.output !== undefined) response.output = job.output
if (job.error !== undefined) response.error = job.error
return NextResponse.json(response)
} catch (error: any) {
} catch (error: unknown) {
const errorMessage = error instanceof Error ? error.message : String(error)
logger.error(`[${requestId}] Error fetching task status:`, error)
if (error.message?.includes('not found') || error.status === 404) {
if (errorMessage?.includes('not found')) {
return createErrorResponse('Task not found', 404)
}

28
apps/sim/app/api/resume/[workflowId]/[executionId]/[contextId]/route.ts
View File

@@ -1,13 +1,11 @@
import { createLogger } from '@sim/logger'
import { type NextRequest, NextResponse } from 'next/server'
import { AuthType } from '@/lib/auth/hybrid'
import { getJobQueue, shouldUseBullMQ } from '@/lib/core/async-jobs'
import { createBullMQJobData } from '@/lib/core/bullmq'
import { getJobQueue } from '@/lib/core/async-jobs'
import { generateRequestId } from '@/lib/core/utils/request'
import { SSE_HEADERS } from '@/lib/core/utils/sse'
import { getBaseUrl } from '@/lib/core/utils/urls'
import { generateId } from '@/lib/core/utils/uuid'
import { enqueueWorkspaceDispatch } from '@/lib/core/workspace-dispatch'
import { setExecutionMeta } from '@/lib/execution/event-buffer'
import { preprocessExecution } from '@/lib/execution/preprocessing'
import { PauseResumeManager } from '@/lib/workflows/executor/human-in-the-loop-manager'
@@ -227,26 +225,10 @@ export async function POST(
let jobId: string
try {
const useBullMQ = shouldUseBullMQ()
if (useBullMQ) {
jobId = await enqueueWorkspaceDispatch({
id: enqueueResult.resumeExecutionId,
workspaceId: workflow.workspaceId,
lane: 'runtime',
queueName: 'resume-execution',
bullmqJobName: 'resume-execution',
bullmqPayload: createBullMQJobData(resumePayload, {
workflowId,
userId,
}),
metadata: { workflowId, userId },
})
} else {
const jobQueue = await getJobQueue()
jobId = await jobQueue.enqueue('resume-execution', resumePayload, {
metadata: { workflowId, workspaceId: workflow.workspaceId, userId },
})
}
const jobQueue = await getJobQueue()
jobId = await jobQueue.enqueue('resume-execution', resumePayload, {
metadata: { workflowId, workspaceId: workflow.workspaceId, userId },
})
logger.info('Enqueued async resume execution', {
jobId,
resumeExecutionId: enqueueResult.resumeExecutionId,

64
apps/sim/app/api/schedules/execute/route.test.ts
View File

@@ -14,7 +14,6 @@ const {
mockDbReturning,
mockDbUpdate,
mockEnqueue,
mockEnqueueWorkspaceDispatch,
mockStartJob,
mockCompleteJob,
mockMarkJobFailed,
@@ -24,7 +23,6 @@ const {
const mockDbSet = vi.fn().mockReturnValue({ where: mockDbWhere })
const mockDbUpdate = vi.fn().mockReturnValue({ set: mockDbSet })
const mockEnqueue = vi.fn().mockResolvedValue('job-id-1')
const mockEnqueueWorkspaceDispatch = vi.fn().mockResolvedValue('job-id-1')
const mockStartJob = vi.fn().mockResolvedValue(undefined)
const mockCompleteJob = vi.fn().mockResolvedValue(undefined)
const mockMarkJobFailed = vi.fn().mockResolvedValue(undefined)
@@ -42,7 +40,6 @@ const {
mockDbReturning,
mockDbUpdate,
mockEnqueue,
mockEnqueueWorkspaceDispatch,
mockStartJob,
mockCompleteJob,
mockMarkJobFailed,
@@ -75,15 +72,6 @@ vi.mock('@/lib/core/async-jobs', () => ({
shouldExecuteInline: vi.fn().mockReturnValue(false),
}))
vi.mock('@/lib/core/bullmq', () => ({
isBullMQEnabled: vi.fn().mockReturnValue(true),
createBullMQJobData: vi.fn((payload: unknown) => ({ payload })),
}))
vi.mock('@/lib/core/workspace-dispatch', () => ({
enqueueWorkspaceDispatch: mockEnqueueWorkspaceDispatch,
}))
vi.mock('@/lib/workflows/utils', () => ({
getWorkflowById: vi.fn().mockResolvedValue({
id: 'workflow-1',
@@ -175,8 +163,6 @@ const SINGLE_JOB = [
cronExpression: '0 * * * *',
failedCount: 0,
lastQueuedAt: undefined,
sourceUserId: 'user-1',
sourceWorkspaceId: 'workspace-1',
sourceType: 'job',
},
]
@@ -250,56 +236,48 @@ describe('Scheduled Workflow Execution API Route', () => {
expect(data).toHaveProperty('executedCount', 2)
})
it('should queue mothership jobs to BullMQ when available', async () => {
it('should execute mothership jobs inline', async () => {
mockDbReturning.mockReturnValueOnce([]).mockReturnValueOnce(SINGLE_JOB)
const response = await GET(createMockRequest())
expect(response.status).toBe(200)
expect(mockEnqueueWorkspaceDispatch).toHaveBeenCalledWith(
expect(mockExecuteJobInline).toHaveBeenCalledWith(
expect.objectContaining({
workspaceId: 'workspace-1',
lane: 'runtime',
queueName: 'mothership-job-execution',
bullmqJobName: 'mothership-job-execution',
bullmqPayload: {
payload: {
scheduleId: 'job-1',
cronExpression: '0 * * * *',
failedCount: 0,
now: expect.any(String),
},
},
scheduleId: 'job-1',
cronExpression: '0 * * * *',
failedCount: 0,
now: expect.any(String),
})
)
expect(mockExecuteJobInline).not.toHaveBeenCalled()
})
it('should enqueue preassigned correlation metadata for schedules', async () => {
mockDbReturning.mockReturnValue(SINGLE_SCHEDULE)
it('should enqueue schedule with correlation metadata via job queue', async () => {
mockDbReturning.mockReturnValueOnce(SINGLE_SCHEDULE).mockReturnValueOnce([])
const response = await GET(createMockRequest())
expect(response.status).toBe(200)
expect(mockEnqueueWorkspaceDispatch).toHaveBeenCalledWith(
expect(mockEnqueue).toHaveBeenCalledWith(
'schedule-execution',
expect.objectContaining({
id: 'schedule-execution-1',
workspaceId: 'workspace-1',
lane: 'runtime',
queueName: 'schedule-execution',
bullmqJobName: 'schedule-execution',
metadata: {
scheduleId: 'schedule-1',
workflowId: 'workflow-1',
executionId: 'schedule-execution-1',
requestId: 'test-request-id',
}),
expect.objectContaining({
metadata: expect.objectContaining({
workflowId: 'workflow-1',
correlation: {
workspaceId: 'workspace-1',
correlation: expect.objectContaining({
executionId: 'schedule-execution-1',
requestId: 'test-request-id',
source: 'schedule',
workflowId: 'workflow-1',
scheduleId: 'schedule-1',
triggerType: 'schedule',
scheduledFor: '2025-01-01T00:00:00.000Z',
},
},
}),
}),
})
)
})

64
apps/sim/app/api/schedules/execute/route.ts
View File

@@ -4,10 +4,8 @@ import { and, eq, isNull, lt, lte, ne, not, or, sql } from 'drizzle-orm'
import { type NextRequest, NextResponse } from 'next/server'
import { verifyCronAuth } from '@/lib/auth/internal'
import { getJobQueue, shouldExecuteInline } from '@/lib/core/async-jobs'
import { createBullMQJobData, isBullMQEnabled } from '@/lib/core/bullmq'
import { generateRequestId } from '@/lib/core/utils/request'
import { generateId } from '@/lib/core/utils/uuid'
import { enqueueWorkspaceDispatch } from '@/lib/core/workspace-dispatch'
import {
executeJobInline,
executeScheduleJob,
@@ -75,8 +73,6 @@ export async function GET(request: NextRequest) {
cronExpression: workflowSchedule.cronExpression,
failedCount: workflowSchedule.failedCount,
lastQueuedAt: workflowSchedule.lastQueuedAt,
sourceWorkspaceId: workflowSchedule.sourceWorkspaceId,
sourceUserId: workflowSchedule.sourceUserId,
sourceType: workflowSchedule.sourceType,
})
@@ -123,38 +119,13 @@ export async function GET(request: NextRequest) {
: null
const resolvedWorkspaceId = resolvedWorkflow?.workspaceId
let jobId: string
if (isBullMQEnabled()) {
if (!resolvedWorkspaceId) {
throw new Error(
`Missing workspace for scheduled workflow ${schedule.workflowId}; refusing to bypass workspace admission`
)
}
jobId = await enqueueWorkspaceDispatch({
id: executionId,
workspaceId: resolvedWorkspaceId,
lane: 'runtime',
queueName: 'schedule-execution',
bullmqJobName: 'schedule-execution',
bullmqPayload: createBullMQJobData(payload, {
workflowId: schedule.workflowId ?? undefined,
correlation,
}),
metadata: {
workflowId: schedule.workflowId ?? undefined,
correlation,
},
})
} else {
jobId = await jobQueue.enqueue('schedule-execution', payload, {
metadata: {
workflowId: schedule.workflowId ?? undefined,
workspaceId: resolvedWorkspaceId ?? undefined,
correlation,
},
})
}
const jobId = await jobQueue.enqueue('schedule-execution', payload, {
metadata: {
workflowId: schedule.workflowId ?? undefined,
workspaceId: resolvedWorkspaceId ?? undefined,
correlation,
},
})
logger.info(
`[${requestId}] Queued schedule execution task ${jobId} for workflow ${schedule.workflowId}`
)
@@ -206,7 +177,7 @@ export async function GET(request: NextRequest) {
}
})
// Mothership jobs use BullMQ when available, otherwise direct inline execution.
// Mothership jobs are executed inline directly.
const jobPromises = dueJobs.map(async (job) => {
const queueTime = job.lastQueuedAt ?? queuedAt
const payload = {
@@ -217,24 +188,7 @@ export async function GET(request: NextRequest) {
}
try {
if (isBullMQEnabled()) {
if (!job.sourceWorkspaceId || !job.sourceUserId) {
throw new Error(`Mothership job ${job.id} is missing workspace/user ownership`)
}
await enqueueWorkspaceDispatch({
workspaceId: job.sourceWorkspaceId!,
lane: 'runtime',
queueName: 'mothership-job-execution',
bullmqJobName: 'mothership-job-execution',
bullmqPayload: createBullMQJobData(payload),
metadata: {
userId: job.sourceUserId,
},
})
} else {
await executeJobInline(payload)
}
await executeJobInline(payload)
} catch (error) {
logger.error(`[${requestId}] Job execution failed for ${job.id}`, {
error: error instanceof Error ? error.message : String(error),

11
apps/sim/app/api/webhooks/trigger/[path]/route.ts
View File

@@ -2,7 +2,6 @@ import { createLogger } from '@sim/logger'
import { type NextRequest, NextResponse } from 'next/server'
import { admissionRejectedResponse, tryAdmit } from '@/lib/core/admission/gate'
import { generateRequestId } from '@/lib/core/utils/request'
import { DispatchQueueFullError } from '@/lib/core/workspace-dispatch'
import {
checkWebhookPreprocessing,
findAllWebhooksForPath,
@@ -167,16 +166,6 @@ async function handleWebhookPost(
})
responses.push(response)
} catch (error) {
if (error instanceof DispatchQueueFullError) {
return NextResponse.json(
{
error: 'Service temporarily at capacity',
message: error.message,
retryAfterSeconds: 10,
},
{ status: 503, headers: { 'Retry-After': '10' } }
)
}
throw error
}
}

34
apps/sim/app/api/workflows/[id]/execute/route.async.test.ts
View File

@@ -10,13 +10,11 @@ const {
mockAuthorizeWorkflowByWorkspacePermission,
mockPreprocessExecution,
mockEnqueue,
mockEnqueueWorkspaceDispatch,
} = vi.hoisted(() => ({
mockCheckHybridAuth: vi.fn(),
mockAuthorizeWorkflowByWorkspacePermission: vi.fn(),
mockPreprocessExecution: vi.fn(),
mockEnqueue: vi.fn().mockResolvedValue('job-123'),
mockEnqueueWorkspaceDispatch: vi.fn().mockResolvedValue('job-123'),
}))
vi.mock('@/lib/auth/hybrid', () => ({
@@ -47,16 +45,6 @@ vi.mock('@/lib/core/async-jobs', () => ({
markJobFailed: vi.fn(),
}),
shouldExecuteInline: vi.fn().mockReturnValue(false),
shouldUseBullMQ: vi.fn().mockReturnValue(true),
}))
vi.mock('@/lib/core/bullmq', () => ({
createBullMQJobData: vi.fn((payload: unknown, metadata?: unknown) => ({ payload, metadata })),
}))
vi.mock('@/lib/core/workspace-dispatch', () => ({
enqueueWorkspaceDispatch: mockEnqueueWorkspaceDispatch,
waitForDispatchJob: vi.fn(),
}))
vi.mock('@/lib/core/utils/request', () => ({
@@ -150,24 +138,28 @@ describe('workflow execute async route', () => {
expect(response.status).toBe(202)
expect(body.executionId).toBe('execution-123')
expect(body.jobId).toBe('job-123')
expect(mockEnqueueWorkspaceDispatch).toHaveBeenCalledWith(
expect(mockEnqueue).toHaveBeenCalledWith(
'workflow-execution',
expect.objectContaining({
id: 'execution-123',
workflowId: 'workflow-1',
userId: 'actor-1',
workspaceId: 'workspace-1',
lane: 'runtime',
queueName: 'workflow-execution',
bullmqJobName: 'workflow-execution',
metadata: {
executionId: 'execution-123',
executionMode: 'async',
}),
expect.objectContaining({
metadata: expect.objectContaining({
workflowId: 'workflow-1',
userId: 'actor-1',
correlation: {
workspaceId: 'workspace-1',
correlation: expect.objectContaining({
executionId: 'execution-123',
requestId: 'req-12345678',
source: 'workflow',
workflowId: 'workflow-1',
triggerType: 'manual',
},
},
}),
}),
})
)
})

234
apps/sim/app/api/workflows/[id]/execute/route.ts
View File

@@ -3,8 +3,7 @@ import { type NextRequest, NextResponse } from 'next/server'
import { z } from 'zod'
import { AuthType, checkHybridAuth, hasExternalApiCredentials } from '@/lib/auth/hybrid'
import { admissionRejectedResponse, tryAdmit } from '@/lib/core/admission/gate'
import { getJobQueue, shouldExecuteInline, shouldUseBullMQ } from '@/lib/core/async-jobs'
import { createBullMQJobData } from '@/lib/core/bullmq'
import { getJobQueue, shouldExecuteInline } from '@/lib/core/async-jobs'
import {
createTimeoutAbortController,
getTimeoutErrorMessage,
@@ -14,13 +13,6 @@ import { generateRequestId } from '@/lib/core/utils/request'
import { SSE_HEADERS } from '@/lib/core/utils/sse'
import { getBaseUrl } from '@/lib/core/utils/urls'
import { generateId, isValidUuid } from '@/lib/core/utils/uuid'
import {
DispatchQueueFullError,
enqueueWorkspaceDispatch,
type WorkspaceDispatchLane,
waitForDispatchJob,
} from '@/lib/core/workspace-dispatch'
import { createBufferedExecutionStream } from '@/lib/execution/buffered-stream'
import {
buildNextCallChain,
parseCallChain,
@@ -43,11 +35,6 @@ import { executeWorkflow } from '@/lib/workflows/executor/execute-workflow'
import { executeWorkflowCore } from '@/lib/workflows/executor/execution-core'
import { type ExecutionEvent, encodeSSEEvent } from '@/lib/workflows/executor/execution-events'
import { handlePostExecutionPauseState } from '@/lib/workflows/executor/pause-persistence'
import {
DIRECT_WORKFLOW_JOB_NAME,
type QueuedWorkflowExecutionPayload,
type QueuedWorkflowExecutionResult,
} from '@/lib/workflows/executor/queued-workflow-execution'
import {
loadDeployedWorkflowState,
loadWorkflowFromNormalizedTables,
@@ -119,8 +106,6 @@ const ExecuteWorkflowSchema = z.object({
export const runtime = 'nodejs'
export const dynamic = 'force-dynamic'
const INLINE_TRIGGER_TYPES = new Set<CoreTriggerType>(['manual', 'workflow'])
function resolveOutputIds(
selectedOutputs: string[] | undefined,
blocks: Record<string, any>
@@ -218,39 +203,19 @@ async function handleAsyncExecution(params: AsyncExecutionParams): Promise<NextR
}
try {
const useBullMQ = shouldUseBullMQ()
const jobQueue = useBullMQ ? null : await getJobQueue()
const jobId = useBullMQ
? await enqueueWorkspaceDispatch({
id: executionId,
workspaceId,
lane: 'runtime',
queueName: 'workflow-execution',
bullmqJobName: 'workflow-execution',
bullmqPayload: createBullMQJobData(payload, {
workflowId,
userId,
correlation,
}),
metadata: {
workflowId,
userId,
correlation,
},
})
: await jobQueue!.enqueue('workflow-execution', payload, {
metadata: { workflowId, workspaceId, userId, correlation },
})
const jobQueue = await getJobQueue()
const jobId = await jobQueue.enqueue('workflow-execution', payload, {
metadata: { workflowId, workspaceId, userId, correlation },
})
asyncLogger.info('Queued async workflow execution', { jobId })
if (shouldExecuteInline() && jobQueue) {
const inlineJobQueue = jobQueue
if (shouldExecuteInline()) {
void (async () => {
try {
await inlineJobQueue.startJob(jobId)
await jobQueue.startJob(jobId)
const output = await executeWorkflowJob(payload)
await inlineJobQueue.completeJob(jobId, output)
await jobQueue.completeJob(jobId, output)
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error)
asyncLogger.error('Async workflow execution failed', {
@@ -258,7 +223,7 @@ async function handleAsyncExecution(params: AsyncExecutionParams): Promise<NextR
error: errorMessage,
})
try {
await inlineJobQueue.markJobFailed(jobId, errorMessage)
await jobQueue.markJobFailed(jobId, errorMessage)
} catch (markFailedError) {
asyncLogger.error('Failed to mark job as failed', {
jobId,
@@ -284,17 +249,6 @@ async function handleAsyncExecution(params: AsyncExecutionParams): Promise<NextR
{ status: 202 }
)
} catch (error: any) {
if (error instanceof DispatchQueueFullError) {
return NextResponse.json(
{
error: 'Service temporarily at capacity',
message: error.message,
retryAfterSeconds: 10,
},
{ status: 503, headers: { 'Retry-After': '10' } }
)
}
asyncLogger.error('Failed to queue async execution', error)
return NextResponse.json(
{ error: `Failed to queue async execution: ${error.message}` },
@@ -303,31 +257,6 @@ async function handleAsyncExecution(params: AsyncExecutionParams): Promise<NextR
}
}
async function enqueueDirectWorkflowExecution(
payload: QueuedWorkflowExecutionPayload,
priority: number,
lane: WorkspaceDispatchLane
) {
return enqueueWorkspaceDispatch({
id: payload.metadata.executionId,
workspaceId: payload.metadata.workspaceId,
lane,
queueName: 'workflow-execution',
bullmqJobName: DIRECT_WORKFLOW_JOB_NAME,
bullmqPayload: createBullMQJobData(payload, {
workflowId: payload.metadata.workflowId,
userId: payload.metadata.userId,
correlation: payload.metadata.correlation,
}),
metadata: {
workflowId: payload.metadata.workflowId,
userId: payload.metadata.userId,
correlation: payload.metadata.correlation,
},
priority,
})
}
/**
* POST /api/workflows/[id]/execute
*
@@ -796,92 +725,6 @@ async function handleExecutePost(
const executionVariables = cachedWorkflowData?.variables ?? workflow.variables ?? {}
if (shouldUseBullMQ() && !INLINE_TRIGGER_TYPES.has(triggerType)) {
try {
const dispatchJobId = await enqueueDirectWorkflowExecution(
{
workflow,
metadata,
input: processedInput,
variables: executionVariables,
selectedOutputs,
includeFileBase64,
base64MaxBytes,
stopAfterBlockId,
timeoutMs: preprocessResult.executionTimeout?.sync,
runFromBlock: resolvedRunFromBlock,
},
5,
'interactive'
)
const resultRecord = await waitForDispatchJob(
dispatchJobId,
(preprocessResult.executionTimeout?.sync ?? 300000) + 30000
)
if (resultRecord.status === 'failed') {
return NextResponse.json(
{
success: false,
executionId,
error: resultRecord.error ?? 'Workflow execution failed',
},
{ status: 500 }
)
}
const result = resultRecord.output as QueuedWorkflowExecutionResult
const resultForResponseBlock = {
success: result.success,
logs: result.logs,
output: result.output,
}
if (
auth.authType !== AuthType.INTERNAL_JWT &&
workflowHasResponseBlock(resultForResponseBlock)
) {
return createHttpResponseFromBlock(resultForResponseBlock)
}
return NextResponse.json(
{
success: result.success,
executionId,
output: result.output,
error: result.error,
metadata: result.metadata,
},
{ status: result.statusCode ?? 200 }
)
} catch (error: unknown) {
if (error instanceof DispatchQueueFullError) {
return NextResponse.json(
{
error: 'Service temporarily at capacity',
message: error.message,
retryAfterSeconds: 10,
},
{ status: 503, headers: { 'Retry-After': '10' } }
)
}
const errorMessage = error instanceof Error ? error.message : 'Unknown error'
reqLogger.error(`Queued non-SSE execution failed: ${errorMessage}`)
return NextResponse.json(
{
success: false,
error: errorMessage,
},
{ status: 500 }
)
}
}
const timeoutController = createTimeoutAbortController(
preprocessResult.executionTimeout?.sync
)
@@ -998,54 +841,6 @@ async function handleExecutePost(
}
if (shouldUseDraftState) {
const shouldDispatchViaQueue = shouldUseBullMQ() && !INLINE_TRIGGER_TYPES.has(triggerType)
if (shouldDispatchViaQueue) {
const metadata: ExecutionMetadata = {
requestId,
executionId,
workflowId,
workspaceId,
userId: actorUserId,
sessionUserId: isClientSession ? userId : undefined,
workflowUserId: workflow.userId,
triggerType,
useDraftState: shouldUseDraftState,
startTime: new Date().toISOString(),
isClientSession,
enforceCredentialAccess: useAuthenticatedUserAsActor,
workflowStateOverride: effectiveWorkflowStateOverride,
callChain,
executionMode: 'sync',
}
const executionVariables = cachedWorkflowData?.variables ?? workflow.variables ?? {}
await enqueueDirectWorkflowExecution(
{
workflow,
metadata,
input: processedInput,
variables: executionVariables,
selectedOutputs,
includeFileBase64,
base64MaxBytes,
stopAfterBlockId,
timeoutMs: preprocessResult.executionTimeout?.sync,
runFromBlock: resolvedRunFromBlock,
streamEvents: true,
},
1,
'interactive'
)
return new NextResponse(createBufferedExecutionStream(executionId), {
headers: {
...SSE_HEADERS,
'X-Execution-Id': executionId,
},
})
}
reqLogger.info('Using SSE console log streaming (manual execution)')
} else {
reqLogger.info('Using streaming API response')
@@ -1524,17 +1319,6 @@ async function handleExecutePost(
},
})
} catch (error: any) {
if (error instanceof DispatchQueueFullError) {
return NextResponse.json(
{
error: 'Service temporarily at capacity',
message: error.message,
retryAfterSeconds: 10,
},
{ status: 503, headers: { 'Retry-After': '10' } }
)
}
reqLogger.error('Failed to start workflow execution:', error)
return NextResponse.json(
{ error: error.message || 'Failed to start workflow execution' },

37
apps/sim/background/workspace-notification-delivery.ts
View File

@@ -16,15 +16,12 @@ import {
import { checkUsageStatus } from '@/lib/billing/calculations/usage-monitor'
import { getHighestPrioritySubscription } from '@/lib/billing/core/subscription'
import { dollarsToCredits } from '@/lib/billing/credits/conversion'
import { createBullMQJobData, isBullMQEnabled } from '@/lib/core/bullmq'
import { acquireLock } from '@/lib/core/config/redis'
import { RateLimiter } from '@/lib/core/rate-limiter'
import { decryptSecret } from '@/lib/core/security/encryption'
import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server'
import { formatDuration } from '@/lib/core/utils/formatting'
import { getBaseUrl } from '@/lib/core/utils/urls'
import { generateId } from '@/lib/core/utils/uuid'
import { enqueueWorkspaceDispatch } from '@/lib/core/workspace-dispatch'
import type { TraceSpan, WorkflowExecutionLog } from '@/lib/logs/types'
import { sendEmail } from '@/lib/messaging/email/mailer'
import type { AlertConfig } from '@/lib/notifications/alert-rules'
@@ -35,8 +32,6 @@ const logger = createLogger('WorkspaceNotificationDelivery')
const MAX_ATTEMPTS = 5
const RETRY_DELAYS = [5 * 1000, 15 * 1000, 60 * 1000, 3 * 60 * 1000, 10 * 60 * 1000]
const NOTIFICATION_DISPATCH_LOCK_TTL_SECONDS = 3
function getRetryDelayWithJitter(baseDelay: number): number {
const jitter = Math.random() * 0.1 * baseDelay
return Math.floor(baseDelay + jitter)
@@ -534,42 +529,14 @@ async function buildRetryLog(params: NotificationDeliveryParams): Promise<Workfl
}
export async function enqueueNotificationDeliveryDispatch(
params: NotificationDeliveryParams
_params: NotificationDeliveryParams
): Promise<boolean> {
if (!isBullMQEnabled()) {
return false
}
const lockAcquired = await acquireLock(
`workspace-notification-dispatch:${params.deliveryId}`,
params.deliveryId,
NOTIFICATION_DISPATCH_LOCK_TTL_SECONDS
)
if (!lockAcquired) {
return false
}
await enqueueWorkspaceDispatch({
workspaceId: params.workspaceId,
lane: 'lightweight',
queueName: 'workspace-notification-delivery',
bullmqJobName: 'workspace-notification-delivery',
bullmqPayload: createBullMQJobData(params),
metadata: {
workflowId: params.log.workflowId ?? undefined,
},
})
return true
return false
}
const STUCK_IN_PROGRESS_THRESHOLD_MS = 5 * 60 * 1000
export async function sweepPendingNotificationDeliveries(limit = 50): Promise<number> {
if (!isBullMQEnabled()) {
return 0
}
const stuckThreshold = new Date(Date.now() - STUCK_IN_PROGRESS_THRESHOLD_MS)
await db

146
apps/sim/lib/billing/workspace-concurrency.test.ts
View File

@@ -1,146 +0,0 @@
/**
* @vitest-environment node
*/
import { beforeEach, describe, expect, it, vi } from 'vitest'
const {
mockGetHighestPrioritySubscription,
mockGetWorkspaceBilledAccountUserId,
mockFeatureFlags,
mockRedisGet,
mockRedisSet,
mockRedisDel,
mockRedisKeys,
mockGetRedisClient,
} = vi.hoisted(() => ({
mockGetHighestPrioritySubscription: vi.fn(),
mockGetWorkspaceBilledAccountUserId: vi.fn(),
mockFeatureFlags: {
isBillingEnabled: true,
},
mockRedisGet: vi.fn(),
mockRedisSet: vi.fn(),
mockRedisDel: vi.fn(),
mockRedisKeys: vi.fn(),
mockGetRedisClient: vi.fn(),
}))
vi.mock('@sim/logger', () => ({
createLogger: () => ({
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
debug: vi.fn(),
}),
}))
vi.mock('@/lib/billing/core/plan', () => ({
getHighestPrioritySubscription: mockGetHighestPrioritySubscription,
}))
vi.mock('@/lib/workspaces/utils', () => ({
getWorkspaceBilledAccountUserId: mockGetWorkspaceBilledAccountUserId,
}))
vi.mock('@/lib/core/config/redis', () => ({
getRedisClient: mockGetRedisClient,
}))
vi.mock('@/lib/core/config/feature-flags', () => mockFeatureFlags)
import {
getWorkspaceConcurrencyLimit,
resetWorkspaceConcurrencyLimitCache,
} from '@/lib/billing/workspace-concurrency'
describe('workspace concurrency billing', () => {
beforeEach(() => {
vi.clearAllMocks()
mockFeatureFlags.isBillingEnabled = true
mockRedisGet.mockResolvedValue(null)
mockRedisSet.mockResolvedValue('OK')
mockRedisDel.mockResolvedValue(1)
mockRedisKeys.mockResolvedValue([])
mockGetRedisClient.mockReturnValue({
get: mockRedisGet,
set: mockRedisSet,
del: mockRedisDel,
keys: mockRedisKeys,
})
})
it('returns free tier when no billed account exists', async () => {
mockGetWorkspaceBilledAccountUserId.mockResolvedValue(null)
await expect(getWorkspaceConcurrencyLimit('workspace-1')).resolves.toBe(5)
})
it('returns pro limit for pro billing accounts', async () => {
mockGetWorkspaceBilledAccountUserId.mockResolvedValue('user-1')
mockGetHighestPrioritySubscription.mockResolvedValue({
plan: 'pro_6000',
metadata: null,
})
await expect(getWorkspaceConcurrencyLimit('workspace-1')).resolves.toBe(50)
})
it('returns max limit for max plan tiers', async () => {
mockGetWorkspaceBilledAccountUserId.mockResolvedValue('user-1')
mockGetHighestPrioritySubscription.mockResolvedValue({
plan: 'pro_25000',
metadata: null,
})
await expect(getWorkspaceConcurrencyLimit('workspace-1')).resolves.toBe(200)
})
it('returns max limit for legacy team plans', async () => {
mockGetWorkspaceBilledAccountUserId.mockResolvedValue('user-1')
mockGetHighestPrioritySubscription.mockResolvedValue({
plan: 'team',
metadata: null,
})
await expect(getWorkspaceConcurrencyLimit('workspace-1')).resolves.toBe(200)
})
it('returns enterprise metadata override when present', async () => {
mockGetWorkspaceBilledAccountUserId.mockResolvedValue('user-1')
mockGetHighestPrioritySubscription.mockResolvedValue({
plan: 'enterprise',
metadata: {
workspaceConcurrencyLimit: '350',
},
})
await expect(getWorkspaceConcurrencyLimit('workspace-1')).resolves.toBe(350)
})
it('uses free-tier limit when billing is disabled', async () => {
mockFeatureFlags.isBillingEnabled = false
mockGetWorkspaceBilledAccountUserId.mockResolvedValue('user-1')
mockGetHighestPrioritySubscription.mockResolvedValue({
plan: 'pro_25000',
metadata: {
workspaceConcurrencyLimit: 999,
},
})
await expect(getWorkspaceConcurrencyLimit('workspace-1')).resolves.toBe(5)
})
it('uses redis cache when available', async () => {
mockRedisGet.mockResolvedValueOnce('123')
await expect(getWorkspaceConcurrencyLimit('workspace-1')).resolves.toBe(123)
expect(mockGetWorkspaceBilledAccountUserId).not.toHaveBeenCalled()
})
it('can clear a specific workspace cache entry', async () => {
await resetWorkspaceConcurrencyLimitCache('workspace-1')
expect(mockRedisDel).toHaveBeenCalledWith('workspace-concurrency-limit:workspace-1')
})
})

170
apps/sim/lib/billing/workspace-concurrency.ts
View File

@@ -1,170 +0,0 @@
import { createLogger } from '@sim/logger'
import { getHighestPrioritySubscription } from '@/lib/billing/core/plan'
import { getPlanTierCredits, isEnterprise, isPro, isTeam } from '@/lib/billing/plan-helpers'
import { parseEnterpriseWorkspaceConcurrencyMetadata } from '@/lib/billing/types'
import { env } from '@/lib/core/config/env'
import { isBillingEnabled } from '@/lib/core/config/feature-flags'
import { getRedisClient } from '@/lib/core/config/redis'
import { getWorkspaceBilledAccountUserId } from '@/lib/workspaces/utils'
const logger = createLogger('WorkspaceConcurrencyBilling')
const CACHE_TTL_MS = 60_000
const CACHE_TTL_SECONDS = Math.floor(CACHE_TTL_MS / 1000)
interface CacheEntry {
value: number
expiresAt: number
}
const inMemoryConcurrencyCache = new Map<string, CacheEntry>()
function cacheKey(workspaceId: string): string {
return `workspace-concurrency-limit:${workspaceId}`
}
function parsePositiveLimit(value: unknown): number | null {
if (typeof value === 'number' && Number.isFinite(value) && value > 0) {
return Math.floor(value)
}
if (typeof value === 'string') {
const parsed = Number.parseInt(value, 10)
if (Number.isFinite(parsed) && parsed > 0) {
return parsed
}
}
return null
}
function getFreeConcurrencyLimit(): number {
return Number.parseInt(env.WORKSPACE_CONCURRENCY_FREE, 10) || 5
}
function getProConcurrencyLimit(): number {
return Number.parseInt(env.WORKSPACE_CONCURRENCY_PRO, 10) || 50
}
function getTeamConcurrencyLimit(): number {
return Number.parseInt(env.WORKSPACE_CONCURRENCY_TEAM, 10) || 200
}
function getEnterpriseDefaultConcurrencyLimit(): number {
return Number.parseInt(env.WORKSPACE_CONCURRENCY_ENTERPRISE, 10) || 200
}
function getEnterpriseConcurrencyLimit(metadata: unknown): number {
const enterpriseMetadata = parseEnterpriseWorkspaceConcurrencyMetadata(metadata)
return enterpriseMetadata?.workspaceConcurrencyLimit ?? getEnterpriseDefaultConcurrencyLimit()
}
function getPlanConcurrencyLimit(plan: string | null | undefined, metadata: unknown): number {
if (!isBillingEnabled) {
return getFreeConcurrencyLimit()
}
if (!plan) {
return getFreeConcurrencyLimit()
}
if (isEnterprise(plan)) {
return getEnterpriseConcurrencyLimit(metadata)
}
if (isTeam(plan)) {
return getTeamConcurrencyLimit()
}
const credits = getPlanTierCredits(plan)
if (credits >= 25_000) {
return getTeamConcurrencyLimit()
}
if (isPro(plan)) {
return getProConcurrencyLimit()
}
return getFreeConcurrencyLimit()
}
export async function getWorkspaceConcurrencyLimit(workspaceId: string): Promise<number> {
const redis = getRedisClient()
if (redis) {
const cached = await redis.get(cacheKey(workspaceId))
const cachedValue = parsePositiveLimit(cached)
if (cachedValue !== null) {
return cachedValue
}
} else {
const cached = inMemoryConcurrencyCache.get(workspaceId)
if (cached && cached.expiresAt > Date.now()) {
return cached.value
}
}
try {
const billedAccountUserId = await getWorkspaceBilledAccountUserId(workspaceId)
if (!billedAccountUserId) {
if (redis) {
await redis.set(
cacheKey(workspaceId),
String(getFreeConcurrencyLimit()),
'EX',
CACHE_TTL_SECONDS
)
} else {
inMemoryConcurrencyCache.set(workspaceId, {
value: getFreeConcurrencyLimit(),
expiresAt: Date.now() + CACHE_TTL_MS,
})
}
return getFreeConcurrencyLimit()
}
const subscription = await getHighestPrioritySubscription(billedAccountUserId)
const limit = getPlanConcurrencyLimit(subscription?.plan, subscription?.metadata)
if (redis) {
await redis.set(cacheKey(workspaceId), String(limit), 'EX', CACHE_TTL_SECONDS)
} else {
inMemoryConcurrencyCache.set(workspaceId, {
value: limit,
expiresAt: Date.now() + CACHE_TTL_MS,
})
}
return limit
} catch (error) {
logger.error('Failed to resolve workspace concurrency limit, using free tier', {
workspaceId,
error,
})
return getFreeConcurrencyLimit()
}
}
export async function resetWorkspaceConcurrencyLimitCache(workspaceId?: string): Promise<void> {
if (!workspaceId) {
inMemoryConcurrencyCache.clear()
} else {
inMemoryConcurrencyCache.delete(workspaceId)
}
const redis = getRedisClient()
if (!redis) {
return
}
if (workspaceId) {
await redis.del(cacheKey(workspaceId))
return
}
const keys = await redis.keys('workspace-concurrency-limit:*')
if (keys.length > 0) {
await redis.del(...keys)
}
}

111
apps/sim/lib/core/async-jobs/backends/bullmq.ts
View File

@@ -1,111 +0,0 @@
import { createLogger } from '@sim/logger'
import type { Job as BullMQJob } from 'bullmq'
import {
type EnqueueOptions,
JOB_STATUS,
type Job,
type JobQueueBackend,
type JobStatus,
type JobType,
} from '@/lib/core/async-jobs/types'
import { type BullMQJobData, createBullMQJobData, getBullMQQueue } from '@/lib/core/bullmq'
const logger = createLogger('BullMQJobQueue')
function mapBullMQStatus(status: string): JobStatus {
switch (status) {
case 'active':
return JOB_STATUS.PROCESSING
case 'completed':
return JOB_STATUS.COMPLETED
case 'failed':
return JOB_STATUS.FAILED
default:
return JOB_STATUS.PENDING
}
}
async function toJob(
queueType: JobType,
bullJob: BullMQJob<BullMQJobData<unknown>> | null
): Promise<Job | null> {
if (!bullJob) {
return null
}
const status = mapBullMQStatus(await bullJob.getState())
return {
id: bullJob.id ?? '',
type: queueType,
payload: bullJob.data.payload,
status,
createdAt: new Date(bullJob.timestamp),
startedAt: bullJob.processedOn ? new Date(bullJob.processedOn) : undefined,
completedAt: bullJob.finishedOn ? new Date(bullJob.finishedOn) : undefined,
attempts: bullJob.attemptsMade,
maxAttempts: bullJob.opts.attempts ?? 1,
error: bullJob.failedReason || undefined,
output: bullJob.returnvalue,
metadata: bullJob.data.metadata ?? {},
}
}
export class BullMQJobQueue implements JobQueueBackend {
async enqueue<TPayload>(
type: JobType,
payload: TPayload,
options?: EnqueueOptions
): Promise<string> {
const queue = getBullMQQueue(type)
const job = await queue.add(
options?.name ?? type,
createBullMQJobData(payload, options?.metadata),
{
jobId: options?.jobId,
attempts: options?.maxAttempts,
priority: options?.priority,
delay: options?.delayMs,
}
)
logger.debug('Enqueued job via BullMQ', {
jobId: job.id,
type,
name: options?.name ?? type,
})
return String(job.id)
}
async getJob(jobId: string): Promise<Job | null> {
const workflowJob = await getBullMQQueue('workflow-execution').getJob(jobId)
if (workflowJob) {
return toJob('workflow-execution', workflowJob)
}
const webhookJob = await getBullMQQueue('webhook-execution').getJob(jobId)
if (webhookJob) {
return toJob('webhook-execution', webhookJob)
}
const scheduleJob = await getBullMQQueue('schedule-execution').getJob(jobId)
if (scheduleJob) {
return toJob('schedule-execution', scheduleJob)
}
const resumeJob = await getBullMQQueue('resume-execution').getJob(jobId)
if (resumeJob) {
return toJob('resume-execution', resumeJob)
}
return null
}
async startJob(_jobId: string): Promise<void> {}
async completeJob(_jobId: string, _output: unknown): Promise<void> {}
async markJobFailed(_jobId: string, _error: string): Promise<void> {}
}

1
apps/sim/lib/core/async-jobs/backends/index.ts
View File

@@ -1,3 +1,2 @@
export { BullMQJobQueue } from './bullmq'
export { DatabaseJobQueue } from './database'
export { TriggerDevJobQueue } from './trigger-dev'

32
apps/sim/lib/core/async-jobs/config.ts
View File

@@ -1,6 +1,5 @@
import { createLogger } from '@sim/logger'
import type { AsyncBackendType, JobQueueBackend } from '@/lib/core/async-jobs/types'
import { isBullMQEnabled } from '@/lib/core/bullmq'
import { isTriggerDevEnabled } from '@/lib/core/config/feature-flags'
const logger = createLogger('AsyncJobsConfig')
@@ -11,17 +10,13 @@ let cachedInlineBackend: JobQueueBackend | null = null
/**
* Determines which async backend to use based on environment configuration.
* Follows the fallback chain: trigger.dev → bullmq → database
* Follows the fallback chain: trigger.dev → database
*/
export function getAsyncBackendType(): AsyncBackendType {
if (isTriggerDevEnabled) {
return 'trigger-dev'
}
if (isBullMQEnabled()) {
return 'bullmq'
}
return 'database'
}
@@ -42,11 +37,6 @@ export async function getJobQueue(): Promise<JobQueueBackend> {
cachedBackend = new TriggerDevJobQueue()
break
}
case 'bullmq': {
const { BullMQJobQueue } = await import('@/lib/core/async-jobs/backends/bullmq')
cachedBackend = new BullMQJobQueue()
break
}
case 'database': {
const { DatabaseJobQueue } = await import('@/lib/core/async-jobs/backends/database')
cachedBackend = new DatabaseJobQueue()
@@ -72,7 +62,7 @@ export function getCurrentBackendType(): AsyncBackendType | null {
}
/**
* Gets a job queue backend that bypasses Trigger.dev (BullMQ -> Database).
* Gets a job queue backend that bypasses Trigger.dev (Database only).
* Used for execution paths that must avoid Trigger.dev cold starts.
*/
export async function getInlineJobQueue(): Promise<JobQueueBackend> {
@@ -80,18 +70,10 @@ export async function getInlineJobQueue(): Promise<JobQueueBackend> {
return cachedInlineBackend
}
let type: string
if (isBullMQEnabled()) {
const { BullMQJobQueue } = await import('@/lib/core/async-jobs/backends/bullmq')
cachedInlineBackend = new BullMQJobQueue()
type = 'bullmq'
} else {
const { DatabaseJobQueue } = await import('@/lib/core/async-jobs/backends/database')
cachedInlineBackend = new DatabaseJobQueue()
type = 'database'
}
const { DatabaseJobQueue } = await import('@/lib/core/async-jobs/backends/database')
cachedInlineBackend = new DatabaseJobQueue()
logger.info(`Inline job backend initialized: ${type}`)
logger.info('Inline job backend initialized: database')
return cachedInlineBackend
}
@@ -103,10 +85,6 @@ export function shouldExecuteInline(): boolean {
return getAsyncBackendType() === 'database'
}
export function shouldUseBullMQ(): boolean {
return isBullMQEnabled()
}
/**
* Resets the cached backend (useful for testing)
*/

1
apps/sim/lib/core/async-jobs/index.ts
View File

@@ -5,7 +5,6 @@ export {
getJobQueue,
resetJobQueueCache,
shouldExecuteInline,
shouldUseBullMQ,
} from './config'
export type {
AsyncBackendType,

2
apps/sim/lib/core/async-jobs/types.ts
View File

@@ -105,4 +105,4 @@ export interface JobQueueBackend {
markJobFailed(jobId: string, error: string): Promise<void>
}
export type AsyncBackendType = 'trigger-dev' | 'bullmq' | 'database'
export type AsyncBackendType = 'trigger-dev' | 'database'

29
apps/sim/lib/core/bullmq/connection.ts
View File

@@ -1,29 +0,0 @@
import type { ConnectionOptions } from 'bullmq'
import { env, isTruthy } from '@/lib/core/config/env'
export function isBullMQEnabled(): boolean {
return isTruthy(env.CONCURRENCY_CONTROL_ENABLED) && Boolean(env.REDIS_URL)
}
export function getBullMQConnectionOptions(): ConnectionOptions {
if (!env.REDIS_URL) {
throw new Error('BullMQ requires REDIS_URL')
}
const redisUrl = new URL(env.REDIS_URL)
const isTls = redisUrl.protocol === 'rediss:'
const port = redisUrl.port ? Number.parseInt(redisUrl.port, 10) : 6379
const dbPath = redisUrl.pathname.replace('/', '')
const db = dbPath ? Number.parseInt(dbPath, 10) : undefined
return {
host: redisUrl.hostname,
port,
username: redisUrl.username || undefined,
password: redisUrl.password || undefined,
db: Number.isFinite(db) ? db : undefined,
maxRetriesPerRequest: null,
enableReadyCheck: false,
...(isTls ? { tls: {} } : {}),
}
}

16
apps/sim/lib/core/bullmq/index.ts
View File

@@ -1,16 +0,0 @@
export { getBullMQConnectionOptions, isBullMQEnabled } from './connection'
export {
type BullMQJobData,
createBullMQJobData,
getBullMQQueue,
getBullMQQueueByName,
getKnowledgeConnectorSyncQueue,
getKnowledgeDocumentProcessingQueue,
getMothershipJobExecutionQueue,
getWorkflowQueueEvents,
getWorkspaceNotificationDeliveryQueue,
KNOWLEDGE_CONNECTOR_SYNC_QUEUE,
KNOWLEDGE_DOCUMENT_PROCESSING_QUEUE,
MOTHERSHIP_JOB_EXECUTION_QUEUE,
WORKSPACE_NOTIFICATION_DELIVERY_QUEUE,
} from './queues'

209
apps/sim/lib/core/bullmq/queues.ts
View File

@@ -1,209 +0,0 @@
import { Queue, QueueEvents } from 'bullmq'
import type { JobMetadata, JobType } from '@/lib/core/async-jobs/types'
import { getBullMQConnectionOptions } from '@/lib/core/bullmq/connection'
import type { WorkspaceDispatchQueueName } from '@/lib/core/workspace-dispatch/types'
export const KNOWLEDGE_CONNECTOR_SYNC_QUEUE = 'knowledge-connector-sync' as const
export const KNOWLEDGE_DOCUMENT_PROCESSING_QUEUE = 'knowledge-process-document' as const
export const MOTHERSHIP_JOB_EXECUTION_QUEUE = 'mothership-job-execution' as const
export const WORKSPACE_NOTIFICATION_DELIVERY_QUEUE = 'workspace-notification-delivery' as const
export interface BullMQJobData<TPayload> {
payload: TPayload
metadata?: JobMetadata
}
let workflowQueueInstance: Queue | null = null
let webhookQueueInstance: Queue | null = null
let scheduleQueueInstance: Queue | null = null
let resumeQueueInstance: Queue | null = null
let knowledgeConnectorSyncQueueInstance: Queue | null = null
let knowledgeDocumentProcessingQueueInstance: Queue | null = null
let mothershipJobExecutionQueueInstance: Queue | null = null
let workspaceNotificationDeliveryQueueInstance: Queue | null = null
let workflowQueueEventsInstance: QueueEvents | null = null
function getQueueDefaultOptions(type: JobType) {
switch (type) {
case 'workflow-execution':
return {
attempts: 3,
backoff: { type: 'exponential' as const, delay: 1000 },
removeOnComplete: { age: 24 * 60 * 60 },
removeOnFail: { age: 7 * 24 * 60 * 60 },
}
case 'webhook-execution':
return {
attempts: 2,
backoff: { type: 'exponential' as const, delay: 2000 },
removeOnComplete: { age: 24 * 60 * 60 },
removeOnFail: { age: 3 * 24 * 60 * 60 },
}
case 'schedule-execution':
return {
attempts: 2,
backoff: { type: 'exponential' as const, delay: 5000 },
removeOnComplete: { age: 24 * 60 * 60 },
removeOnFail: { age: 3 * 24 * 60 * 60 },
}
case 'resume-execution':
return {
attempts: 1,
removeOnComplete: { age: 24 * 60 * 60 },
removeOnFail: { age: 3 * 24 * 60 * 60 },
}
}
}
function createQueue(type: JobType): Queue {
return new Queue(type, {
connection: getBullMQConnectionOptions(),
defaultJobOptions: getQueueDefaultOptions(type),
})
}
function createNamedQueue(
name:
| typeof KNOWLEDGE_CONNECTOR_SYNC_QUEUE
| typeof KNOWLEDGE_DOCUMENT_PROCESSING_QUEUE
| typeof MOTHERSHIP_JOB_EXECUTION_QUEUE
| typeof WORKSPACE_NOTIFICATION_DELIVERY_QUEUE
): Queue {
switch (name) {
case KNOWLEDGE_CONNECTOR_SYNC_QUEUE:
return new Queue(name, {
connection: getBullMQConnectionOptions(),
defaultJobOptions: {
attempts: 3,
backoff: { type: 'exponential', delay: 5000 },
removeOnComplete: { age: 24 * 60 * 60 },
removeOnFail: { age: 7 * 24 * 60 * 60 },
},
})
case KNOWLEDGE_DOCUMENT_PROCESSING_QUEUE:
return new Queue(name, {
connection: getBullMQConnectionOptions(),
defaultJobOptions: {
attempts: 3,
backoff: { type: 'exponential', delay: 1000 },
removeOnComplete: { age: 24 * 60 * 60 },
removeOnFail: { age: 7 * 24 * 60 * 60 },
},
})
case MOTHERSHIP_JOB_EXECUTION_QUEUE:
return new Queue(name, {
connection: getBullMQConnectionOptions(),
defaultJobOptions: {
attempts: 1,
removeOnComplete: { age: 24 * 60 * 60 },
removeOnFail: { age: 7 * 24 * 60 * 60 },
},
})
case WORKSPACE_NOTIFICATION_DELIVERY_QUEUE:
return new Queue(name, {
connection: getBullMQConnectionOptions(),
defaultJobOptions: {
attempts: 1,
removeOnComplete: { age: 24 * 60 * 60 },
removeOnFail: { age: 7 * 24 * 60 * 60 },
},
})
}
}
export function getBullMQQueue(type: JobType): Queue {
switch (type) {
case 'workflow-execution':
if (!workflowQueueInstance) {
workflowQueueInstance = createQueue(type)
}
return workflowQueueInstance
case 'webhook-execution':
if (!webhookQueueInstance) {
webhookQueueInstance = createQueue(type)
}
return webhookQueueInstance
case 'schedule-execution':
if (!scheduleQueueInstance) {
scheduleQueueInstance = createQueue(type)
}
return scheduleQueueInstance
case 'resume-execution':
if (!resumeQueueInstance) {
resumeQueueInstance = createQueue(type)
}
return resumeQueueInstance
}
}
export function getBullMQQueueByName(queueName: WorkspaceDispatchQueueName): Queue {
switch (queueName) {
case 'workflow-execution':
case 'webhook-execution':
case 'schedule-execution':
case 'resume-execution':
return getBullMQQueue(queueName)
case KNOWLEDGE_CONNECTOR_SYNC_QUEUE:
return getKnowledgeConnectorSyncQueue()
case KNOWLEDGE_DOCUMENT_PROCESSING_QUEUE:
return getKnowledgeDocumentProcessingQueue()
case MOTHERSHIP_JOB_EXECUTION_QUEUE:
return getMothershipJobExecutionQueue()
case WORKSPACE_NOTIFICATION_DELIVERY_QUEUE:
return getWorkspaceNotificationDeliveryQueue()
}
}
export function getWorkflowQueueEvents(): QueueEvents {
if (!workflowQueueEventsInstance) {
workflowQueueEventsInstance = new QueueEvents('workflow-execution', {
connection: getBullMQConnectionOptions(),
})
}
return workflowQueueEventsInstance
}
export function getKnowledgeConnectorSyncQueue(): Queue {
if (!knowledgeConnectorSyncQueueInstance) {
knowledgeConnectorSyncQueueInstance = createNamedQueue(KNOWLEDGE_CONNECTOR_SYNC_QUEUE)
}
return knowledgeConnectorSyncQueueInstance
}
export function getKnowledgeDocumentProcessingQueue(): Queue {
if (!knowledgeDocumentProcessingQueueInstance) {
knowledgeDocumentProcessingQueueInstance = createNamedQueue(KNOWLEDGE_DOCUMENT_PROCESSING_QUEUE)
}
return knowledgeDocumentProcessingQueueInstance
}
export function getMothershipJobExecutionQueue(): Queue {
if (!mothershipJobExecutionQueueInstance) {
mothershipJobExecutionQueueInstance = createNamedQueue(MOTHERSHIP_JOB_EXECUTION_QUEUE)
}
return mothershipJobExecutionQueueInstance
}
export function getWorkspaceNotificationDeliveryQueue(): Queue {
if (!workspaceNotificationDeliveryQueueInstance) {
workspaceNotificationDeliveryQueueInstance = createNamedQueue(
WORKSPACE_NOTIFICATION_DELIVERY_QUEUE
)
}
return workspaceNotificationDeliveryQueueInstance
}
export function createBullMQJobData<TPayload>(
payload: TPayload,
metadata?: JobMetadata
): BullMQJobData<TPayload> {
return {
payload,
metadata: metadata ?? {},
}
}

8
apps/sim/lib/core/config/env.ts
View File

@@ -190,10 +190,7 @@ export const env = createEnv({
FREE_PLAN_LOG_RETENTION_DAYS: z.string().optional(), // Log retention days for free plan users
// Admission & Burst Protection
CONCURRENCY_CONTROL_ENABLED: z.string().optional().default('false'), // Set to 'true' to enable BullMQ-based concurrency control (default: inline execution)
ADMISSION_GATE_MAX_INFLIGHT: z.string().optional().default('500'), // Max concurrent in-flight execution requests per pod
DISPATCH_MAX_QUEUE_PER_WORKSPACE: z.string().optional().default('1000'), // Max queued dispatch jobs per workspace
DISPATCH_MAX_QUEUE_GLOBAL: z.string().optional().default('50000'), // Max queued dispatch jobs globally
// Rate Limiting Configuration
RATE_LIMIT_WINDOW_MS: z.string().optional().default('60000'), // Rate limit window duration in milliseconds (default: 1 minute)
@@ -206,11 +203,6 @@ export const env = createEnv({
RATE_LIMIT_TEAM_ASYNC: z.string().optional().default('2500'), // Team tier async API executions per minute
RATE_LIMIT_ENTERPRISE_SYNC: z.string().optional().default('600'), // Enterprise tier sync API executions per minute
RATE_LIMIT_ENTERPRISE_ASYNC: z.string().optional().default('5000'), // Enterprise tier async API executions per minute
WORKSPACE_CONCURRENCY_FREE: z.string().optional().default('5'), // Free tier concurrent workspace executions
WORKSPACE_CONCURRENCY_PRO: z.string().optional().default('50'), // Pro tier concurrent workspace executions
WORKSPACE_CONCURRENCY_TEAM: z.string().optional().default('200'), // Team/Max tier concurrent workspace executions
WORKSPACE_CONCURRENCY_ENTERPRISE: z.string().optional().default('200'), // Enterprise default concurrent workspace executions
// Timeout Configuration
EXECUTION_TIMEOUT_FREE: z.string().optional().default('300'), // 5 minutes
EXECUTION_TIMEOUT_PRO: z.string().optional().default('3000'), // 50 minutes

80
apps/sim/lib/core/workspace-dispatch/adapter.ts
View File

@@ -1,80 +0,0 @@
import type {
WorkspaceDispatchClaimResult,
WorkspaceDispatchEnqueueInput,
WorkspaceDispatchJobRecord,
WorkspaceDispatchLane,
} from '@/lib/core/workspace-dispatch/types'
export interface WorkspaceDispatchStorageAdapter {
saveDispatchJob(record: WorkspaceDispatchJobRecord): Promise<void>
getDispatchJobRecord(jobId: string): Promise<WorkspaceDispatchJobRecord | null>
listDispatchJobsByStatuses(
statuses: readonly WorkspaceDispatchJobRecord['status'][]
): Promise<WorkspaceDispatchJobRecord[]>
updateDispatchJobRecord(
jobId: string,
updater: (record: WorkspaceDispatchJobRecord) => WorkspaceDispatchJobRecord
): Promise<WorkspaceDispatchJobRecord | null>
enqueueWorkspaceDispatchJob(
input: WorkspaceDispatchEnqueueInput
): Promise<WorkspaceDispatchJobRecord>
restoreWorkspaceDispatchJob(record: WorkspaceDispatchJobRecord): Promise<void>
claimWorkspaceJob(
workspaceId: string,
options: {
lanes: readonly WorkspaceDispatchLane[]
concurrencyLimit: number
leaseId: string
now: number
leaseTtlMs: number
}
): Promise<WorkspaceDispatchClaimResult>
getWorkspaceQueueDepth(
workspaceId: string,
lanes: readonly WorkspaceDispatchLane[]
): Promise<number>
getGlobalQueueDepth(): Promise<number>
reconcileGlobalQueueDepth(knownCount: number): Promise<void>
popNextWorkspaceId(): Promise<string | null>
getQueuedWorkspaceCount(): Promise<number>
hasActiveWorkspace(workspaceId: string): Promise<boolean>
ensureWorkspaceActive(workspaceId: string, readyAt?: number): Promise<void>
requeueWorkspaceId(workspaceId: string): Promise<void>
workspaceHasPendingJobs(
workspaceId: string,
lanes: readonly WorkspaceDispatchLane[]
): Promise<boolean>
getNextWorkspaceJob(
workspaceId: string,
lanes: readonly WorkspaceDispatchLane[]
): Promise<WorkspaceDispatchJobRecord | null>
removeWorkspaceJobFromLane(
workspaceId: string,
lane: WorkspaceDispatchLane,
jobId: string
): Promise<void>
cleanupExpiredWorkspaceLeases(workspaceId: string): Promise<void>
countActiveWorkspaceLeases(workspaceId: string): Promise<number>
hasWorkspaceLease(workspaceId: string, leaseId: string): Promise<boolean>
createWorkspaceLease(workspaceId: string, leaseId: string, ttlMs: number): Promise<number>
refreshWorkspaceLease(workspaceId: string, leaseId: string, ttlMs: number): Promise<number>
releaseWorkspaceLease(workspaceId: string, leaseId: string): Promise<void>
removeWorkspaceIfIdle(workspaceId: string, lanes: readonly WorkspaceDispatchLane[]): Promise<void>
markDispatchJobAdmitted(
jobId: string,
workspaceId: string,
leaseId: string,
leaseExpiresAt: number
): Promise<void>
markDispatchJobAdmitting(
jobId: string,
workspaceId: string,
leaseId: string,
leaseExpiresAt: number
): Promise<void>
markDispatchJobRunning(jobId: string): Promise<void>
markDispatchJobCompleted(jobId: string, output: unknown): Promise<void>
markDispatchJobFailed(jobId: string, error: string): Promise<void>
clear(): Promise<void>
dispose(): void
}

175
apps/sim/lib/core/workspace-dispatch/dispatcher.test.ts
View File

@@ -1,175 +0,0 @@
/**
* @vitest-environment node
*/
import { beforeEach, describe, expect, it, vi } from 'vitest'
const { mockGetWorkspaceConcurrencyLimit, mockAcquireLock, mockReleaseLock } = vi.hoisted(() => ({
mockGetWorkspaceConcurrencyLimit: vi.fn(),
mockAcquireLock: vi.fn(),
mockReleaseLock: vi.fn(),
}))
vi.mock('@/lib/billing/workspace-concurrency', () => ({
getWorkspaceConcurrencyLimit: mockGetWorkspaceConcurrencyLimit,
}))
vi.mock('@/lib/core/config/redis', () => ({
acquireLock: mockAcquireLock,
releaseLock: mockReleaseLock,
getRedisClient: vi.fn().mockReturnValue(null),
}))
vi.mock('@/lib/core/bullmq', () => ({
getBullMQQueueByName: vi.fn().mockReturnValue({
add: vi.fn().mockResolvedValue({ id: 'bullmq-1' }),
}),
}))
import { MemoryWorkspaceDispatchStorage } from '@/lib/core/workspace-dispatch/memory-store'
import {
DISPATCH_SCAN_RESULTS,
dispatchNextAdmissibleWorkspaceJob,
} from '@/lib/core/workspace-dispatch/planner'
import {
enqueueWorkspaceDispatchJob,
setWorkspaceDispatchStorageAdapter,
} from '@/lib/core/workspace-dispatch/store'
describe('workspace dispatch integration (memory-backed)', () => {
let store: MemoryWorkspaceDispatchStorage
beforeEach(async () => {
vi.clearAllMocks()
store = new MemoryWorkspaceDispatchStorage()
setWorkspaceDispatchStorageAdapter(store)
mockGetWorkspaceConcurrencyLimit.mockResolvedValue(5)
mockAcquireLock.mockResolvedValue(true)
mockReleaseLock.mockResolvedValue(true)
})
async function enqueue(
workspaceId: string,
overrides: { lane?: string; delayMs?: number; priority?: number } = {}
) {
return enqueueWorkspaceDispatchJob({
workspaceId,
lane: (overrides.lane ?? 'runtime') as 'runtime',
queueName: 'workflow-execution',
bullmqJobName: 'workflow-execution',
bullmqPayload: { payload: { workflowId: 'wf-1' } },
metadata: { workflowId: 'wf-1' },
delayMs: overrides.delayMs,
priority: overrides.priority,
})
}
it('admits jobs round-robin across workspaces', async () => {
await enqueue('ws-a')
await enqueue('ws-b')
await enqueue('ws-a')
const r1 = await dispatchNextAdmissibleWorkspaceJob()
const r2 = await dispatchNextAdmissibleWorkspaceJob()
const r3 = await dispatchNextAdmissibleWorkspaceJob()
expect(r1).toBe(DISPATCH_SCAN_RESULTS.ADMITTED)
expect(r2).toBe(DISPATCH_SCAN_RESULTS.ADMITTED)
expect(r3).toBe(DISPATCH_SCAN_RESULTS.ADMITTED)
})
it('respects workspace concurrency limits', async () => {
mockGetWorkspaceConcurrencyLimit.mockResolvedValue(1)
await enqueue('ws-a')
await enqueue('ws-a')
const r1 = await dispatchNextAdmissibleWorkspaceJob()
expect(r1).toBe(DISPATCH_SCAN_RESULTS.ADMITTED)
const r2 = await dispatchNextAdmissibleWorkspaceJob()
expect(r2).toBe(DISPATCH_SCAN_RESULTS.NO_PROGRESS)
})
it('skips delayed jobs and admits ready ones in same lane', async () => {
await enqueue('ws-a', { delayMs: 60_000 })
await enqueue('ws-a', { delayMs: 0 })
const r1 = await dispatchNextAdmissibleWorkspaceJob()
expect(r1).toBe(DISPATCH_SCAN_RESULTS.ADMITTED)
})
it('returns delayed when all jobs are delayed', async () => {
await enqueue('ws-a', { delayMs: 60_000 })
const r1 = await dispatchNextAdmissibleWorkspaceJob()
expect(r1).toBe(DISPATCH_SCAN_RESULTS.NO_PROGRESS)
})
it('returns no_workspace when queue is empty', async () => {
const result = await dispatchNextAdmissibleWorkspaceJob()
expect(result).toBe(DISPATCH_SCAN_RESULTS.NO_WORKSPACE)
})
it('lease cleanup frees capacity for new admissions', async () => {
mockGetWorkspaceConcurrencyLimit.mockResolvedValue(1)
const record = await enqueue('ws-a')
await enqueue('ws-a')
const r1 = await dispatchNextAdmissibleWorkspaceJob()
expect(r1).toBe(DISPATCH_SCAN_RESULTS.ADMITTED)
const updated = await store.getDispatchJobRecord(record.id)
if (updated?.lease) {
await store.releaseWorkspaceLease('ws-a', updated.lease.leaseId)
}
const r2 = await dispatchNextAdmissibleWorkspaceJob()
expect(r2).toBe(DISPATCH_SCAN_RESULTS.ADMITTED)
})
it('expired leases are cleaned up during claim', async () => {
mockGetWorkspaceConcurrencyLimit.mockResolvedValue(1)
await enqueue('ws-a')
await enqueue('ws-a')
const claimResult = await store.claimWorkspaceJob('ws-a', {
lanes: ['runtime'],
concurrencyLimit: 1,
leaseId: 'old-lease',
now: Date.now(),
leaseTtlMs: 1,
})
expect(claimResult.type).toBe('admitted')
await new Promise((resolve) => setTimeout(resolve, 10))
const r2 = await dispatchNextAdmissibleWorkspaceJob()
expect(r2).toBe(DISPATCH_SCAN_RESULTS.ADMITTED)
})
it('recovers job to waiting via restoreWorkspaceDispatchJob', async () => {
const record = await enqueue('ws-a')
await store.claimWorkspaceJob('ws-a', {
lanes: ['runtime'],
concurrencyLimit: 1,
leaseId: 'lease-1',
now: Date.now(),
leaseTtlMs: 1000,
})
await store.markDispatchJobAdmitted(record.id, 'ws-a', 'lease-1', Date.now() + 10000)
const admitted = await store.getDispatchJobRecord(record.id)
expect(admitted).toBeDefined()
const resetRecord = { ...admitted!, status: 'waiting' as const, lease: undefined }
await store.restoreWorkspaceDispatchJob(resetRecord)
const restored = await store.getDispatchJobRecord(record.id)
expect(restored?.status).toBe('waiting')
expect(restored?.lease).toBeUndefined()
})
})

156
apps/sim/lib/core/workspace-dispatch/dispatcher.ts
View File

@@ -1,156 +0,0 @@
import { createLogger } from '@sim/logger'
import { env } from '@/lib/core/config/env'
import {
enqueueWorkspaceDispatchJob,
getDispatchJobRecord,
getGlobalQueueDepth,
getQueuedWorkspaceCount,
getWorkspaceQueueDepth,
} from '@/lib/core/workspace-dispatch/store'
import {
WORKSPACE_DISPATCH_LANES,
type WorkspaceDispatchEnqueueInput,
type WorkspaceDispatchJobRecord,
} from '@/lib/core/workspace-dispatch/types'
import { DISPATCH_SCAN_RESULTS, dispatchNextAdmissibleWorkspaceJob } from './planner'
import { reconcileWorkspaceDispatchState } from './reconciler'
const logger = createLogger('WorkspaceDispatcher')
const WAIT_POLL_INTERVAL_MS = 250
const RECONCILE_INTERVAL_MS = 30_000
const MAX_QUEUE_PER_WORKSPACE = Number.parseInt(env.DISPATCH_MAX_QUEUE_PER_WORKSPACE ?? '') || 1000
const MAX_QUEUE_GLOBAL = Number.parseInt(env.DISPATCH_MAX_QUEUE_GLOBAL ?? '') || 50_000
let dispatcherRunning = false
let dispatcherWakePending = false
let lastReconcileAt = 0
async function runDispatcherLoop(): Promise<void> {
if (dispatcherRunning) {
dispatcherWakePending = true
return
}
dispatcherRunning = true
try {
const now = Date.now()
if (now - lastReconcileAt >= RECONCILE_INTERVAL_MS) {
await reconcileWorkspaceDispatchState()
lastReconcileAt = now
}
do {
dispatcherWakePending = false
const queuedWorkspaces = await getQueuedWorkspaceCount()
if (queuedWorkspaces === 0) {
continue
}
let admitted = 0
let scanned = 0
const loopStartMs = Date.now()
for (let index = 0; index < queuedWorkspaces; index++) {
scanned++
const result = await dispatchNextAdmissibleWorkspaceJob()
if (result === DISPATCH_SCAN_RESULTS.ADMITTED) {
admitted++
}
if (result === DISPATCH_SCAN_RESULTS.NO_WORKSPACE) {
break
}
}
if (admitted > 0) {
dispatcherWakePending = true
}
if (admitted > 0 || scanned > 0) {
logger.info('Dispatcher pass', {
admitted,
scanned,
queuedWorkspaces,
durationMs: Date.now() - loopStartMs,
})
}
} while (dispatcherWakePending)
} catch (error) {
logger.error('Workspace dispatcher loop failed', { error })
} finally {
dispatcherRunning = false
}
}
export class DispatchQueueFullError extends Error {
readonly statusCode = 503
constructor(
readonly scope: 'workspace' | 'global',
readonly depth: number,
readonly limit: number
) {
super(
scope === 'workspace'
? `Workspace queue is at capacity (${depth}/${limit})`
: `Global dispatch queue is at capacity (${depth}/${limit})`
)
this.name = 'DispatchQueueFullError'
}
}
export async function enqueueWorkspaceDispatch(
input: WorkspaceDispatchEnqueueInput
): Promise<string> {
const [workspaceDepth, globalDepth] = await Promise.all([
getWorkspaceQueueDepth(input.workspaceId, WORKSPACE_DISPATCH_LANES),
getGlobalQueueDepth(),
])
if (workspaceDepth >= MAX_QUEUE_PER_WORKSPACE) {
logger.warn('Workspace dispatch queue at capacity', {
workspaceId: input.workspaceId,
depth: workspaceDepth,
limit: MAX_QUEUE_PER_WORKSPACE,
})
throw new DispatchQueueFullError('workspace', workspaceDepth, MAX_QUEUE_PER_WORKSPACE)
}
if (globalDepth >= MAX_QUEUE_GLOBAL) {
logger.warn('Global dispatch queue at capacity', {
depth: globalDepth,
limit: MAX_QUEUE_GLOBAL,
})
throw new DispatchQueueFullError('global', globalDepth, MAX_QUEUE_GLOBAL)
}
const record = await enqueueWorkspaceDispatchJob(input)
void runDispatcherLoop()
return record.id
}
export async function wakeWorkspaceDispatcher(): Promise<void> {
await runDispatcherLoop()
}
export async function waitForDispatchJob(
dispatchJobId: string,
timeoutMs: number
): Promise<WorkspaceDispatchJobRecord> {
const deadline = Date.now() + timeoutMs
while (Date.now() < deadline) {
const record = await getDispatchJobRecord(dispatchJobId)
if (!record) {
throw new Error(`Dispatch job not found: ${dispatchJobId}`)
}
if (record.status === 'completed' || record.status === 'failed') {
return record
}
await new Promise((resolve) => setTimeout(resolve, WAIT_POLL_INTERVAL_MS))
}
throw new Error(`Timed out waiting for dispatch job ${dispatchJobId}`)
}

42
apps/sim/lib/core/workspace-dispatch/factory.ts
View File

@@ -1,42 +0,0 @@
import { createLogger } from '@sim/logger'
import { getRedisClient } from '@/lib/core/config/redis'
import type { WorkspaceDispatchStorageAdapter } from '@/lib/core/workspace-dispatch/adapter'
import { MemoryWorkspaceDispatchStorage } from '@/lib/core/workspace-dispatch/memory-store'
import { RedisWorkspaceDispatchStorage } from '@/lib/core/workspace-dispatch/redis-store'
const logger = createLogger('WorkspaceDispatchFactory')
let cachedAdapter: WorkspaceDispatchStorageAdapter | null = null
export function createWorkspaceDispatchStorageAdapter(): WorkspaceDispatchStorageAdapter {
if (cachedAdapter) {
return cachedAdapter
}
const redis = getRedisClient()
if (redis) {
logger.info('Workspace dispatcher: Using Redis storage')
const adapter = new RedisWorkspaceDispatchStorage(redis)
cachedAdapter = adapter
return adapter
}
logger.warn(
'Workspace dispatcher: Using in-memory storage; distributed fairness is disabled in multi-process deployments'
)
const adapter = new MemoryWorkspaceDispatchStorage()
cachedAdapter = adapter
return adapter
}
export function setWorkspaceDispatchStorageAdapter(adapter: WorkspaceDispatchStorageAdapter): void {
cachedAdapter = adapter
}
export function resetWorkspaceDispatchStorageAdapter(): void {
if (cachedAdapter) {
cachedAdapter.dispose()
cachedAdapter = null
}
}

32
apps/sim/lib/core/workspace-dispatch/index.ts
View File

@@ -1,32 +0,0 @@
export type { WorkspaceDispatchStorageAdapter } from './adapter'
export {
DispatchQueueFullError,
enqueueWorkspaceDispatch,
waitForDispatchJob,
wakeWorkspaceDispatcher,
} from './dispatcher'
export {
createWorkspaceDispatchStorageAdapter,
resetWorkspaceDispatchStorageAdapter,
} from './factory'
export {
markDispatchJobAdmitted,
markDispatchJobAdmitting,
markDispatchJobCompleted,
markDispatchJobFailed,
markDispatchJobRunning,
refreshWorkspaceLease,
releaseWorkspaceLease,
} from './store'
export {
WORKSPACE_DISPATCH_LANES,
WORKSPACE_DISPATCH_STATUSES,
type WorkspaceDispatchEnqueueInput,
type WorkspaceDispatchJobContext,
type WorkspaceDispatchJobRecord,
type WorkspaceDispatchLane,
type WorkspaceDispatchLeaseInfo,
type WorkspaceDispatchQueueName,
type WorkspaceDispatchStatus,
} from './types'
export { getDispatchRuntimeMetadata, runDispatchedJob } from './worker'

65
apps/sim/lib/core/workspace-dispatch/memory-store.test.ts
View File

@@ -1,65 +0,0 @@
/**
* @vitest-environment node
*/
import { afterEach, describe, expect, it } from 'vitest'
import { MemoryWorkspaceDispatchStorage } from '@/lib/core/workspace-dispatch/memory-store'
describe('memory workspace dispatch storage', () => {
const store = new MemoryWorkspaceDispatchStorage()
afterEach(async () => {
await store.clear()
})
it('claims a runnable job and marks it admitting with a lease', async () => {
const record = await store.enqueueWorkspaceDispatchJob({
workspaceId: 'workspace-1',
lane: 'runtime',
queueName: 'workflow-execution',
bullmqJobName: 'workflow-execution',
bullmqPayload: { payload: { workflowId: 'workflow-1' } },
metadata: {
workflowId: 'workflow-1',
},
})
const result = await store.claimWorkspaceJob('workspace-1', {
lanes: ['runtime'],
concurrencyLimit: 1,
leaseId: 'lease-1',
now: Date.now(),
leaseTtlMs: 1000,
})
expect(result.type).toBe('admitted')
if (result.type === 'admitted') {
expect(result.record.id).toBe(record.id)
expect(result.record.status).toBe('admitting')
expect(result.record.lease?.leaseId).toBe('lease-1')
}
})
it('returns delayed when only delayed jobs exist', async () => {
await store.enqueueWorkspaceDispatchJob({
workspaceId: 'workspace-1',
lane: 'runtime',
queueName: 'workflow-execution',
bullmqJobName: 'workflow-execution',
bullmqPayload: { payload: { workflowId: 'workflow-1' } },
metadata: {
workflowId: 'workflow-1',
},
delayMs: 5000,
})
const result = await store.claimWorkspaceJob('workspace-1', {
lanes: ['runtime'],
concurrencyLimit: 1,
leaseId: 'lease-2',
now: Date.now(),
leaseTtlMs: 1000,
})
expect(result.type).toBe('delayed')
})
})

506
apps/sim/lib/core/workspace-dispatch/memory-store.ts
View File

@@ -1,506 +0,0 @@
import { createLogger } from '@sim/logger'
import { generateId } from '@/lib/core/utils/uuid'
import type { WorkspaceDispatchStorageAdapter } from '@/lib/core/workspace-dispatch/adapter'
import {
WORKSPACE_DISPATCH_CLAIM_RESULTS,
type WorkspaceDispatchClaimResult,
type WorkspaceDispatchEnqueueInput,
type WorkspaceDispatchJobRecord,
type WorkspaceDispatchLane,
} from '@/lib/core/workspace-dispatch/types'
const logger = createLogger('WorkspaceDispatchMemoryStore')
const JOB_TTL_MS = 48 * 60 * 60 * 1000
export class MemoryWorkspaceDispatchStorage implements WorkspaceDispatchStorageAdapter {
private jobs = new Map<string, WorkspaceDispatchJobRecord>()
private workspaceOrder: string[] = []
private laneQueues = new Map<string, string[]>()
private leases = new Map<string, Map<string, number>>()
private cleanupInterval: NodeJS.Timeout | null = null
constructor() {
this.cleanupInterval = setInterval(() => {
void this.clearExpiredState()
}, 60_000)
this.cleanupInterval.unref()
}
private queueKey(workspaceId: string, lane: WorkspaceDispatchLane): string {
return `${workspaceId}:${lane}`
}
private ensureWorkspaceQueued(workspaceId: string): void {
if (!this.workspaceOrder.includes(workspaceId)) {
this.workspaceOrder.push(workspaceId)
}
}
private getLaneQueue(workspaceId: string, lane: WorkspaceDispatchLane): string[] {
const key = this.queueKey(workspaceId, lane)
const existing = this.laneQueues.get(key)
if (existing) {
return existing
}
const queue: string[] = []
this.laneQueues.set(key, queue)
return queue
}
private sortQueue(queue: string[]): void {
queue.sort((leftId, rightId) => {
const left = this.jobs.get(leftId)
const right = this.jobs.get(rightId)
if (!left || !right) {
return 0
}
if (left.priority !== right.priority) {
return left.priority - right.priority
}
return left.createdAt - right.createdAt
})
}
private getLeaseMap(workspaceId: string): Map<string, number> {
const existing = this.leases.get(workspaceId)
if (existing) {
return existing
}
const leaseMap = new Map<string, number>()
this.leases.set(workspaceId, leaseMap)
return leaseMap
}
private async clearExpiredState(): Promise<void> {
const now = Date.now()
for (const [jobId, record] of this.jobs.entries()) {
if (
(record.status === 'completed' || record.status === 'failed') &&
record.completedAt &&
now - record.completedAt > JOB_TTL_MS
) {
this.jobs.delete(jobId)
}
}
for (const [workspaceId, leaseMap] of this.leases.entries()) {
for (const [leaseId, expiresAt] of leaseMap.entries()) {
if (expiresAt <= now) {
leaseMap.delete(leaseId)
}
}
if (leaseMap.size === 0) {
this.leases.delete(workspaceId)
}
}
}
async saveDispatchJob(record: WorkspaceDispatchJobRecord): Promise<void> {
this.jobs.set(record.id, record)
}
async getDispatchJobRecord(jobId: string): Promise<WorkspaceDispatchJobRecord | null> {
return this.jobs.get(jobId) ?? null
}
async listDispatchJobsByStatuses(
statuses: readonly WorkspaceDispatchJobRecord['status'][]
): Promise<WorkspaceDispatchJobRecord[]> {
return Array.from(this.jobs.values()).filter((record) => statuses.includes(record.status))
}
private static readonly TERMINAL_STATUSES = new Set(['completed', 'failed'])
async updateDispatchJobRecord(
jobId: string,
updater: (record: WorkspaceDispatchJobRecord) => WorkspaceDispatchJobRecord
): Promise<WorkspaceDispatchJobRecord | null> {
const current = this.jobs.get(jobId)
if (!current) {
return null
}
const updated = updater(current)
if (
MemoryWorkspaceDispatchStorage.TERMINAL_STATUSES.has(current.status) &&
!MemoryWorkspaceDispatchStorage.TERMINAL_STATUSES.has(updated.status)
) {
return current
}
this.jobs.set(jobId, updated)
return updated
}
async enqueueWorkspaceDispatchJob(
input: WorkspaceDispatchEnqueueInput
): Promise<WorkspaceDispatchJobRecord> {
const id = input.id ?? `dispatch_${generateId().replace(/-/g, '').slice(0, 20)}`
const createdAt = Date.now()
const record: WorkspaceDispatchJobRecord = {
id,
workspaceId: input.workspaceId,
lane: input.lane,
queueName: input.queueName,
bullmqJobName: input.bullmqJobName,
bullmqPayload: input.bullmqPayload,
metadata: input.metadata,
priority: input.priority ?? 100,
maxAttempts: input.maxAttempts,
delayMs: input.delayMs,
status: 'waiting',
createdAt,
}
this.jobs.set(id, record)
const queue = this.getLaneQueue(record.workspaceId, record.lane)
queue.push(id)
this.sortQueue(queue)
this.ensureWorkspaceQueued(record.workspaceId)
return record
}
async restoreWorkspaceDispatchJob(record: WorkspaceDispatchJobRecord): Promise<void> {
this.jobs.set(record.id, record)
const queue = this.getLaneQueue(record.workspaceId, record.lane)
if (!queue.includes(record.id)) {
queue.push(record.id)
this.sortQueue(queue)
}
this.ensureWorkspaceQueued(record.workspaceId)
}
async claimWorkspaceJob(
workspaceId: string,
options: {
lanes: readonly WorkspaceDispatchLane[]
concurrencyLimit: number
leaseId: string
now: number
leaseTtlMs: number
}
): Promise<WorkspaceDispatchClaimResult> {
await this.cleanupExpiredWorkspaceLeases(workspaceId)
if (this.getLeaseMap(workspaceId).size >= options.concurrencyLimit) {
this.ensureWorkspaceQueued(workspaceId)
return { type: WORKSPACE_DISPATCH_CLAIM_RESULTS.LIMIT_REACHED }
}
let selectedRecord: WorkspaceDispatchJobRecord | null = null
let selectedLane: WorkspaceDispatchLane | null = null
let nextReadyAt: number | null = null
for (const lane of options.lanes) {
const queue = this.getLaneQueue(workspaceId, lane)
for (let scanIndex = 0; scanIndex < queue.length && scanIndex < 20; ) {
const jobId = queue[scanIndex]
const record = this.jobs.get(jobId)
if (!record) {
queue.splice(scanIndex, 1)
continue
}
const readyAt = record.createdAt + (record.delayMs ?? 0)
if (readyAt <= options.now) {
selectedRecord = record
selectedLane = lane
queue.splice(scanIndex, 1)
break
}
nextReadyAt = nextReadyAt ? Math.min(nextReadyAt, readyAt) : readyAt
scanIndex++
}
if (selectedRecord) {
break
}
}
if (!selectedRecord || !selectedLane) {
const hasPending = await this.workspaceHasPendingJobs(workspaceId, options.lanes)
if (!hasPending) {
this.workspaceOrder = this.workspaceOrder.filter((value) => value !== workspaceId)
return { type: WORKSPACE_DISPATCH_CLAIM_RESULTS.EMPTY }
}
this.ensureWorkspaceQueued(workspaceId)
return {
type: WORKSPACE_DISPATCH_CLAIM_RESULTS.DELAYED,
nextReadyAt: nextReadyAt ?? options.now,
}
}
const leaseExpiresAt = options.now + options.leaseTtlMs
this.getLeaseMap(workspaceId).set(options.leaseId, leaseExpiresAt)
const updatedRecord: WorkspaceDispatchJobRecord = {
...selectedRecord,
status: 'admitting',
lease: {
workspaceId,
leaseId: options.leaseId,
},
metadata: {
...selectedRecord.metadata,
dispatchLeaseExpiresAt: leaseExpiresAt,
},
}
this.jobs.set(updatedRecord.id, updatedRecord)
const hasPending = await this.workspaceHasPendingJobs(workspaceId, options.lanes)
if (hasPending) {
this.ensureWorkspaceQueued(workspaceId)
} else {
this.workspaceOrder = this.workspaceOrder.filter((value) => value !== workspaceId)
}
return {
type: WORKSPACE_DISPATCH_CLAIM_RESULTS.ADMITTED,
record: updatedRecord,
leaseId: options.leaseId,
leaseExpiresAt,
}
}
async getWorkspaceQueueDepth(
workspaceId: string,
lanes: readonly WorkspaceDispatchLane[]
): Promise<number> {
let depth = 0
for (const lane of lanes) {
depth += this.getLaneQueue(workspaceId, lane).length
}
return depth
}
async getGlobalQueueDepth(): Promise<number> {
const terminalStatuses = new Set(['completed', 'failed'])
let count = 0
for (const job of this.jobs.values()) {
if (!terminalStatuses.has(job.status)) {
count++
}
}
return count
}
async reconcileGlobalQueueDepth(_knownCount: number): Promise<void> {
// no-op: memory store computes depth on the fly
}
async popNextWorkspaceId(): Promise<string | null> {
const now = Date.now()
const maxScans = this.workspaceOrder.length
for (let i = 0; i < maxScans; i++) {
const id = this.workspaceOrder.shift()
if (!id) return null
const readyAt = this.workspaceReadyAt.get(id)
if (readyAt && readyAt > now) {
this.workspaceOrder.push(id)
continue
}
this.workspaceReadyAt.delete(id)
return id
}
return null
}
async getQueuedWorkspaceCount(): Promise<number> {
return this.workspaceOrder.length
}
async hasActiveWorkspace(workspaceId: string): Promise<boolean> {
return this.workspaceOrder.includes(workspaceId)
}
private workspaceReadyAt = new Map<string, number>()
async ensureWorkspaceActive(workspaceId: string, readyAt?: number): Promise<void> {
if (readyAt && readyAt > Date.now()) {
this.workspaceReadyAt.set(workspaceId, readyAt)
}
this.ensureWorkspaceQueued(workspaceId)
}
async requeueWorkspaceId(workspaceId: string): Promise<void> {
this.ensureWorkspaceQueued(workspaceId)
}
async workspaceHasPendingJobs(
workspaceId: string,
lanes: readonly WorkspaceDispatchLane[]
): Promise<boolean> {
return lanes.some((lane) => this.getLaneQueue(workspaceId, lane).length > 0)
}
async getNextWorkspaceJob(
workspaceId: string,
lanes: readonly WorkspaceDispatchLane[]
): Promise<WorkspaceDispatchJobRecord | null> {
for (const lane of lanes) {
const queue = this.getLaneQueue(workspaceId, lane)
while (queue.length > 0) {
const jobId = queue[0]
const job = this.jobs.get(jobId)
if (job) {
return job
}
queue.shift()
}
}
return null
}
async removeWorkspaceJobFromLane(
workspaceId: string,
lane: WorkspaceDispatchLane,
jobId: string
): Promise<void> {
const queue = this.getLaneQueue(workspaceId, lane)
const index = queue.indexOf(jobId)
if (index >= 0) {
queue.splice(index, 1)
}
}
async cleanupExpiredWorkspaceLeases(workspaceId: string): Promise<void> {
const leaseMap = this.getLeaseMap(workspaceId)
const now = Date.now()
for (const [leaseId, expiresAt] of leaseMap.entries()) {
if (expiresAt <= now) {
leaseMap.delete(leaseId)
}
}
}
async countActiveWorkspaceLeases(workspaceId: string): Promise<number> {
await this.cleanupExpiredWorkspaceLeases(workspaceId)
return this.getLeaseMap(workspaceId).size
}
async hasWorkspaceLease(workspaceId: string, leaseId: string): Promise<boolean> {
await this.cleanupExpiredWorkspaceLeases(workspaceId)
return this.getLeaseMap(workspaceId).has(leaseId)
}
async createWorkspaceLease(workspaceId: string, leaseId: string, ttlMs: number): Promise<number> {
const expiresAt = Date.now() + ttlMs
this.getLeaseMap(workspaceId).set(leaseId, expiresAt)
return expiresAt
}
async refreshWorkspaceLease(
workspaceId: string,
leaseId: string,
ttlMs: number
): Promise<number> {
return this.createWorkspaceLease(workspaceId, leaseId, ttlMs)
}
async releaseWorkspaceLease(workspaceId: string, leaseId: string): Promise<void> {
this.getLeaseMap(workspaceId).delete(leaseId)
}
async removeWorkspaceIfIdle(
workspaceId: string,
lanes: readonly WorkspaceDispatchLane[]
): Promise<void> {
const hasPending = await this.workspaceHasPendingJobs(workspaceId, lanes)
if (!hasPending) {
this.workspaceOrder = this.workspaceOrder.filter((value) => value !== workspaceId)
}
}
async markDispatchJobAdmitted(
jobId: string,
workspaceId: string,
leaseId: string,
leaseExpiresAt: number
): Promise<void> {
await this.updateDispatchJobRecord(jobId, (record) => ({
...record,
status: 'admitted',
admittedAt: Date.now(),
lease: {
workspaceId,
leaseId,
},
metadata: {
...record.metadata,
dispatchLeaseExpiresAt: leaseExpiresAt,
},
}))
}
async markDispatchJobAdmitting(
jobId: string,
workspaceId: string,
leaseId: string,
leaseExpiresAt: number
): Promise<void> {
await this.updateDispatchJobRecord(jobId, (record) => ({
...record,
status: 'admitting',
lease: {
workspaceId,
leaseId,
},
metadata: {
...record.metadata,
dispatchLeaseExpiresAt: leaseExpiresAt,
},
}))
}
async markDispatchJobRunning(jobId: string): Promise<void> {
await this.updateDispatchJobRecord(jobId, (record) => ({
...record,
status: 'running',
startedAt: record.startedAt ?? Date.now(),
}))
}
async markDispatchJobCompleted(jobId: string, output: unknown): Promise<void> {
await this.updateDispatchJobRecord(jobId, (record) => ({
...record,
status: 'completed',
completedAt: Date.now(),
output,
}))
}
async markDispatchJobFailed(jobId: string, error: string): Promise<void> {
await this.updateDispatchJobRecord(jobId, (record) => ({
...record,
status: 'failed',
completedAt: Date.now(),
error,
}))
}
async clear(): Promise<void> {
this.jobs.clear()
this.workspaceOrder = []
this.laneQueues.clear()
this.leases.clear()
this.workspaceReadyAt.clear()
}
dispose(): void {
if (this.cleanupInterval) {
clearInterval(this.cleanupInterval)
this.cleanupInterval = null
}
void this.clear().catch((error) => {
logger.error('Failed to clear memory workspace dispatch storage', { error })
})
}
}

155
apps/sim/lib/core/workspace-dispatch/planner.ts
View File

@@ -1,155 +0,0 @@
import { createLogger } from '@sim/logger'
import { getWorkspaceConcurrencyLimit } from '@/lib/billing/workspace-concurrency'
import { type BullMQJobData, getBullMQQueueByName } from '@/lib/core/bullmq'
import { acquireLock, releaseLock } from '@/lib/core/config/redis'
import { generateId } from '@/lib/core/utils/uuid'
import {
claimWorkspaceJob,
markDispatchJobAdmitted,
popNextWorkspaceId,
releaseWorkspaceLease,
removeWorkspaceIfIdle,
requeueWorkspaceId,
} from '@/lib/core/workspace-dispatch/store'
import {
WORKSPACE_DISPATCH_CLAIM_RESULTS,
WORKSPACE_DISPATCH_LANES,
type WorkspaceDispatchJobRecord,
} from '@/lib/core/workspace-dispatch/types'
const logger = createLogger('WorkspaceDispatchPlanner')
const LEASE_TTL_MS = 15 * 60 * 1000
const WORKSPACE_CLAIM_LOCK_TTL_SECONDS = 10
export const DISPATCH_SCAN_RESULTS = {
NO_WORKSPACE: 'no_workspace',
NO_PROGRESS: 'no_progress',
ADMITTED: 'admitted',
} as const
export type DispatchScanResult = (typeof DISPATCH_SCAN_RESULTS)[keyof typeof DISPATCH_SCAN_RESULTS]
function attachDispatchMetadata(
bullmqPayload: unknown,
record: WorkspaceDispatchJobRecord,
leaseId: string,
leaseExpiresAt: number
): BullMQJobData<unknown> {
if (
bullmqPayload &&
typeof bullmqPayload === 'object' &&
'payload' in bullmqPayload &&
'metadata' in bullmqPayload
) {
const data = bullmqPayload as BullMQJobData<unknown>
return {
payload: data.payload,
metadata: {
...(data.metadata ?? {}),
dispatchJobId: record.id,
dispatchWorkspaceId: record.workspaceId,
dispatchLeaseId: leaseId,
dispatchLeaseExpiresAt: leaseExpiresAt,
},
}
}
return {
payload: bullmqPayload,
metadata: {
...record.metadata,
dispatchJobId: record.id,
dispatchWorkspaceId: record.workspaceId,
dispatchLeaseId: leaseId,
dispatchLeaseExpiresAt: leaseExpiresAt,
},
}
}
async function finalizeAdmittedJob(
record: WorkspaceDispatchJobRecord,
leaseId: string,
leaseExpiresAt: number
): Promise<void> {
try {
await getBullMQQueueByName(record.queueName).add(
record.bullmqJobName,
attachDispatchMetadata(record.bullmqPayload, record, leaseId, leaseExpiresAt),
{
jobId: record.id,
attempts: record.maxAttempts,
priority: record.priority,
}
)
await markDispatchJobAdmitted(record.id, record.workspaceId, leaseId, leaseExpiresAt)
} catch (error) {
await releaseWorkspaceLease(record.workspaceId, leaseId).catch(() => undefined)
throw error
}
}
export async function dispatchNextAdmissibleWorkspaceJob(): Promise<DispatchScanResult> {
const workspaceId = await popNextWorkspaceId()
if (!workspaceId) {
return DISPATCH_SCAN_RESULTS.NO_WORKSPACE
}
const lockValue = `lock_${generateId()}`
try {
const lockKey = `workspace-dispatch:claim-lock:${workspaceId}`
const acquired = await acquireLock(lockKey, lockValue, WORKSPACE_CLAIM_LOCK_TTL_SECONDS)
if (!acquired) {
await requeueWorkspaceId(workspaceId)
return DISPATCH_SCAN_RESULTS.NO_PROGRESS
}
const limit = await getWorkspaceConcurrencyLimit(workspaceId)
const leaseId = `lease_${generateId()}`
const claimResult = await claimWorkspaceJob(workspaceId, {
lanes: WORKSPACE_DISPATCH_LANES,
concurrencyLimit: limit,
leaseId,
now: Date.now(),
leaseTtlMs: LEASE_TTL_MS,
})
switch (claimResult.type) {
case WORKSPACE_DISPATCH_CLAIM_RESULTS.LIMIT_REACHED:
logger.debug('Workspace concurrency limit reached', { workspaceId, limit })
await requeueWorkspaceId(workspaceId)
return DISPATCH_SCAN_RESULTS.NO_PROGRESS
case WORKSPACE_DISPATCH_CLAIM_RESULTS.DELAYED:
logger.debug('Workspace has only delayed jobs', {
workspaceId,
nextReadyAt: claimResult.nextReadyAt,
})
return DISPATCH_SCAN_RESULTS.NO_PROGRESS
case WORKSPACE_DISPATCH_CLAIM_RESULTS.EMPTY:
await removeWorkspaceIfIdle(workspaceId, WORKSPACE_DISPATCH_LANES)
return DISPATCH_SCAN_RESULTS.NO_PROGRESS
case WORKSPACE_DISPATCH_CLAIM_RESULTS.ADMITTED:
logger.info('Admitting workspace job', {
workspaceId,
dispatchJobId: claimResult.record.id,
lane: claimResult.record.lane,
queueName: claimResult.record.queueName,
})
await finalizeAdmittedJob(
claimResult.record,
claimResult.leaseId,
claimResult.leaseExpiresAt
)
return DISPATCH_SCAN_RESULTS.ADMITTED
}
} catch (error) {
logger.error('Failed to dispatch workspace job', { workspaceId, error })
await requeueWorkspaceId(workspaceId)
return DISPATCH_SCAN_RESULTS.NO_PROGRESS
} finally {
await releaseLock(`workspace-dispatch:claim-lock:${workspaceId}`, lockValue).catch(
() => undefined
)
}
}

225
apps/sim/lib/core/workspace-dispatch/reconciler.test.ts
View File

@@ -1,225 +0,0 @@
/**
* @vitest-environment node
*/
import { beforeEach, describe, expect, it, vi } from 'vitest'
const {
mockGetBullMQQueueByName,
mockHasActiveWorkspace,
mockEnsureWorkspaceActive,
mockHasWorkspaceLease,
mockListDispatchJobsByStatuses,
mockMarkDispatchJobAdmitted,
mockMarkDispatchJobCompleted,
mockMarkDispatchJobFailed,
mockRefreshWorkspaceLease,
mockReleaseWorkspaceLease,
mockRemoveWorkspaceJobFromLane,
mockRestoreWorkspaceDispatchJob,
mockWakeWorkspaceDispatcher,
} = vi.hoisted(() => ({
mockGetBullMQQueueByName: vi.fn(),
mockHasActiveWorkspace: vi.fn(),
mockEnsureWorkspaceActive: vi.fn(),
mockHasWorkspaceLease: vi.fn(),
mockListDispatchJobsByStatuses: vi.fn(),
mockMarkDispatchJobAdmitted: vi.fn(),
mockMarkDispatchJobCompleted: vi.fn(),
mockMarkDispatchJobFailed: vi.fn(),
mockRefreshWorkspaceLease: vi.fn(),
mockReleaseWorkspaceLease: vi.fn(),
mockRemoveWorkspaceJobFromLane: vi.fn(),
mockRestoreWorkspaceDispatchJob: vi.fn(),
mockWakeWorkspaceDispatcher: vi.fn(),
}))
vi.mock('@sim/logger', () => ({
createLogger: () => ({
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
debug: vi.fn(),
}),
}))
vi.mock('@/lib/core/bullmq', () => ({
getBullMQQueueByName: mockGetBullMQQueueByName,
}))
vi.mock('@/lib/core/workspace-dispatch/store', () => ({
ensureWorkspaceActive: mockEnsureWorkspaceActive,
hasActiveWorkspace: mockHasActiveWorkspace,
hasWorkspaceLease: mockHasWorkspaceLease,
listDispatchJobsByStatuses: mockListDispatchJobsByStatuses,
markDispatchJobAdmitted: mockMarkDispatchJobAdmitted,
markDispatchJobCompleted: mockMarkDispatchJobCompleted,
markDispatchJobFailed: mockMarkDispatchJobFailed,
reconcileGlobalQueueDepth: vi.fn().mockResolvedValue(undefined),
refreshWorkspaceLease: mockRefreshWorkspaceLease,
releaseWorkspaceLease: mockReleaseWorkspaceLease,
removeWorkspaceJobFromLane: mockRemoveWorkspaceJobFromLane,
restoreWorkspaceDispatchJob: mockRestoreWorkspaceDispatchJob,
}))
vi.mock('@/lib/core/workspace-dispatch/dispatcher', () => ({
wakeWorkspaceDispatcher: mockWakeWorkspaceDispatcher,
}))
import { reconcileWorkspaceDispatchState } from '@/lib/core/workspace-dispatch/reconciler'
describe('workspace dispatch reconciler', () => {
beforeEach(() => {
vi.clearAllMocks()
mockHasActiveWorkspace.mockResolvedValue(true)
mockRemoveWorkspaceJobFromLane.mockResolvedValue(undefined)
})
it('marks dispatch job completed when BullMQ job is completed', async () => {
mockListDispatchJobsByStatuses.mockResolvedValue([
{
id: 'dispatch-1',
workspaceId: 'workspace-1',
lane: 'runtime',
queueName: 'workflow-execution',
bullmqJobName: 'workflow-execution',
bullmqPayload: {},
metadata: {},
priority: 10,
status: 'running',
createdAt: 1,
lease: {
workspaceId: 'workspace-1',
leaseId: 'lease-1',
},
},
])
mockGetBullMQQueueByName.mockReturnValue({
getJob: vi.fn().mockResolvedValue({
getState: vi.fn().mockResolvedValue('completed'),
returnvalue: { ok: true },
}),
})
await reconcileWorkspaceDispatchState()
expect(mockMarkDispatchJobCompleted).toHaveBeenCalledWith('dispatch-1', { ok: true })
expect(mockReleaseWorkspaceLease).toHaveBeenCalledWith('workspace-1', 'lease-1')
expect(mockWakeWorkspaceDispatcher).toHaveBeenCalled()
})
it('restores admitted jobs to waiting when lease and BullMQ job are gone', async () => {
mockListDispatchJobsByStatuses.mockResolvedValue([
{
id: 'dispatch-2',
workspaceId: 'workspace-2',
lane: 'runtime',
queueName: 'workflow-execution',
bullmqJobName: 'workflow-execution',
bullmqPayload: {},
metadata: {},
priority: 10,
status: 'admitted',
createdAt: 1,
admittedAt: 2,
lease: {
workspaceId: 'workspace-2',
leaseId: 'lease-2',
},
},
])
mockGetBullMQQueueByName.mockReturnValue({
getJob: vi.fn().mockResolvedValue(null),
})
mockHasWorkspaceLease.mockResolvedValue(false)
await reconcileWorkspaceDispatchState()
expect(mockRestoreWorkspaceDispatchJob).toHaveBeenCalledWith(
expect.objectContaining({
id: 'dispatch-2',
status: 'waiting',
lease: undefined,
})
)
expect(mockWakeWorkspaceDispatcher).toHaveBeenCalled()
})
it('reacquires the lease for a live admitting BullMQ job', async () => {
mockListDispatchJobsByStatuses.mockResolvedValue([
{
id: 'dispatch-3',
workspaceId: 'workspace-3',
lane: 'runtime',
queueName: 'workflow-execution',
bullmqJobName: 'workflow-execution',
bullmqPayload: {},
metadata: {
dispatchLeaseExpiresAt: 12345,
},
priority: 10,
status: 'admitting',
createdAt: 1,
lease: {
workspaceId: 'workspace-3',
leaseId: 'lease-3',
},
},
])
mockGetBullMQQueueByName.mockReturnValue({
getJob: vi.fn().mockResolvedValue({
getState: vi.fn().mockResolvedValue('active'),
}),
})
mockHasWorkspaceLease.mockResolvedValue(false)
await reconcileWorkspaceDispatchState()
expect(mockRefreshWorkspaceLease).toHaveBeenCalledWith('workspace-3', 'lease-3', 15 * 60 * 1000)
expect(mockMarkDispatchJobAdmitted).toHaveBeenCalledWith(
'dispatch-3',
'workspace-3',
'lease-3',
12345
)
expect(mockRemoveWorkspaceJobFromLane).toHaveBeenCalledWith(
'workspace-3',
'runtime',
'dispatch-3'
)
})
it('releases leaked lease and restores waiting when BullMQ job is gone but lease remains', async () => {
mockListDispatchJobsByStatuses.mockResolvedValue([
{
id: 'dispatch-4',
workspaceId: 'workspace-4',
lane: 'runtime',
queueName: 'workflow-execution',
bullmqJobName: 'workflow-execution',
bullmqPayload: {},
metadata: {},
priority: 10,
status: 'running',
createdAt: 1,
lease: {
workspaceId: 'workspace-4',
leaseId: 'lease-4',
},
},
])
mockGetBullMQQueueByName.mockReturnValue({
getJob: vi.fn().mockResolvedValue(null),
})
mockHasWorkspaceLease.mockResolvedValue(true)
await reconcileWorkspaceDispatchState()
expect(mockReleaseWorkspaceLease).toHaveBeenCalledWith('workspace-4', 'lease-4')
expect(mockRestoreWorkspaceDispatchJob).toHaveBeenCalledWith(
expect.objectContaining({
id: 'dispatch-4',
status: 'waiting',
})
)
})
})

226
apps/sim/lib/core/workspace-dispatch/reconciler.ts
View File

@@ -1,226 +0,0 @@
import { createLogger } from '@sim/logger'
import { getBullMQQueueByName } from '@/lib/core/bullmq'
import {
ensureWorkspaceActive,
hasActiveWorkspace,
hasWorkspaceLease,
listDispatchJobsByStatuses,
markDispatchJobAdmitted,
markDispatchJobCompleted,
markDispatchJobFailed,
markDispatchJobRunning,
reconcileGlobalQueueDepth,
refreshWorkspaceLease,
releaseWorkspaceLease,
removeWorkspaceJobFromLane,
restoreWorkspaceDispatchJob,
} from '@/lib/core/workspace-dispatch/store'
import type { WorkspaceDispatchJobRecord } from '@/lib/core/workspace-dispatch/types'
import { wakeWorkspaceDispatcher } from './dispatcher'
const logger = createLogger('WorkspaceDispatchReconciler')
const LEASE_TTL_MS = 15 * 60 * 1000
function resetToWaiting(record: WorkspaceDispatchJobRecord): WorkspaceDispatchJobRecord {
return {
...record,
status: 'waiting',
admittedAt: undefined,
startedAt: undefined,
completedAt: undefined,
output: undefined,
error: undefined,
lease: undefined,
}
}
async function reconcileTerminalBullMQState(record: WorkspaceDispatchJobRecord): Promise<boolean> {
const queue = getBullMQQueueByName(record.queueName)
const job = await queue.getJob(record.id)
if (!job) {
return false
}
const state = await job.getState()
if (state === 'completed') {
await markDispatchJobCompleted(record.id, job.returnvalue)
if (record.lease) {
await releaseWorkspaceLease(record.lease.workspaceId, record.lease.leaseId)
}
return true
}
if (state === 'failed' && job.attemptsMade >= (job.opts.attempts ?? 1)) {
await markDispatchJobFailed(record.id, job.failedReason || 'Job failed')
if (record.lease) {
await releaseWorkspaceLease(record.lease.workspaceId, record.lease.leaseId)
}
return true
}
return false
}
async function reconcileStrandedDispatchJob(record: WorkspaceDispatchJobRecord): Promise<boolean> {
if (!record.lease && record.status !== 'waiting') {
await restoreWorkspaceDispatchJob(resetToWaiting(record))
return true
}
if (!record.lease) {
return false
}
const hasLease = await hasWorkspaceLease(record.lease.workspaceId, record.lease.leaseId)
const queue = getBullMQQueueByName(record.queueName)
const job = await queue.getJob(record.id)
if (hasLease) {
if (!job) {
await releaseWorkspaceLease(record.lease.workspaceId, record.lease.leaseId)
await restoreWorkspaceDispatchJob(resetToWaiting(record))
return true
}
return false
}
if (job) {
if (record.status === 'admitting') {
await refreshWorkspaceLease(record.lease.workspaceId, record.lease.leaseId, LEASE_TTL_MS)
await markDispatchJobAdmitted(
record.id,
record.lease.workspaceId,
record.lease.leaseId,
(record.metadata as { dispatchLeaseExpiresAt?: number }).dispatchLeaseExpiresAt ??
Date.now()
)
await removeWorkspaceJobFromLane(record.workspaceId, record.lane, record.id).catch(
() => undefined
)
return true
}
await refreshWorkspaceLease(record.lease.workspaceId, record.lease.leaseId, LEASE_TTL_MS)
if (record.status === 'admitted') {
await markDispatchJobRunning(record.id)
return true
}
return false
}
await restoreWorkspaceDispatchJob(resetToWaiting(record))
return true
}
async function reconcileTerminalDispatchLease(
record: WorkspaceDispatchJobRecord
): Promise<boolean> {
if ((record.status !== 'completed' && record.status !== 'failed') || !record.lease) {
return false
}
const hasLease = await hasWorkspaceLease(record.lease.workspaceId, record.lease.leaseId)
if (!hasLease) {
return false
}
await releaseWorkspaceLease(record.lease.workspaceId, record.lease.leaseId)
return true
}
async function reconcileWaitingWorkspaceTracking(
waitingJobs: WorkspaceDispatchJobRecord[]
): Promise<boolean> {
let changed = false
const earliestByWorkspace = new Map<string, number>()
for (const record of waitingJobs) {
const readyAt = record.createdAt + (record.delayMs ?? 0)
const current = earliestByWorkspace.get(record.workspaceId)
if (current === undefined || readyAt < current) {
earliestByWorkspace.set(record.workspaceId, readyAt)
}
}
for (const [workspaceId, nextReadyAt] of earliestByWorkspace.entries()) {
const active = await hasActiveWorkspace(workspaceId)
if (!active) {
await ensureWorkspaceActive(workspaceId, nextReadyAt)
changed = true
}
}
return changed
}
export async function reconcileWorkspaceDispatchState(): Promise<void> {
const allJobs = await listDispatchJobsByStatuses([
'waiting',
'admitting',
'admitted',
'running',
'completed',
'failed',
])
const activeJobs: WorkspaceDispatchJobRecord[] = []
const waitingJobs: WorkspaceDispatchJobRecord[] = []
const terminalJobs: WorkspaceDispatchJobRecord[] = []
let nonTerminalCount = 0
for (const job of allJobs) {
switch (job.status) {
case 'admitting':
case 'admitted':
case 'running':
activeJobs.push(job)
nonTerminalCount++
break
case 'waiting':
waitingJobs.push(job)
nonTerminalCount++
break
case 'completed':
case 'failed':
terminalJobs.push(job)
break
}
}
let changed = false
for (const record of activeJobs) {
const terminal = await reconcileTerminalBullMQState(record)
if (terminal) {
changed = true
continue
}
const restored = await reconcileStrandedDispatchJob(record)
if (restored) {
changed = true
}
}
if (await reconcileWaitingWorkspaceTracking(waitingJobs)) {
changed = true
}
for (const record of terminalJobs) {
if (await reconcileTerminalDispatchLease(record)) {
changed = true
}
}
await reconcileGlobalQueueDepth(nonTerminalCount).catch((error) => {
logger.error('Failed to reconcile global queue depth', { error })
})
if (changed) {
logger.info('Workspace dispatch reconciliation updated state', {
activeJobsInspected: activeJobs.length,
waitingJobsInspected: waitingJobs.length,
terminalJobsInspected: terminalJobs.length,
})
await wakeWorkspaceDispatcher()
}
}

578
apps/sim/lib/core/workspace-dispatch/redis-store.ts
View File

@@ -1,578 +0,0 @@
import { createLogger } from '@sim/logger'
import type Redis from 'ioredis'
import { generateId } from '@/lib/core/utils/uuid'
import type { WorkspaceDispatchStorageAdapter } from '@/lib/core/workspace-dispatch/adapter'
import {
WORKSPACE_DISPATCH_CLAIM_RESULTS,
type WorkspaceDispatchClaimResult,
type WorkspaceDispatchEnqueueInput,
type WorkspaceDispatchJobRecord,
type WorkspaceDispatchLane,
} from '@/lib/core/workspace-dispatch/types'
const logger = createLogger('WorkspaceDispatchRedisStore')
const DISPATCH_PREFIX = 'workspace-dispatch:v1'
const JOB_TTL_SECONDS = 48 * 60 * 60
const SEQUENCE_KEY = `${DISPATCH_PREFIX}:sequence`
const ACTIVE_WORKSPACES_KEY = `${DISPATCH_PREFIX}:workspaces`
const GLOBAL_DEPTH_KEY = `${DISPATCH_PREFIX}:global-depth`
const CLAIM_JOB_SCRIPT = `
local workspaceId = ARGV[1]
local now = tonumber(ARGV[2])
local concurrencyLimit = tonumber(ARGV[3])
local leaseId = ARGV[4]
local leaseExpiresAt = tonumber(ARGV[5])
local lanes = cjson.decode(ARGV[6])
local sequenceKey = ARGV[7]
local activeWorkspacesKey = ARGV[8]
local jobPrefix = ARGV[9]
local workspacePrefix = ARGV[10]
local jobTtlSeconds = tonumber(ARGV[11])
local function laneKey(lane)
return workspacePrefix .. workspaceId .. ':lane:' .. lane
end
local function leaseKey()
return workspacePrefix .. workspaceId .. ':leases'
end
local function workspaceHasPending()
local minReadyAt = nil
local hasPending = false
for _, lane in ipairs(lanes) do
local ids = redis.call('ZRANGE', laneKey(lane), 0, 0)
if #ids > 0 then
local raw = redis.call('GET', jobPrefix .. ids[1])
if raw then
hasPending = true
local record = cjson.decode(raw)
local readyAt = (record.createdAt or 0) + (record.delayMs or 0)
if (minReadyAt == nil) or (readyAt < minReadyAt) then
minReadyAt = readyAt
end
else
redis.call('ZREM', laneKey(lane), ids[1])
end
end
end
return hasPending, minReadyAt
end
redis.call('ZREMRANGEBYSCORE', leaseKey(), 0, now)
local activeLeaseCount = redis.call('ZCARD', leaseKey())
if activeLeaseCount >= concurrencyLimit then
return cjson.encode({ type = 'limit_reached' })
end
local selectedId = nil
local selectedLane = nil
local selectedRecord = nil
local delayedNextReadyAt = nil
local maxScanPerLane = 20
for _, lane in ipairs(lanes) do
local ids = redis.call('ZRANGE', laneKey(lane), 0, maxScanPerLane - 1)
for _, candidateId in ipairs(ids) do
local raw = redis.call('GET', jobPrefix .. candidateId)
if raw then
local record = cjson.decode(raw)
local readyAt = (record.createdAt or 0) + (record.delayMs or 0)
if readyAt <= now then
selectedId = candidateId
selectedLane = lane
selectedRecord = record
break
end
if (delayedNextReadyAt == nil) or (readyAt < delayedNextReadyAt) then
delayedNextReadyAt = readyAt
end
else
redis.call('ZREM', laneKey(lane), candidateId)
end
end
if selectedRecord then
break
end
end
if selectedRecord == nil then
local hasPending, minReadyAt = workspaceHasPending()
if not hasPending then
return cjson.encode({ type = 'empty' })
end
local sequence = redis.call('INCR', sequenceKey)
local score = sequence
if minReadyAt ~= nil and minReadyAt > now then
score = minReadyAt * 1000000 + sequence
end
redis.call('ZADD', activeWorkspacesKey, score, workspaceId)
return cjson.encode({
type = 'delayed',
nextReadyAt = delayedNextReadyAt or minReadyAt or now
})
end
redis.call('ZADD', leaseKey(), leaseExpiresAt, leaseId)
selectedRecord.status = 'admitting'
selectedRecord.lease = {
workspaceId = workspaceId,
leaseId = leaseId
}
if selectedRecord.metadata == nil then
selectedRecord.metadata = {}
end
selectedRecord.metadata.dispatchLeaseExpiresAt = leaseExpiresAt
redis.call('SET', jobPrefix .. selectedId, cjson.encode(selectedRecord), 'EX', jobTtlSeconds)
redis.call('ZREM', laneKey(selectedLane), selectedId)
local hasPending, minReadyAt = workspaceHasPending()
if hasPending then
local sequence = redis.call('INCR', sequenceKey)
local score = sequence
if minReadyAt ~= nil and minReadyAt > now then
score = minReadyAt * 1000000 + sequence
end
redis.call('ZADD', activeWorkspacesKey, score, workspaceId)
end
return cjson.encode({
type = 'admitted',
record = selectedRecord,
leaseId = leaseId,
leaseExpiresAt = leaseExpiresAt
})
`
function jobKey(jobId: string): string {
return `${DISPATCH_PREFIX}:job:${jobId}`
}
function workspaceLaneKey(workspaceId: string, lane: WorkspaceDispatchLane): string {
return `${DISPATCH_PREFIX}:workspace:${workspaceId}:lane:${lane}`
}
function workspaceLeaseKey(workspaceId: string): string {
return `${DISPATCH_PREFIX}:workspace:${workspaceId}:leases`
}
function createPriorityScore(priority: number, sequence: number): number {
return priority * 1_000_000_000_000 + sequence
}
export class RedisWorkspaceDispatchStorage implements WorkspaceDispatchStorageAdapter {
constructor(private redis: Redis) {}
private async nextSequence(): Promise<number> {
return this.redis.incr(SEQUENCE_KEY)
}
async saveDispatchJob(record: WorkspaceDispatchJobRecord): Promise<void> {
await this.redis.set(jobKey(record.id), JSON.stringify(record), 'EX', JOB_TTL_SECONDS)
}
async getDispatchJobRecord(jobId: string): Promise<WorkspaceDispatchJobRecord | null> {
const raw = await this.redis.get(jobKey(jobId))
if (!raw) {
return null
}
try {
return JSON.parse(raw) as WorkspaceDispatchJobRecord
} catch (error) {
logger.warn('Corrupted dispatch job record, deleting', { jobId, error })
await this.redis.del(jobKey(jobId))
return null
}
}
async listDispatchJobsByStatuses(
statuses: readonly WorkspaceDispatchJobRecord['status'][]
): Promise<WorkspaceDispatchJobRecord[]> {
let cursor = '0'
const jobs: WorkspaceDispatchJobRecord[] = []
do {
const [nextCursor, keys] = await this.redis.scan(
cursor,
'MATCH',
`${DISPATCH_PREFIX}:job:*`,
'COUNT',
100
)
cursor = nextCursor
if (keys.length === 0) {
continue
}
const values = await this.redis.mget(...keys)
for (const value of values) {
if (!value) {
continue
}
try {
const record = JSON.parse(value) as WorkspaceDispatchJobRecord
if (statuses.includes(record.status)) {
jobs.push(record)
}
} catch {
// Best effort during reconciliation scans.
}
}
} while (cursor !== '0')
return jobs
}
private static readonly TERMINAL_STATUSES = new Set(['completed', 'failed'])
async updateDispatchJobRecord(
jobId: string,
updater: (record: WorkspaceDispatchJobRecord) => WorkspaceDispatchJobRecord
): Promise<WorkspaceDispatchJobRecord | null> {
const current = await this.getDispatchJobRecord(jobId)
if (!current) {
return null
}
const updated = updater(current)
if (
RedisWorkspaceDispatchStorage.TERMINAL_STATUSES.has(current.status) &&
!RedisWorkspaceDispatchStorage.TERMINAL_STATUSES.has(updated.status)
) {
return current
}
await this.saveDispatchJob(updated)
return updated
}
async enqueueWorkspaceDispatchJob(
input: WorkspaceDispatchEnqueueInput
): Promise<WorkspaceDispatchJobRecord> {
const id = input.id ?? `dispatch_${generateId().replace(/-/g, '').slice(0, 20)}`
const createdAt = Date.now()
const sequence = await this.nextSequence()
const record: WorkspaceDispatchJobRecord = {
id,
workspaceId: input.workspaceId,
lane: input.lane,
queueName: input.queueName,
bullmqJobName: input.bullmqJobName,
bullmqPayload: input.bullmqPayload,
metadata: input.metadata,
priority: input.priority ?? 100,
maxAttempts: input.maxAttempts,
delayMs: input.delayMs,
status: 'waiting',
createdAt,
}
const score = createPriorityScore(record.priority, sequence)
const pipeline = this.redis.pipeline()
pipeline.set(jobKey(id), JSON.stringify(record), 'EX', JOB_TTL_SECONDS)
pipeline.zadd(workspaceLaneKey(record.workspaceId, record.lane), score, id)
pipeline.zadd(ACTIVE_WORKSPACES_KEY, 'NX', sequence, record.workspaceId)
pipeline.incr(GLOBAL_DEPTH_KEY)
await pipeline.exec()
return record
}
async restoreWorkspaceDispatchJob(record: WorkspaceDispatchJobRecord): Promise<void> {
const sequence = await this.nextSequence()
const score = createPriorityScore(record.priority, sequence)
const pipeline = this.redis.pipeline()
pipeline.set(jobKey(record.id), JSON.stringify(record), 'EX', JOB_TTL_SECONDS)
pipeline.zadd(workspaceLaneKey(record.workspaceId, record.lane), score, record.id)
pipeline.zadd(ACTIVE_WORKSPACES_KEY, 'NX', sequence, record.workspaceId)
await pipeline.exec()
}
async claimWorkspaceJob(
workspaceId: string,
options: {
lanes: readonly WorkspaceDispatchLane[]
concurrencyLimit: number
leaseId: string
now: number
leaseTtlMs: number
}
): Promise<WorkspaceDispatchClaimResult> {
const raw = await this.redis.eval(
CLAIM_JOB_SCRIPT,
0,
workspaceId,
String(options.now),
String(options.concurrencyLimit),
options.leaseId,
String(options.now + options.leaseTtlMs),
JSON.stringify(options.lanes),
SEQUENCE_KEY,
ACTIVE_WORKSPACES_KEY,
`${DISPATCH_PREFIX}:job:`,
`${DISPATCH_PREFIX}:workspace:`,
String(JOB_TTL_SECONDS)
)
const parsed = JSON.parse(String(raw)) as WorkspaceDispatchClaimResult
switch (parsed.type) {
case WORKSPACE_DISPATCH_CLAIM_RESULTS.ADMITTED:
case WORKSPACE_DISPATCH_CLAIM_RESULTS.DELAYED:
case WORKSPACE_DISPATCH_CLAIM_RESULTS.LIMIT_REACHED:
case WORKSPACE_DISPATCH_CLAIM_RESULTS.EMPTY:
return parsed
default:
throw new Error(
`Unknown dispatch claim result: ${String((parsed as { type?: string }).type)}`
)
}
}
async getWorkspaceQueueDepth(
workspaceId: string,
lanes: readonly WorkspaceDispatchLane[]
): Promise<number> {
if (lanes.length === 0) return 0
const pipeline = this.redis.pipeline()
for (const lane of lanes) {
pipeline.zcard(workspaceLaneKey(workspaceId, lane))
}
const results = await pipeline.exec()
let depth = 0
for (const result of results ?? []) {
if (result && !result[0]) {
depth += (result[1] as number) ?? 0
}
}
return depth
}
async getGlobalQueueDepth(): Promise<number> {
const count = await this.redis.get(GLOBAL_DEPTH_KEY)
return count ? Math.max(0, Number.parseInt(count, 10)) : 0
}
async reconcileGlobalQueueDepth(knownCount: number): Promise<void> {
await this.redis.set(GLOBAL_DEPTH_KEY, knownCount)
}
async popNextWorkspaceId(): Promise<string | null> {
const result = await this.redis.zpopmin(ACTIVE_WORKSPACES_KEY)
if (!result || result.length === 0) {
return null
}
return result[0] ?? null
}
async getQueuedWorkspaceCount(): Promise<number> {
return this.redis.zcard(ACTIVE_WORKSPACES_KEY)
}
async hasActiveWorkspace(workspaceId: string): Promise<boolean> {
return (await this.redis.zscore(ACTIVE_WORKSPACES_KEY, workspaceId)) !== null
}
async ensureWorkspaceActive(workspaceId: string, readyAt?: number): Promise<void> {
const sequence = await this.nextSequence()
const score = readyAt && readyAt > Date.now() ? readyAt * 1_000_000 + sequence : sequence
await this.redis.zadd(ACTIVE_WORKSPACES_KEY, 'NX', score, workspaceId)
}
async requeueWorkspaceId(workspaceId: string): Promise<void> {
const sequence = await this.nextSequence()
await this.redis.zadd(ACTIVE_WORKSPACES_KEY, sequence, workspaceId)
}
async workspaceHasPendingJobs(
workspaceId: string,
lanes: readonly WorkspaceDispatchLane[]
): Promise<boolean> {
for (const lane of lanes) {
const count = await this.redis.zcard(workspaceLaneKey(workspaceId, lane))
if (count > 0) {
return true
}
}
return false
}
async getNextWorkspaceJob(
workspaceId: string,
lanes: readonly WorkspaceDispatchLane[]
): Promise<WorkspaceDispatchJobRecord | null> {
for (const lane of lanes) {
const ids = await this.redis.zrange(workspaceLaneKey(workspaceId, lane), 0, 0)
if (ids.length === 0) {
continue
}
const record = await this.getDispatchJobRecord(ids[0])
if (!record) {
await this.redis.zrem(workspaceLaneKey(workspaceId, lane), ids[0])
continue
}
return record
}
return null
}
async removeWorkspaceJobFromLane(
workspaceId: string,
lane: WorkspaceDispatchLane,
jobId: string
): Promise<void> {
await this.redis.zrem(workspaceLaneKey(workspaceId, lane), jobId)
}
async cleanupExpiredWorkspaceLeases(workspaceId: string): Promise<void> {
await this.redis.zremrangebyscore(workspaceLeaseKey(workspaceId), 0, Date.now())
}
async countActiveWorkspaceLeases(workspaceId: string): Promise<number> {
await this.cleanupExpiredWorkspaceLeases(workspaceId)
return this.redis.zcard(workspaceLeaseKey(workspaceId))
}
async hasWorkspaceLease(workspaceId: string, leaseId: string): Promise<boolean> {
await this.cleanupExpiredWorkspaceLeases(workspaceId)
return (await this.redis.zscore(workspaceLeaseKey(workspaceId), leaseId)) !== null
}
async createWorkspaceLease(workspaceId: string, leaseId: string, ttlMs: number): Promise<number> {
const expiresAt = Date.now() + ttlMs
await this.redis.zadd(workspaceLeaseKey(workspaceId), expiresAt, leaseId)
return expiresAt
}
async refreshWorkspaceLease(
workspaceId: string,
leaseId: string,
ttlMs: number
): Promise<number> {
return this.createWorkspaceLease(workspaceId, leaseId, ttlMs)
}
async releaseWorkspaceLease(workspaceId: string, leaseId: string): Promise<void> {
await this.redis.zrem(workspaceLeaseKey(workspaceId), leaseId)
}
async removeWorkspaceIfIdle(
workspaceId: string,
lanes: readonly WorkspaceDispatchLane[]
): Promise<void> {
const hasPendingJobs = await this.workspaceHasPendingJobs(workspaceId, lanes)
if (!hasPendingJobs) {
await this.redis.zrem(ACTIVE_WORKSPACES_KEY, workspaceId)
}
}
async markDispatchJobAdmitted(
jobId: string,
workspaceId: string,
leaseId: string,
leaseExpiresAt: number
): Promise<void> {
await this.updateDispatchJobRecord(jobId, (record) => ({
...record,
status: 'admitted',
admittedAt: Date.now(),
lease: {
workspaceId,
leaseId,
},
metadata: {
...record.metadata,
dispatchLeaseExpiresAt: leaseExpiresAt,
},
}))
}
async markDispatchJobAdmitting(
jobId: string,
workspaceId: string,
leaseId: string,
leaseExpiresAt: number
): Promise<void> {
await this.updateDispatchJobRecord(jobId, (record) => ({
...record,
status: 'admitting',
lease: {
workspaceId,
leaseId,
},
metadata: {
...record.metadata,
dispatchLeaseExpiresAt: leaseExpiresAt,
},
}))
}
async markDispatchJobRunning(jobId: string): Promise<void> {
await this.updateDispatchJobRecord(jobId, (record) => ({
...record,
status: 'running',
startedAt: record.startedAt ?? Date.now(),
}))
}
async markDispatchJobCompleted(jobId: string, output: unknown): Promise<void> {
await this.updateDispatchJobRecord(jobId, (record) => ({
...record,
status: 'completed',
completedAt: Date.now(),
output,
}))
await this.redis.decr(GLOBAL_DEPTH_KEY).catch(() => undefined)
}
async markDispatchJobFailed(jobId: string, error: string): Promise<void> {
await this.updateDispatchJobRecord(jobId, (record) => ({
...record,
status: 'failed',
completedAt: Date.now(),
error,
}))
await this.redis.decr(GLOBAL_DEPTH_KEY).catch(() => undefined)
}
async clear(): Promise<void> {
let cursor = '0'
const keys: string[] = []
do {
const [nextCursor, foundKeys] = await this.redis.scan(
cursor,
'MATCH',
`${DISPATCH_PREFIX}:*`,
'COUNT',
100
)
cursor = nextCursor
keys.push(...foundKeys)
} while (cursor !== '0')
if (keys.length > 0) {
await this.redis.del(...keys)
}
}
dispose(): void {
logger.info('Redis workspace dispatch storage disposed')
}
}

102
apps/sim/lib/core/workspace-dispatch/status.test.ts
View File

@@ -1,102 +0,0 @@
/**
* @vitest-environment node
*/
import { describe, expect, it } from 'vitest'
import { presentDispatchOrJobStatus } from '@/lib/core/workspace-dispatch/status'
describe('workspace dispatch status presentation', () => {
it('presents waiting dispatch jobs with queue metadata', () => {
const result = presentDispatchOrJobStatus(
{
id: 'dispatch-1',
workspaceId: 'workspace-1',
lane: 'runtime',
queueName: 'workflow-execution',
bullmqJobName: 'workflow-execution',
bullmqPayload: {},
metadata: { workflowId: 'workflow-1' },
priority: 10,
status: 'waiting',
createdAt: 1000,
},
null
)
expect(result).toEqual({
status: 'waiting',
metadata: {
createdAt: new Date(1000),
admittedAt: undefined,
startedAt: undefined,
completedAt: undefined,
queueName: 'workflow-execution',
lane: 'runtime',
workspaceId: 'workspace-1',
},
estimatedDuration: 300000,
})
})
it('presents admitting dispatch jobs distinctly', () => {
const result = presentDispatchOrJobStatus(
{
id: 'dispatch-1a',
workspaceId: 'workspace-1',
lane: 'runtime',
queueName: 'workflow-execution',
bullmqJobName: 'workflow-execution',
bullmqPayload: {},
metadata: { workflowId: 'workflow-1' },
priority: 10,
status: 'admitting',
createdAt: 1000,
},
null
)
expect(result.status).toBe('admitting')
expect(result.estimatedDuration).toBe(300000)
})
it('presents completed dispatch jobs with output and duration', () => {
const result = presentDispatchOrJobStatus(
{
id: 'dispatch-2',
workspaceId: 'workspace-1',
lane: 'interactive',
queueName: 'workflow-execution',
bullmqJobName: 'direct-workflow-execution',
bullmqPayload: {},
metadata: { workflowId: 'workflow-1' },
priority: 1,
status: 'completed',
createdAt: 1000,
admittedAt: 1500,
startedAt: 2000,
completedAt: 7000,
output: { success: true },
},
null
)
expect(result.status).toBe('completed')
expect(result.output).toEqual({ success: true })
expect(result.metadata.duration).toBe(5000)
})
it('falls back to legacy job status when no dispatch record exists', () => {
const result = presentDispatchOrJobStatus(null, {
id: 'job-1',
type: 'workflow-execution',
payload: {},
status: 'pending',
createdAt: new Date(1000),
attempts: 0,
maxAttempts: 3,
metadata: {},
})
expect(result.status).toBe('queued')
expect(result.estimatedDuration).toBe(300000)
})
})

110
apps/sim/lib/core/workspace-dispatch/status.ts
View File

@@ -1,110 +0,0 @@
import type { Job, JobStatus } from '@/lib/core/async-jobs/types'
import type { WorkspaceDispatchJobRecord } from '@/lib/core/workspace-dispatch/types'
export type DispatchPresentedStatus =
| 'waiting'
| 'admitting'
| 'admitted'
| 'running'
| 'completed'
| 'failed'
| 'queued'
| JobStatus
export interface DispatchStatusPresentation {
status: DispatchPresentedStatus
metadata: {
createdAt?: Date
admittedAt?: Date
startedAt?: Date
completedAt?: Date
queueName?: string
lane?: string
workspaceId?: string
duration?: number
}
output?: unknown
error?: string
estimatedDuration?: number
}
export function presentDispatchOrJobStatus(
dispatchJob: WorkspaceDispatchJobRecord | null,
job: Job | null
): DispatchStatusPresentation {
if (dispatchJob) {
const startedAt = dispatchJob.startedAt ? new Date(dispatchJob.startedAt) : undefined
const completedAt = dispatchJob.completedAt ? new Date(dispatchJob.completedAt) : undefined
const response: DispatchStatusPresentation = {
status: dispatchJob.status,
metadata: {
createdAt: new Date(dispatchJob.createdAt),
admittedAt: dispatchJob.admittedAt ? new Date(dispatchJob.admittedAt) : undefined,
startedAt,
completedAt,
queueName: dispatchJob.queueName,
lane: dispatchJob.lane,
workspaceId: dispatchJob.workspaceId,
},
}
if (startedAt && completedAt) {
response.metadata.duration = completedAt.getTime() - startedAt.getTime()
}
if (dispatchJob.status === 'completed') {
response.output = dispatchJob.output
}
if (dispatchJob.status === 'failed') {
response.error = dispatchJob.error
}
if (
dispatchJob.status === 'waiting' ||
dispatchJob.status === 'admitting' ||
dispatchJob.status === 'admitted' ||
dispatchJob.status === 'running'
) {
response.estimatedDuration = 300000
}
return response
}
if (!job) {
return {
status: 'queued',
metadata: {},
}
}
const mappedStatus = job.status === 'pending' ? 'queued' : job.status
const response: DispatchStatusPresentation = {
status: mappedStatus,
metadata: {
createdAt: job.createdAt,
startedAt: job.startedAt,
completedAt: job.completedAt,
},
}
if (job.startedAt && job.completedAt) {
response.metadata.duration = job.completedAt.getTime() - job.startedAt.getTime()
}
if (job.status === 'completed') {
response.output = job.output
}
if (job.status === 'failed') {
response.error = job.error
}
if (job.status === 'processing' || job.status === 'pending') {
response.estimatedDuration = 300000
}
return response
}

193
apps/sim/lib/core/workspace-dispatch/store.ts
View File

@@ -1,193 +0,0 @@
import type { WorkspaceDispatchStorageAdapter } from '@/lib/core/workspace-dispatch/adapter'
import {
setWorkspaceDispatchStorageAdapter as _setAdapter,
createWorkspaceDispatchStorageAdapter,
} from '@/lib/core/workspace-dispatch/factory'
import type {
WorkspaceDispatchClaimResult,
WorkspaceDispatchEnqueueInput,
WorkspaceDispatchJobRecord,
WorkspaceDispatchLane,
} from '@/lib/core/workspace-dispatch/types'
function getAdapter() {
return createWorkspaceDispatchStorageAdapter()
}
export function setWorkspaceDispatchStorageAdapter(adapter: WorkspaceDispatchStorageAdapter): void {
_setAdapter(adapter)
}
export async function saveDispatchJob(record: WorkspaceDispatchJobRecord): Promise<void> {
return getAdapter().saveDispatchJob(record)
}
export async function getDispatchJobRecord(
jobId: string
): Promise<WorkspaceDispatchJobRecord | null> {
return getAdapter().getDispatchJobRecord(jobId)
}
export async function listDispatchJobsByStatuses(
statuses: readonly WorkspaceDispatchJobRecord['status'][]
): Promise<WorkspaceDispatchJobRecord[]> {
return getAdapter().listDispatchJobsByStatuses(statuses)
}
export async function updateDispatchJobRecord(
jobId: string,
updater: (record: WorkspaceDispatchJobRecord) => WorkspaceDispatchJobRecord
): Promise<WorkspaceDispatchJobRecord | null> {
return getAdapter().updateDispatchJobRecord(jobId, updater)
}
export async function enqueueWorkspaceDispatchJob(
input: WorkspaceDispatchEnqueueInput
): Promise<WorkspaceDispatchJobRecord> {
return getAdapter().enqueueWorkspaceDispatchJob(input)
}
export async function restoreWorkspaceDispatchJob(
record: WorkspaceDispatchJobRecord
): Promise<void> {
return getAdapter().restoreWorkspaceDispatchJob(record)
}
export async function claimWorkspaceJob(
workspaceId: string,
options: {
lanes: readonly WorkspaceDispatchLane[]
concurrencyLimit: number
leaseId: string
now: number
leaseTtlMs: number
}
): Promise<WorkspaceDispatchClaimResult> {
return getAdapter().claimWorkspaceJob(workspaceId, options)
}
export async function getWorkspaceQueueDepth(
workspaceId: string,
lanes: readonly WorkspaceDispatchLane[]
): Promise<number> {
return getAdapter().getWorkspaceQueueDepth(workspaceId, lanes)
}
export async function getGlobalQueueDepth(): Promise<number> {
return getAdapter().getGlobalQueueDepth()
}
export async function reconcileGlobalQueueDepth(knownCount: number): Promise<void> {
return getAdapter().reconcileGlobalQueueDepth(knownCount)
}
export async function popNextWorkspaceId(): Promise<string | null> {
return getAdapter().popNextWorkspaceId()
}
export async function getQueuedWorkspaceCount(): Promise<number> {
return getAdapter().getQueuedWorkspaceCount()
}
export async function hasActiveWorkspace(workspaceId: string): Promise<boolean> {
return getAdapter().hasActiveWorkspace(workspaceId)
}
export async function ensureWorkspaceActive(workspaceId: string, readyAt?: number): Promise<void> {
return getAdapter().ensureWorkspaceActive(workspaceId, readyAt)
}
export async function requeueWorkspaceId(workspaceId: string): Promise<void> {
return getAdapter().requeueWorkspaceId(workspaceId)
}
export async function workspaceHasPendingJobs(
workspaceId: string,
lanes: readonly WorkspaceDispatchLane[]
): Promise<boolean> {
return getAdapter().workspaceHasPendingJobs(workspaceId, lanes)
}
export async function getNextWorkspaceJob(
workspaceId: string,
lanes: readonly WorkspaceDispatchLane[]
): Promise<WorkspaceDispatchJobRecord | null> {
return getAdapter().getNextWorkspaceJob(workspaceId, lanes)
}
export async function removeWorkspaceJobFromLane(
workspaceId: string,
lane: WorkspaceDispatchLane,
jobId: string
): Promise<void> {
return getAdapter().removeWorkspaceJobFromLane(workspaceId, lane, jobId)
}
export async function cleanupExpiredWorkspaceLeases(workspaceId: string): Promise<void> {
return getAdapter().cleanupExpiredWorkspaceLeases(workspaceId)
}
export async function countActiveWorkspaceLeases(workspaceId: string): Promise<number> {
return getAdapter().countActiveWorkspaceLeases(workspaceId)
}
export async function hasWorkspaceLease(workspaceId: string, leaseId: string): Promise<boolean> {
return getAdapter().hasWorkspaceLease(workspaceId, leaseId)
}
export async function createWorkspaceLease(
workspaceId: string,
leaseId: string,
ttlMs: number
): Promise<number> {
return getAdapter().createWorkspaceLease(workspaceId, leaseId, ttlMs)
}
export async function refreshWorkspaceLease(
workspaceId: string,
leaseId: string,
ttlMs: number
): Promise<number> {
return getAdapter().refreshWorkspaceLease(workspaceId, leaseId, ttlMs)
}
export async function releaseWorkspaceLease(workspaceId: string, leaseId: string): Promise<void> {
return getAdapter().releaseWorkspaceLease(workspaceId, leaseId)
}
export async function removeWorkspaceIfIdle(
workspaceId: string,
lanes: readonly WorkspaceDispatchLane[]
): Promise<void> {
return getAdapter().removeWorkspaceIfIdle(workspaceId, lanes)
}
export async function markDispatchJobAdmitted(
jobId: string,
workspaceId: string,
leaseId: string,
leaseExpiresAt: number
): Promise<void> {
return getAdapter().markDispatchJobAdmitted(jobId, workspaceId, leaseId, leaseExpiresAt)
}
export async function markDispatchJobAdmitting(
jobId: string,
workspaceId: string,
leaseId: string,
leaseExpiresAt: number
): Promise<void> {
return getAdapter().markDispatchJobAdmitting(jobId, workspaceId, leaseId, leaseExpiresAt)
}
export async function markDispatchJobRunning(jobId: string): Promise<void> {
return getAdapter().markDispatchJobRunning(jobId)
}
export async function markDispatchJobCompleted(jobId: string, output: unknown): Promise<void> {
return getAdapter().markDispatchJobCompleted(jobId, output)
}
export async function markDispatchJobFailed(jobId: string, error: string): Promise<void> {
return getAdapter().markDispatchJobFailed(jobId, error)
}

107
apps/sim/lib/core/workspace-dispatch/types.ts
View File

@@ -1,107 +0,0 @@
import type { JobMetadata, JobType } from '@/lib/core/async-jobs/types'
import type {
KNOWLEDGE_CONNECTOR_SYNC_QUEUE,
KNOWLEDGE_DOCUMENT_PROCESSING_QUEUE,
MOTHERSHIP_JOB_EXECUTION_QUEUE,
WORKSPACE_NOTIFICATION_DELIVERY_QUEUE,
} from '@/lib/core/bullmq/queues'
export const WORKSPACE_DISPATCH_LANES = [
'interactive',
'runtime',
'knowledge',
'lightweight',
] as const
export type WorkspaceDispatchLane = (typeof WORKSPACE_DISPATCH_LANES)[number]
export type WorkspaceDispatchQueueName =
| JobType
| typeof KNOWLEDGE_CONNECTOR_SYNC_QUEUE
| typeof KNOWLEDGE_DOCUMENT_PROCESSING_QUEUE
| typeof MOTHERSHIP_JOB_EXECUTION_QUEUE
| typeof WORKSPACE_NOTIFICATION_DELIVERY_QUEUE
export const WORKSPACE_DISPATCH_STATUSES = {
WAITING: 'waiting',
ADMITTING: 'admitting',
ADMITTED: 'admitted',
RUNNING: 'running',
COMPLETED: 'completed',
FAILED: 'failed',
} as const
export type WorkspaceDispatchStatus =
(typeof WORKSPACE_DISPATCH_STATUSES)[keyof typeof WORKSPACE_DISPATCH_STATUSES]
export interface WorkspaceDispatchLeaseInfo {
workspaceId: string
leaseId: string
}
export interface WorkspaceDispatchJobContext {
dispatchJobId: string
workspaceId: string
lane: WorkspaceDispatchLane
queueName: WorkspaceDispatchQueueName
bullmqJobName: string
priority: number
}
export interface WorkspaceDispatchJobRecord {
id: string
workspaceId: string
lane: WorkspaceDispatchLane
queueName: WorkspaceDispatchQueueName
bullmqJobName: string
bullmqPayload: unknown
metadata: JobMetadata
priority: number
maxAttempts?: number
delayMs?: number
status: WorkspaceDispatchStatus
createdAt: number
admittedAt?: number
startedAt?: number
completedAt?: number
output?: unknown
error?: string
lease?: WorkspaceDispatchLeaseInfo
}
export interface WorkspaceDispatchEnqueueInput {
id?: string
workspaceId: string
lane: WorkspaceDispatchLane
queueName: WorkspaceDispatchQueueName
bullmqJobName: string
bullmqPayload: unknown
metadata: JobMetadata
priority?: number
maxAttempts?: number
delayMs?: number
}
export const WORKSPACE_DISPATCH_CLAIM_RESULTS = {
ADMITTED: 'admitted',
LIMIT_REACHED: 'limit_reached',
DELAYED: 'delayed',
EMPTY: 'empty',
} as const
export type WorkspaceDispatchClaimResult =
| {
type: typeof WORKSPACE_DISPATCH_CLAIM_RESULTS.ADMITTED
record: WorkspaceDispatchJobRecord
leaseId: string
leaseExpiresAt: number
}
| {
type:
| typeof WORKSPACE_DISPATCH_CLAIM_RESULTS.LIMIT_REACHED
| typeof WORKSPACE_DISPATCH_CLAIM_RESULTS.EMPTY
}
| {
type: typeof WORKSPACE_DISPATCH_CLAIM_RESULTS.DELAYED
nextReadyAt: number
}

98
apps/sim/lib/core/workspace-dispatch/worker.test.ts
View File

@@ -1,98 +0,0 @@
/**
* @vitest-environment node
*/
import { beforeEach, describe, expect, it, vi } from 'vitest'
const {
mockMarkDispatchJobCompleted,
mockMarkDispatchJobFailed,
mockMarkDispatchJobRunning,
mockReleaseWorkspaceLease,
mockWakeWorkspaceDispatcher,
} = vi.hoisted(() => ({
mockMarkDispatchJobCompleted: vi.fn(),
mockMarkDispatchJobFailed: vi.fn(),
mockMarkDispatchJobRunning: vi.fn(),
mockReleaseWorkspaceLease: vi.fn(),
mockWakeWorkspaceDispatcher: vi.fn(),
}))
vi.mock('@sim/logger', () => ({
createLogger: () => ({
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
debug: vi.fn(),
}),
}))
vi.mock('@/lib/core/workspace-dispatch', () => ({
markDispatchJobCompleted: mockMarkDispatchJobCompleted,
markDispatchJobFailed: mockMarkDispatchJobFailed,
markDispatchJobRunning: mockMarkDispatchJobRunning,
releaseWorkspaceLease: mockReleaseWorkspaceLease,
wakeWorkspaceDispatcher: mockWakeWorkspaceDispatcher,
}))
import { getDispatchRuntimeMetadata, runDispatchedJob } from '@/lib/core/workspace-dispatch/worker'
describe('workspace dispatch worker lifecycle', () => {
beforeEach(() => {
vi.clearAllMocks()
})
it('returns null for missing metadata', () => {
expect(getDispatchRuntimeMetadata(undefined)).toBeNull()
})
it('extracts dispatch runtime metadata when all fields are present', () => {
expect(
getDispatchRuntimeMetadata({
dispatchJobId: 'dispatch-1',
dispatchWorkspaceId: 'workspace-1',
dispatchLeaseId: 'lease-1',
})
).toEqual({
dispatchJobId: 'dispatch-1',
dispatchWorkspaceId: 'workspace-1',
dispatchLeaseId: 'lease-1',
})
})
it('marks running, completed, releases lease, and wakes dispatcher on success', async () => {
const result = await runDispatchedJob(
{
dispatchJobId: 'dispatch-1',
dispatchWorkspaceId: 'workspace-1',
dispatchLeaseId: 'lease-1',
},
async () => ({ success: true })
)
expect(result).toEqual({ success: true })
expect(mockMarkDispatchJobRunning).toHaveBeenCalledWith('dispatch-1')
expect(mockMarkDispatchJobCompleted).toHaveBeenCalledWith('dispatch-1', { success: true })
expect(mockReleaseWorkspaceLease).toHaveBeenCalledWith('workspace-1', 'lease-1')
expect(mockWakeWorkspaceDispatcher).toHaveBeenCalled()
})
it('marks failed and still releases lease on error', async () => {
await expect(
runDispatchedJob(
{
dispatchJobId: 'dispatch-2',
dispatchWorkspaceId: 'workspace-2',
dispatchLeaseId: 'lease-2',
},
async () => {
throw new Error('boom')
}
)
).rejects.toThrow('boom')
expect(mockMarkDispatchJobRunning).toHaveBeenCalledWith('dispatch-2')
expect(mockMarkDispatchJobFailed).toHaveBeenCalledWith('dispatch-2', 'boom')
expect(mockReleaseWorkspaceLease).toHaveBeenCalledWith('workspace-2', 'lease-2')
expect(mockWakeWorkspaceDispatcher).toHaveBeenCalled()
})
})

104
apps/sim/lib/core/workspace-dispatch/worker.ts
View File

@@ -1,104 +0,0 @@
import { createLogger } from '@sim/logger'
import {
markDispatchJobCompleted,
markDispatchJobFailed,
markDispatchJobRunning,
refreshWorkspaceLease,
releaseWorkspaceLease,
wakeWorkspaceDispatcher,
} from '@/lib/core/workspace-dispatch'
const logger = createLogger('WorkspaceDispatchWorker')
interface DispatchRuntimeMetadata {
dispatchJobId: string
dispatchWorkspaceId: string
dispatchLeaseId: string
}
interface RunDispatchedJobOptions {
isFinalAttempt?: boolean
leaseTtlMs?: number
}
const DEFAULT_LEASE_TTL_MS = 15 * 60 * 1000
const LEASE_HEARTBEAT_INTERVAL_MS = 60_000
export function getDispatchRuntimeMetadata(metadata: unknown): DispatchRuntimeMetadata | null {
if (!metadata || typeof metadata !== 'object') {
return null
}
const value = metadata as Partial<DispatchRuntimeMetadata>
if (!value.dispatchJobId || !value.dispatchWorkspaceId || !value.dispatchLeaseId) {
return null
}
return {
dispatchJobId: value.dispatchJobId,
dispatchWorkspaceId: value.dispatchWorkspaceId,
dispatchLeaseId: value.dispatchLeaseId,
}
}
export async function runDispatchedJob<T>(
metadata: unknown,
run: () => Promise<T>,
options: RunDispatchedJobOptions = {}
): Promise<T> {
const dispatchMetadata = getDispatchRuntimeMetadata(metadata)
if (!dispatchMetadata) {
return run()
}
const leaseTtlMs = options.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS
const isFinalAttempt = options.isFinalAttempt ?? true
await markDispatchJobRunning(dispatchMetadata.dispatchJobId)
let heartbeatTimer: NodeJS.Timeout | null = setInterval(() => {
void refreshWorkspaceLease(
dispatchMetadata.dispatchWorkspaceId,
dispatchMetadata.dispatchLeaseId,
leaseTtlMs
).catch((error) => {
logger.error('Failed to refresh dispatch lease', { error, dispatchMetadata })
})
}, LEASE_HEARTBEAT_INTERVAL_MS)
heartbeatTimer.unref()
let succeeded = false
try {
const result = await run()
succeeded = true
await markDispatchJobCompleted(dispatchMetadata.dispatchJobId, result)
return result
} catch (error) {
if (isFinalAttempt && !succeeded) {
await markDispatchJobFailed(
dispatchMetadata.dispatchJobId,
error instanceof Error ? error.message : String(error)
)
}
throw error
} finally {
if (heartbeatTimer) {
clearInterval(heartbeatTimer)
heartbeatTimer = null
}
const shouldReleaseLease = succeeded || isFinalAttempt
if (shouldReleaseLease) {
try {
await releaseWorkspaceLease(
dispatchMetadata.dispatchWorkspaceId,
dispatchMetadata.dispatchLeaseId
)
await wakeWorkspaceDispatcher()
} catch (error) {
logger.error('Failed to release dispatch lease', { error, dispatchMetadata })
}
}
}
}

34
apps/sim/lib/knowledge/connectors/sync-engine.ts
View File

@@ -9,10 +9,8 @@ import {
import { createLogger } from '@sim/logger'
import { and, eq, gt, inArray, isNull, lt, ne, or, sql } from 'drizzle-orm'
import { decryptApiKey } from '@/lib/api-key/crypto'
import { createBullMQJobData, isBullMQEnabled } from '@/lib/core/bullmq'
import { getInternalApiBaseUrl } from '@/lib/core/utils/urls'
import { generateId } from '@/lib/core/utils/uuid'
import { enqueueWorkspaceDispatch } from '@/lib/core/workspace-dispatch'
import type { DocumentData } from '@/lib/knowledge/documents/service'
import {
hardDeleteDocuments,
@@ -180,38 +178,6 @@ export async function dispatchSync(
{ tags }
)
logger.info(`Dispatched connector sync to Trigger.dev`, { connectorId, requestId })
} else if (isBullMQEnabled()) {
const connectorRows = await db
.select({
workspaceId: knowledgeBase.workspaceId,
userId: knowledgeBase.userId,
})
.from(knowledgeConnector)
.innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId))
.where(eq(knowledgeConnector.id, connectorId))
.limit(1)
const workspaceId = connectorRows[0]?.workspaceId
const userId = connectorRows[0]?.userId
if (!workspaceId || !userId) {
throw new Error(`No workspace found for connector ${connectorId}`)
}
await enqueueWorkspaceDispatch({
workspaceId,
lane: 'knowledge',
queueName: 'knowledge-connector-sync',
bullmqJobName: 'knowledge-connector-sync',
bullmqPayload: createBullMQJobData({
connectorId,
fullSync: options?.fullSync,
requestId,
}),
metadata: {
userId,
},
})
logger.info(`Dispatched connector sync to BullMQ`, { connectorId, requestId })
} else {
executeSync(connectorId, { fullSync: options?.fullSync }).catch((error) => {
logger.error(`Sync failed for connector ${connectorId}`, {

42
apps/sim/lib/knowledge/documents/service.ts
View File

@@ -27,11 +27,9 @@ import {
} from 'drizzle-orm'
import { recordUsage } from '@/lib/billing/core/usage-log'
import { checkAndBillOverageThreshold } from '@/lib/billing/threshold-billing'
import { createBullMQJobData, isBullMQEnabled } from '@/lib/core/bullmq'
import { env } from '@/lib/core/config/env'
import { getCostMultiplier, isTriggerDevEnabled } from '@/lib/core/config/feature-flags'
import { generateId } from '@/lib/core/utils/uuid'
import { enqueueWorkspaceDispatch } from '@/lib/core/workspace-dispatch'
import { processDocument } from '@/lib/knowledge/documents/document-processor'
import type { DocumentSortField, SortOrder } from '@/lib/knowledge/documents/types'
import { generateEmbeddings } from '@/lib/knowledge/embeddings'
@@ -85,15 +83,8 @@ const PROCESSING_CONFIG = {
delayBetweenDocuments: (env.KB_CONFIG_DELAY_BETWEEN_DOCUMENTS || 50) * 2,
}
const REDIS_PROCESSING_CONFIG = {
maxConcurrentDocuments: env.KB_CONFIG_CONCURRENCY_LIMIT || 20,
batchSize: env.KB_CONFIG_BATCH_SIZE || 20,
delayBetweenBatches: env.KB_CONFIG_DELAY_BETWEEN_BATCHES || 100,
delayBetweenDocuments: env.KB_CONFIG_DELAY_BETWEEN_DOCUMENTS || 50,
}
export function getProcessingConfig() {
return isBullMQEnabled() ? REDIS_PROCESSING_CONFIG : PROCESSING_CONFIG
return PROCESSING_CONFIG
}
export interface DocumentData {
@@ -130,35 +121,6 @@ export async function dispatchDocumentProcessingJob(payload: DocumentJobData): P
return
}
if (isBullMQEnabled()) {
const workspaceRows = await db
.select({
workspaceId: knowledgeBase.workspaceId,
userId: knowledgeBase.userId,
})
.from(knowledgeBase)
.where(and(eq(knowledgeBase.id, payload.knowledgeBaseId), isNull(knowledgeBase.deletedAt)))
.limit(1)
const workspaceId = workspaceRows[0]?.workspaceId
const userId = workspaceRows[0]?.userId
if (!workspaceId || !userId) {
throw new Error(`Knowledge base not found: ${payload.knowledgeBaseId}`)
}
await enqueueWorkspaceDispatch({
workspaceId,
lane: 'knowledge',
queueName: 'knowledge-process-document',
bullmqJobName: 'knowledge-process-document',
bullmqPayload: createBullMQJobData(payload),
metadata: {
userId,
},
})
return
}
await processDocumentAsync(
payload.knowledgeBaseId,
payload.documentId,
@@ -379,7 +341,7 @@ export async function processDocumentsWithQueue(
logger.info(
`[${requestId}] Dispatching background processing for ${jobPayloads.length} documents`,
{
backend: isTriggerAvailable() ? 'trigger-dev' : isBullMQEnabled() ? 'bullmq' : 'direct',
backend: isTriggerAvailable() ? 'trigger-dev' : 'direct',
}
)

5
apps/sim/lib/logs/events.ts
View File

@@ -12,7 +12,6 @@ import {
} from '@/lib/notifications/alert-rules'
import { getActiveWorkflowContext } from '@/lib/workflows/active-context'
import {
enqueueNotificationDeliveryDispatch,
executeNotificationDelivery,
workspaceNotificationDeliveryTask,
} from '@/background/workspace-notification-delivery'
@@ -149,10 +148,6 @@ export async function emitWorkflowExecutionCompleted(log: WorkflowExecutionLog):
logger.info(
`Enqueued ${subscription.notificationType} notification ${deliveryId} via Trigger.dev`
)
} else if (await enqueueNotificationDeliveryDispatch(payload)) {
logger.info(
`Enqueued ${subscription.notificationType} notification ${deliveryId} via BullMQ`
)
} else {
void executeNotificationDelivery(payload).catch((error) => {
logger.error(`Direct notification delivery failed for ${deliveryId}`, { error })

24
apps/sim/lib/webhooks/processor.test.ts
View File

@@ -9,14 +9,12 @@ const {
mockGenerateId,
mockPreprocessExecution,
mockEnqueue,
mockEnqueueWorkspaceDispatch,
mockGetJobQueue,
mockShouldExecuteInline,
} = vi.hoisted(() => ({
mockGenerateId: vi.fn(),
mockPreprocessExecution: vi.fn(),
mockEnqueue: vi.fn(),
mockEnqueueWorkspaceDispatch: vi.fn(),
mockGetJobQueue: vi.fn(),
mockShouldExecuteInline: vi.fn(),
}))
@@ -68,15 +66,6 @@ vi.mock('@/lib/core/async-jobs', () => ({
shouldExecuteInline: mockShouldExecuteInline,
}))
vi.mock('@/lib/core/bullmq', () => ({
isBullMQEnabled: vi.fn().mockReturnValue(true),
createBullMQJobData: vi.fn((payload: unknown, metadata?: unknown) => ({ payload, metadata })),
}))
vi.mock('@/lib/core/workspace-dispatch', () => ({
enqueueWorkspaceDispatch: mockEnqueueWorkspaceDispatch,
}))
vi.mock('@/lib/core/config/feature-flags', () => ({
isProd: false,
}))
@@ -150,7 +139,6 @@ describe('webhook processor execution identity', () => {
actorUserId: 'actor-user-1',
})
mockEnqueue.mockResolvedValue('job-1')
mockEnqueueWorkspaceDispatch.mockResolvedValue('job-1')
mockGetJobQueue.mockResolvedValue({ enqueue: mockEnqueue })
mockShouldExecuteInline.mockReturnValue(false)
mockGenerateId.mockReturnValue('generated-execution-id')
@@ -211,14 +199,16 @@ describe('webhook processor execution identity', () => {
)
expect(mockGenerateId).toHaveBeenCalledTimes(1)
expect(mockEnqueueWorkspaceDispatch).toHaveBeenCalledWith(
expect(mockEnqueue).toHaveBeenCalledWith(
'webhook-execution',
expect.objectContaining({
workflowId: 'workflow-1',
provider: 'gmail',
}),
expect.objectContaining({
id: 'generated-execution-id',
workspaceId: 'workspace-1',
lane: 'runtime',
queueName: 'webhook-execution',
metadata: expect.objectContaining({
workflowId: 'workflow-1',
workspaceId: 'workspace-1',
userId: 'actor-user-1',
correlation: preprocessingResult.correlation,
}),

146
apps/sim/lib/webhooks/processor.ts
View File

@@ -7,10 +7,8 @@ import { isOrganizationOnTeamOrEnterprisePlan } from '@/lib/billing/core/subscri
import { tryAdmit } from '@/lib/core/admission/gate'
import { getInlineJobQueue, getJobQueue, shouldExecuteInline } from '@/lib/core/async-jobs'
import type { AsyncExecutionCorrelation } from '@/lib/core/async-jobs/types'
import { createBullMQJobData, isBullMQEnabled } from '@/lib/core/bullmq'
import { isProd } from '@/lib/core/config/feature-flags'
import { generateId } from '@/lib/core/utils/uuid'
import { DispatchQueueFullError, enqueueWorkspaceDispatch } from '@/lib/core/workspace-dispatch'
import { getEffectiveDecryptedEnv } from '@/lib/environment/utils'
import { preprocessExecution } from '@/lib/execution/preprocessing'
import {
@@ -568,68 +566,32 @@ export async function queueWebhookExecution(
const isPolling = isPollingWebhookProvider(payload.provider)
if (isPolling && !shouldExecuteInline()) {
const jobId = isBullMQEnabled()
? await enqueueWorkspaceDispatch({
id: executionId,
workspaceId: foundWorkflow.workspaceId,
lane: 'runtime',
queueName: 'webhook-execution',
bullmqJobName: 'webhook-execution',
bullmqPayload: createBullMQJobData(payload, {
workflowId: foundWorkflow.id,
userId: actorUserId,
correlation,
}),
metadata: {
workflowId: foundWorkflow.id,
userId: actorUserId,
correlation,
},
})
: await (await getJobQueue()).enqueue('webhook-execution', payload, {
metadata: {
workflowId: foundWorkflow.id,
workspaceId: foundWorkflow.workspaceId,
userId: actorUserId,
correlation,
},
})
const jobId = await (await getJobQueue()).enqueue('webhook-execution', payload, {
metadata: {
workflowId: foundWorkflow.id,
workspaceId: foundWorkflow.workspaceId,
userId: actorUserId,
correlation,
},
})
logger.info(
`[${options.requestId}] Queued polling webhook execution task ${jobId} for ${foundWebhook.provider} webhook via job queue`
)
} else {
const jobQueue = await getInlineJobQueue()
const jobId = isBullMQEnabled()
? await enqueueWorkspaceDispatch({
id: executionId,
workspaceId: foundWorkflow.workspaceId,
lane: 'runtime',
queueName: 'webhook-execution',
bullmqJobName: 'webhook-execution',
bullmqPayload: createBullMQJobData(payload, {
workflowId: foundWorkflow.id,
userId: actorUserId,
correlation,
}),
metadata: {
workflowId: foundWorkflow.id,
userId: actorUserId,
correlation,
},
})
: await jobQueue.enqueue('webhook-execution', payload, {
metadata: {
workflowId: foundWorkflow.id,
workspaceId: foundWorkflow.workspaceId,
userId: actorUserId,
correlation,
},
})
const jobId = await jobQueue.enqueue('webhook-execution', payload, {
metadata: {
workflowId: foundWorkflow.id,
workspaceId: foundWorkflow.workspaceId,
userId: actorUserId,
correlation,
},
})
logger.info(
`[${options.requestId}] Queued ${foundWebhook.provider} webhook execution ${jobId} via inline backend`
)
if (!isBullMQEnabled()) {
if (shouldExecuteInline()) {
void (async () => {
try {
await jobQueue.startJob(jobId)
@@ -795,66 +757,30 @@ export async function processPolledWebhookEvent(
}
if (isPollingWebhookProvider(payload.provider) && !shouldExecuteInline()) {
const jobId = isBullMQEnabled()
? await enqueueWorkspaceDispatch({
id: executionId,
workspaceId: foundWorkflow.workspaceId,
lane: 'runtime',
queueName: 'webhook-execution',
bullmqJobName: 'webhook-execution',
bullmqPayload: createBullMQJobData(payload, {
workflowId: foundWorkflow.id,
userId: actorUserId,
correlation,
}),
metadata: {
workflowId: foundWorkflow.id,
userId: actorUserId,
correlation,
},
})
: await (await getJobQueue()).enqueue('webhook-execution', payload, {
metadata: {
workflowId: foundWorkflow.id,
workspaceId: foundWorkflow.workspaceId,
userId: actorUserId,
correlation,
},
})
const jobId = await (await getJobQueue()).enqueue('webhook-execution', payload, {
metadata: {
workflowId: foundWorkflow.id,
workspaceId: foundWorkflow.workspaceId,
userId: actorUserId,
correlation,
},
})
logger.info(
`[${requestId}] Queued polling webhook execution task ${jobId} for ${provider} webhook via job queue`
)
} else {
const jobQueue = await getInlineJobQueue()
const jobId = isBullMQEnabled()
? await enqueueWorkspaceDispatch({
id: executionId,
workspaceId: foundWorkflow.workspaceId,
lane: 'runtime',
queueName: 'webhook-execution',
bullmqJobName: 'webhook-execution',
bullmqPayload: createBullMQJobData(payload, {
workflowId: foundWorkflow.id,
userId: actorUserId,
correlation,
}),
metadata: {
workflowId: foundWorkflow.id,
userId: actorUserId,
correlation,
},
})
: await jobQueue.enqueue('webhook-execution', payload, {
metadata: {
workflowId: foundWorkflow.id,
workspaceId: foundWorkflow.workspaceId,
userId: actorUserId,
correlation,
},
})
const jobId = await jobQueue.enqueue('webhook-execution', payload, {
metadata: {
workflowId: foundWorkflow.id,
workspaceId: foundWorkflow.workspaceId,
userId: actorUserId,
correlation,
},
})
logger.info(`[${requestId}] Queued ${provider} webhook execution ${jobId} via inline backend`)
if (!isBullMQEnabled()) {
if (shouldExecuteInline()) {
void (async () => {
try {
await jobQueue.startJob(jobId)
@@ -884,10 +810,6 @@ export async function processPolledWebhookEvent(
return { success: true }
} catch (error: unknown) {
if (error instanceof DispatchQueueFullError) {
logger.warn(`[${requestId}] Dispatch queue full for polled webhook: ${error.message}`)
return { success: false, error: 'Service temporarily at capacity', statusCode: 503 }
}
logger.error(`[${requestId}] Failed to process polled webhook event:`, error)
return { success: false, error: 'Internal server error', statusCode: 500 }
} finally {

4
apps/sim/package.json
View File

@@ -11,7 +11,7 @@
"dev": "next dev --port 3000",
"dev:webpack": "next dev --webpack",
"dev:sockets": "bun run socket/index.ts",
"dev:full": "bunx concurrently -n \"App,Realtime,Worker\" -c \"cyan,magenta,yellow\" \"bun run dev\" \"bun run dev:sockets\" \"bun run worker\"",
"dev:full": "bunx concurrently -n \"App,Realtime\" -c \"cyan,magenta\" \"bun run dev\" \"bun run dev:sockets\"",
"load:workflow": "bun run load:workflow:baseline",
"load:workflow:baseline": "BASE_URL=${BASE_URL:-http://localhost:3000} WARMUP_DURATION=${WARMUP_DURATION:-10} WARMUP_RATE=${WARMUP_RATE:-2} PEAK_RATE=${PEAK_RATE:-8} HOLD_DURATION=${HOLD_DURATION:-20} bunx artillery run scripts/load/workflow-concurrency.yml",
"load:workflow:waves": "BASE_URL=${BASE_URL:-http://localhost:3000} WAVE_ONE_DURATION=${WAVE_ONE_DURATION:-10} WAVE_ONE_RATE=${WAVE_ONE_RATE:-6} QUIET_DURATION=${QUIET_DURATION:-5} WAVE_TWO_DURATION=${WAVE_TWO_DURATION:-15} WAVE_TWO_RATE=${WAVE_TWO_RATE:-8} WAVE_THREE_DURATION=${WAVE_THREE_DURATION:-20} WAVE_THREE_RATE=${WAVE_THREE_RATE:-10} bunx artillery run scripts/load/workflow-waves.yml",
@@ -20,7 +20,6 @@
"build:pptx-worker": "bun build ./lib/execution/pptx-worker.cjs --target=node --format=cjs --outfile ./dist/pptx-worker.cjs",
"build:doc-worker": "bun build ./lib/execution/doc-worker.cjs --target=node --format=cjs --outfile ./dist/doc-worker.cjs",
"start": "next start",
"worker": "NODE_ENV=production bun run worker/index.ts",
"prepare": "cd ../.. && bun husky",
"test": "vitest run",
"test:watch": "vitest",
@@ -102,7 +101,6 @@
"better-auth-harmony": "1.3.1",
"binary-extensions": "^2.0.0",
"browser-image-compression": "^2.0.2",
"bullmq": "5.71.0",
"chalk": "5.6.2",
"chart.js": "4.5.1",
"cheerio": "1.1.2",

10
apps/sim/scripts/load/README.md
View File

@@ -11,14 +11,8 @@ These local-only Artillery scenarios exercise `POST /api/workflows/[id]/execute`
The default rates are tuned for these local limits:
- `ADMISSION_GATE_MAX_INFLIGHT=500`
- `DISPATCH_MAX_QUEUE_PER_WORKSPACE=1000`
- `DISPATCH_MAX_QUEUE_GLOBAL=50000`
- `WORKSPACE_CONCURRENCY_FREE=5`
- `WORKSPACE_CONCURRENCY_PRO=50`
- `WORKSPACE_CONCURRENCY_TEAM=200`
- `WORKSPACE_CONCURRENCY_ENTERPRISE=200`
That means the defaults are intentionally aimed at forcing queueing for a Free workspace without overwhelming a single local dev server process.
The admission gate caps total in-flight requests per pod.
## Baseline Concurrency
@@ -109,5 +103,5 @@ Optional variables:
- `load:workflow` is an alias for `load:workflow:baseline`
- All scenarios send `x-execution-mode: async`
- Artillery output will show request counts and response codes, which is usually enough for quick local verification
- At these defaults, you should observe queueing behavior before you approach `ADMISSION_GATE_MAX_INFLIGHT=500` or `DISPATCH_MAX_QUEUE_PER_WORKSPACE=1000`
- At these defaults, you should see `429` responses when approaching `ADMISSION_GATE_MAX_INFLIGHT=500`
- If you still see lots of `429` or `ETIMEDOUT` responses locally, lower the rates again before increasing durations

77
apps/sim/worker/health.ts
View File

@@ -1,77 +0,0 @@
import { createServer } from 'http'
import { createLogger } from '@sim/logger'
import { getRedisClient } from '@/lib/core/config/redis'
const logger = createLogger('BullMQWorkerHealth')
export interface WorkerHealthServer {
close: () => Promise<void>
}
interface WorkerHealthCheck {
redisConnected: boolean
dispatcherLastWakeAt: number
}
let healthState: WorkerHealthCheck = {
redisConnected: false,
dispatcherLastWakeAt: 0,
}
export function updateWorkerHealthState(update: Partial<WorkerHealthCheck>): void {
healthState = { ...healthState, ...update }
}
export function startWorkerHealthServer(port: number): WorkerHealthServer {
const server = createServer((req, res) => {
if (req.method === 'GET' && req.url === '/health') {
const redis = getRedisClient()
const redisConnected = redis !== null
const dispatcherActive =
healthState.dispatcherLastWakeAt > 0 &&
Date.now() - healthState.dispatcherLastWakeAt < 30_000
const healthy = redisConnected && dispatcherActive
res.writeHead(healthy ? 200 : 503, { 'Content-Type': 'application/json' })
res.end(
JSON.stringify({
ok: healthy,
redis: redisConnected,
dispatcher: dispatcherActive,
lastWakeAgoMs: healthState.dispatcherLastWakeAt
? Date.now() - healthState.dispatcherLastWakeAt
: null,
})
)
return
}
if (req.method === 'GET' && req.url === '/health/live') {
res.writeHead(200, { 'Content-Type': 'application/json' })
res.end(JSON.stringify({ ok: true }))
return
}
res.writeHead(404, { 'Content-Type': 'application/json' })
res.end(JSON.stringify({ error: 'Not found' }))
})
server.listen(port, '0.0.0.0', () => {
logger.info(`Worker health server listening on port ${port}`)
})
return {
close: () =>
new Promise((resolve, reject) => {
server.close((error) => {
if (error) {
reject(error)
return
}
resolve()
})
}),
}
}

201
apps/sim/worker/index.ts
View File

@@ -1,201 +0,0 @@
import { createLogger } from '@sim/logger'
import { Worker } from 'bullmq'
import {
getBullMQConnectionOptions,
isBullMQEnabled,
KNOWLEDGE_CONNECTOR_SYNC_QUEUE,
KNOWLEDGE_DOCUMENT_PROCESSING_QUEUE,
MOTHERSHIP_JOB_EXECUTION_QUEUE,
WORKSPACE_NOTIFICATION_DELIVERY_QUEUE,
} from '@/lib/core/bullmq'
import { wakeWorkspaceDispatcher } from '@/lib/core/workspace-dispatch'
import { sweepPendingNotificationDeliveries } from '@/background/workspace-notification-delivery'
import { startWorkerHealthServer, updateWorkerHealthState } from '@/worker/health'
import { processKnowledgeConnectorSync } from '@/worker/processors/knowledge-connector-sync'
import { processKnowledgeDocument } from '@/worker/processors/knowledge-document-processing'
import { processMothershipJobExecution } from '@/worker/processors/mothership-job-execution'
import { processResume } from '@/worker/processors/resume'
import { processSchedule } from '@/worker/processors/schedule'
import { processWebhook } from '@/worker/processors/webhook'
import { processWorkflow } from '@/worker/processors/workflow'
import { processWorkspaceNotificationDelivery } from '@/worker/processors/workspace-notification-delivery'
const logger = createLogger('BullMQWorker')
const DEFAULT_WORKER_PORT = 3001
const DEFAULT_WORKFLOW_CONCURRENCY = 50
const DEFAULT_WEBHOOK_CONCURRENCY = 30
const DEFAULT_SCHEDULE_CONCURRENCY = 20
const DEFAULT_RESUME_CONCURRENCY = 20
const DEFAULT_MOTHERSHIP_JOB_CONCURRENCY = 10
const DEFAULT_CONNECTOR_SYNC_CONCURRENCY = 5
const DEFAULT_DOCUMENT_PROCESSING_CONCURRENCY = 20
const DEFAULT_NOTIFICATION_DELIVERY_CONCURRENCY = 10
const DISPATCHER_WAKE_INTERVAL_MS = 5_000
const NOTIFICATION_SWEEPER_INTERVAL_MS = 10_000
function parseWorkerNumber(value: string | undefined, fallback: number): number {
const parsed = Number.parseInt(value ?? '', 10)
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback
}
async function main() {
const workerPort = parseWorkerNumber(process.env.WORKER_PORT, DEFAULT_WORKER_PORT)
const healthServer = startWorkerHealthServer(workerPort)
if (!isBullMQEnabled()) {
logger.warn('BullMQ worker started without REDIS_URL; worker will remain idle')
const shutdownWithoutRedis = async () => {
await healthServer.close()
process.exit(0)
}
process.on('SIGINT', shutdownWithoutRedis)
process.on('SIGTERM', shutdownWithoutRedis)
return
}
const connection = getBullMQConnectionOptions()
const workflowWorker = new Worker('workflow-execution', processWorkflow, {
connection,
concurrency: parseWorkerNumber(
process.env.WORKER_CONCURRENCY_WORKFLOW,
DEFAULT_WORKFLOW_CONCURRENCY
),
})
const webhookWorker = new Worker('webhook-execution', processWebhook, {
connection,
concurrency: parseWorkerNumber(
process.env.WORKER_CONCURRENCY_WEBHOOK,
DEFAULT_WEBHOOK_CONCURRENCY
),
})
const scheduleWorker = new Worker('schedule-execution', processSchedule, {
connection,
concurrency: parseWorkerNumber(
process.env.WORKER_CONCURRENCY_SCHEDULE,
DEFAULT_SCHEDULE_CONCURRENCY
),
})
const resumeWorker = new Worker('resume-execution', processResume, {
connection,
concurrency: parseWorkerNumber(
process.env.WORKER_CONCURRENCY_RESUME,
DEFAULT_RESUME_CONCURRENCY
),
})
const mothershipJobWorker = new Worker(
MOTHERSHIP_JOB_EXECUTION_QUEUE,
processMothershipJobExecution,
{
connection,
concurrency: parseWorkerNumber(
process.env.WORKER_CONCURRENCY_MOTHERSHIP_JOB,
DEFAULT_MOTHERSHIP_JOB_CONCURRENCY
),
}
)
const connectorSyncWorker = new Worker(
KNOWLEDGE_CONNECTOR_SYNC_QUEUE,
processKnowledgeConnectorSync,
{
connection,
concurrency: parseWorkerNumber(
process.env.WORKER_CONCURRENCY_CONNECTOR_SYNC,
DEFAULT_CONNECTOR_SYNC_CONCURRENCY
),
}
)
const documentProcessingWorker = new Worker(
KNOWLEDGE_DOCUMENT_PROCESSING_QUEUE,
processKnowledgeDocument,
{
connection,
concurrency: parseWorkerNumber(
process.env.WORKER_CONCURRENCY_DOCUMENT_PROCESSING,
DEFAULT_DOCUMENT_PROCESSING_CONCURRENCY
),
}
)
const notificationDeliveryWorker = new Worker(
WORKSPACE_NOTIFICATION_DELIVERY_QUEUE,
processWorkspaceNotificationDelivery,
{
connection,
concurrency: parseWorkerNumber(
process.env.WORKER_CONCURRENCY_NOTIFICATION_DELIVERY,
DEFAULT_NOTIFICATION_DELIVERY_CONCURRENCY
),
}
)
const workers = [
workflowWorker,
webhookWorker,
scheduleWorker,
resumeWorker,
mothershipJobWorker,
connectorSyncWorker,
documentProcessingWorker,
notificationDeliveryWorker,
]
for (const worker of workers) {
worker.on('failed', (job, error) => {
logger.error('BullMQ job failed', {
queue: worker.name,
jobId: job?.id,
name: job?.name,
error: error.message,
})
})
}
const dispatcherWakeInterval = setInterval(() => {
void wakeWorkspaceDispatcher()
.then(() => {
updateWorkerHealthState({ dispatcherLastWakeAt: Date.now() })
})
.catch((error) => {
logger.error('Periodic workspace dispatcher wake failed', { error })
})
}, DISPATCHER_WAKE_INTERVAL_MS)
dispatcherWakeInterval.unref()
const notificationSweeperInterval = setInterval(() => {
void sweepPendingNotificationDeliveries().catch((error) => {
logger.error('Pending notification sweeper failed', { error })
})
}, NOTIFICATION_SWEEPER_INTERVAL_MS)
notificationSweeperInterval.unref()
const shutdown = async () => {
logger.info('Shutting down BullMQ worker')
clearInterval(dispatcherWakeInterval)
clearInterval(notificationSweeperInterval)
await Promise.allSettled(workers.map((worker) => worker.close()))
await healthServer.close()
process.exit(0)
}
process.on('SIGINT', shutdown)
process.on('SIGTERM', shutdown)
}
main().catch((error) => {
logger.error('Failed to start BullMQ worker', {
error: error instanceof Error ? error.message : String(error),
})
process.exit(1)
})

22
apps/sim/worker/processors/knowledge-connector-sync.ts
View File

@@ -1,22 +0,0 @@
import { createLogger } from '@sim/logger'
import type { Job } from 'bullmq'
import type { BullMQJobData } from '@/lib/core/bullmq'
import { runDispatchedJob } from '@/lib/core/workspace-dispatch'
import { executeSync } from '@/lib/knowledge/connectors/sync-engine'
import type { ConnectorSyncPayload } from '@/background/knowledge-connector-sync'
const logger = createLogger('BullMQKnowledgeConnectorSync')
export async function processKnowledgeConnectorSync(job: Job<BullMQJobData<ConnectorSyncPayload>>) {
const { connectorId, fullSync } = job.data.payload
const isFinalAttempt = job.attemptsMade + 1 >= (job.opts.attempts ?? 1)
logger.info('Processing connector sync job', {
jobId: job.id,
connectorId,
})
return runDispatchedJob(job.data.metadata, () => executeSync(connectorId, { fullSync }), {
isFinalAttempt,
})
}

26
apps/sim/worker/processors/knowledge-document-processing.ts
View File

@@ -1,26 +0,0 @@
import { createLogger } from '@sim/logger'
import type { Job } from 'bullmq'
import type { BullMQJobData } from '@/lib/core/bullmq'
import { runDispatchedJob } from '@/lib/core/workspace-dispatch'
import { type DocumentJobData, processDocumentAsync } from '@/lib/knowledge/documents/service'
const logger = createLogger('BullMQKnowledgeDocumentProcessing')
export async function processKnowledgeDocument(job: Job<BullMQJobData<DocumentJobData>>) {
const { knowledgeBaseId, documentId, docData, processingOptions } = job.data.payload
const isFinalAttempt = job.attemptsMade + 1 >= (job.opts.attempts ?? 1)
logger.info('Processing knowledge document job', {
jobId: job.id,
knowledgeBaseId,
documentId,
})
await runDispatchedJob(
job.data.metadata,
() => processDocumentAsync(knowledgeBaseId, documentId, docData, processingOptions),
{
isFinalAttempt,
}
)
}

20
apps/sim/worker/processors/mothership-job-execution.ts
View File

@@ -1,20 +0,0 @@
import { createLogger } from '@sim/logger'
import type { Job } from 'bullmq'
import type { BullMQJobData } from '@/lib/core/bullmq'
import { runDispatchedJob } from '@/lib/core/workspace-dispatch'
import { executeJobInline, type JobExecutionPayload } from '@/background/schedule-execution'
const logger = createLogger('BullMQMothershipJobExecution')
export async function processMothershipJobExecution(job: Job<BullMQJobData<JobExecutionPayload>>) {
const isFinalAttempt = job.attemptsMade + 1 >= (job.opts.attempts ?? 1)
logger.info('Processing mothership scheduled job', {
jobId: job.id,
scheduleId: job.data.payload.scheduleId,
})
await runDispatchedJob(job.data.metadata, () => executeJobInline(job.data.payload), {
isFinalAttempt,
})
}

22
apps/sim/worker/processors/resume.ts
View File

@@ -1,22 +0,0 @@
import { createLogger } from '@sim/logger'
import type { Job } from 'bullmq'
import type { BullMQJobData } from '@/lib/core/bullmq'
import { runDispatchedJob } from '@/lib/core/workspace-dispatch'
import { executeResumeJob, type ResumeExecutionPayload } from '@/background/resume-execution'
const logger = createLogger('BullMQResumeProcessor')
export async function processResume(job: Job<BullMQJobData<ResumeExecutionPayload>>) {
const { payload } = job.data
const isFinalAttempt = job.attemptsMade + 1 >= (job.opts.attempts ?? 1)
logger.info('Processing resume execution job', {
jobId: job.id,
resumeExecutionId: payload.resumeExecutionId,
workflowId: payload.workflowId,
})
return runDispatchedJob(job.data.metadata, () => executeResumeJob(payload), {
isFinalAttempt,
})
}

21
apps/sim/worker/processors/schedule.ts
View File

@@ -1,21 +0,0 @@
import { createLogger } from '@sim/logger'
import type { Job } from 'bullmq'
import type { BullMQJobData } from '@/lib/core/bullmq'
import { runDispatchedJob } from '@/lib/core/workspace-dispatch'
import { executeScheduleJob, type ScheduleExecutionPayload } from '@/background/schedule-execution'
const logger = createLogger('BullMQScheduleProcessor')
export async function processSchedule(job: Job<BullMQJobData<ScheduleExecutionPayload>>) {
const { payload } = job.data
const isFinalAttempt = job.attemptsMade + 1 >= (job.opts.attempts ?? 1)
logger.info('Processing schedule job', {
jobId: job.id,
name: job.name,
})
return runDispatchedJob(job.data.metadata, () => executeScheduleJob(payload), {
isFinalAttempt,
})
}

21
apps/sim/worker/processors/webhook.ts
View File

@@ -1,21 +0,0 @@
import { createLogger } from '@sim/logger'
import type { Job } from 'bullmq'
import type { BullMQJobData } from '@/lib/core/bullmq'
import { runDispatchedJob } from '@/lib/core/workspace-dispatch'
import { executeWebhookJob, type WebhookExecutionPayload } from '@/background/webhook-execution'
const logger = createLogger('BullMQWebhookProcessor')
export async function processWebhook(job: Job<BullMQJobData<WebhookExecutionPayload>>) {
const { payload } = job.data
const isFinalAttempt = job.attemptsMade + 1 >= (job.opts.attempts ?? 1)
logger.info('Processing webhook job', {
jobId: job.id,
name: job.name,
})
return runDispatchedJob(job.data.metadata, () => executeWebhookJob(payload), {
isFinalAttempt,
})
}

51
apps/sim/worker/processors/workflow.ts
View File

@@ -1,51 +0,0 @@
import { createLogger } from '@sim/logger'
import type { Job } from 'bullmq'
import type { BullMQJobData } from '@/lib/core/bullmq'
import { runDispatchedJob } from '@/lib/core/workspace-dispatch'
import {
DIRECT_WORKFLOW_JOB_NAME,
executeQueuedWorkflowJob,
type QueuedWorkflowExecutionPayload,
} from '@/lib/workflows/executor/queued-workflow-execution'
import { executeWorkflowJob, type WorkflowExecutionPayload } from '@/background/workflow-execution'
const logger = createLogger('BullMQWorkflowProcessor')
type WorkflowQueueJobData =
| BullMQJobData<QueuedWorkflowExecutionPayload>
| BullMQJobData<WorkflowExecutionPayload>
function isDirectWorkflowJob(
job: Job<WorkflowQueueJobData>
): job is Job<BullMQJobData<QueuedWorkflowExecutionPayload>> {
return job.name === DIRECT_WORKFLOW_JOB_NAME
}
function isBackgroundWorkflowJob(
job: Job<WorkflowQueueJobData>
): job is Job<BullMQJobData<WorkflowExecutionPayload>> {
return job.name !== DIRECT_WORKFLOW_JOB_NAME
}
export async function processWorkflow(job: Job<WorkflowQueueJobData>) {
const isFinalAttempt = job.attemptsMade + 1 >= (job.opts.attempts ?? 1)
logger.info('Processing workflow job', {
jobId: job.id,
name: job.name,
})
if (isDirectWorkflowJob(job)) {
return runDispatchedJob(job.data.metadata, () => executeQueuedWorkflowJob(job.data.payload), {
isFinalAttempt,
})
}
if (isBackgroundWorkflowJob(job)) {
return runDispatchedJob(job.data.metadata, () => executeWorkflowJob(job.data.payload), {
isFinalAttempt,
})
}
throw new Error(`Unsupported workflow job type: ${job.name}`)
}

32
apps/sim/worker/processors/workspace-notification-delivery.ts
View File

@@ -1,32 +0,0 @@
import { createLogger } from '@sim/logger'
import type { Job } from 'bullmq'
import type { BullMQJobData } from '@/lib/core/bullmq'
import { runDispatchedJob } from '@/lib/core/workspace-dispatch'
import {
executeNotificationDelivery,
type NotificationDeliveryParams,
} from '@/background/workspace-notification-delivery'
const logger = createLogger('BullMQWorkspaceNotificationDelivery')
export async function processWorkspaceNotificationDelivery(
job: Job<BullMQJobData<NotificationDeliveryParams>>
) {
const isFinalAttempt = job.attemptsMade + 1 >= (job.opts.attempts ?? 1)
logger.info('Processing workspace notification delivery job', {
jobId: job.id,
deliveryId: job.data.payload.deliveryId,
})
const result = await runDispatchedJob(
job.data.metadata,
() => executeNotificationDelivery(job.data.payload),
{
isFinalAttempt,
}
)
// Retry scheduling is persisted in the notification delivery row and
// rehydrated by the periodic sweeper, which makes retries crash-safe.
}

31
bun.lock
View File

@@ -124,7 +124,6 @@
"better-auth-harmony": "1.3.1",
"binary-extensions": "^2.0.0",
"browser-image-compression": "^2.0.2",
"bullmq": "5.71.0",
"chalk": "5.6.2",
"chart.js": "4.5.1",
"cheerio": "1.1.2",
@@ -870,18 +869,6 @@
"@mongodb-js/saslprep": ["@mongodb-js/saslprep@1.4.6", "", { "dependencies": { "sparse-bitfield": "^3.0.3" } }, "sha512-y+x3H1xBZd38n10NZF/rEBlvDOOMQ6LKUTHqr8R9VkJ+mmQOYtJFxIlkkK8fZrtOiL6VixbOBWMbZGBdal3Z1g=="],
"@msgpackr-extract/msgpackr-extract-darwin-arm64": ["@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3", "", { "os": "darwin", "cpu": "arm64" }, "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw=="],
"@msgpackr-extract/msgpackr-extract-darwin-x64": ["@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3", "", { "os": "darwin", "cpu": "x64" }, "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw=="],
"@msgpackr-extract/msgpackr-extract-linux-arm": ["@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3", "", { "os": "linux", "cpu": "arm" }, "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw=="],
"@msgpackr-extract/msgpackr-extract-linux-arm64": ["@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3", "", { "os": "linux", "cpu": "arm64" }, "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg=="],
"@msgpackr-extract/msgpackr-extract-linux-x64": ["@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3", "", { "os": "linux", "cpu": "x64" }, "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg=="],
"@msgpackr-extract/msgpackr-extract-win32-x64": ["@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3", "", { "os": "win32", "cpu": "x64" }, "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ=="],
"@napi-rs/canvas": ["@napi-rs/canvas@0.1.97", "", { "optionalDependencies": { "@napi-rs/canvas-android-arm64": "0.1.97", "@napi-rs/canvas-darwin-arm64": "0.1.97", "@napi-rs/canvas-darwin-x64": "0.1.97", "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.97", "@napi-rs/canvas-linux-arm64-gnu": "0.1.97", "@napi-rs/canvas-linux-arm64-musl": "0.1.97", "@napi-rs/canvas-linux-riscv64-gnu": "0.1.97", "@napi-rs/canvas-linux-x64-gnu": "0.1.97", "@napi-rs/canvas-linux-x64-musl": "0.1.97", "@napi-rs/canvas-win32-arm64-msvc": "0.1.97", "@napi-rs/canvas-win32-x64-msvc": "0.1.97" } }, "sha512-8cFniXvrIEnVwuNSRCW9wirRZbHvrD3JVujdS2P5n5xiJZNZMOZcfOvJ1pb66c7jXMKHHglJEDVJGbm8XWFcXQ=="],
"@napi-rs/canvas-android-arm64": ["@napi-rs/canvas-android-arm64@0.1.97", "", { "os": "android", "cpu": "arm64" }, "sha512-V1c/WVw+NzH8vk7ZK/O8/nyBSCQimU8sfMsB/9qeSvdkGKNU7+mxy/bIF0gTgeBFmHpj30S4E9WHMSrxXGQuVQ=="],
@@ -1890,8 +1877,6 @@
"buildcheck": ["buildcheck@0.0.7", "", {}, "sha512-lHblz4ahamxpTmnsk+MNTRWsjYKv965MwOrSJyeD588rR3Jcu7swE+0wN5F+PbL5cjgu/9ObkhfzEPuofEMwLA=="],
"bullmq": ["bullmq@5.71.0", "", { "dependencies": { "cron-parser": "4.9.0", "ioredis": "5.9.3", "msgpackr": "1.11.5", "node-abort-controller": "3.1.1", "semver": "7.7.4", "tslib": "2.8.1", "uuid": "11.1.0" } }, "sha512-aeNWh4drsafSKnAJeiNH/nZP/5O8ZdtdMbnOPZmpjXj7NZUP5YC901U3bIH41iZValm7d1i3c34ojv7q31m30w=="],
"bytes": ["bytes@3.1.2", "", {}, "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="],
"c12": ["c12@3.1.0", "", { "dependencies": { "chokidar": "^4.0.3", "confbox": "^0.2.2", "defu": "^6.1.4", "dotenv": "^16.6.1", "exsolve": "^1.0.7", "giget": "^2.0.0", "jiti": "^2.4.2", "ohash": "^2.0.11", "pathe": "^2.0.3", "perfect-debounce": "^1.0.0", "pkg-types": "^2.2.0", "rc9": "^2.1.2" }, "peerDependencies": { "magicast": "^0.3.5" }, "optionalPeers": ["magicast"] }, "sha512-uWoS8OU1MEIsOv8p/5a82c3H31LsWVR5qiyXVfBNOzfffjUWtPnhAb4BYI2uG2HfGmZmFjCtui5XNWaps+iFuw=="],
@@ -2036,8 +2021,6 @@
"critters": ["critters@0.0.25", "", { "dependencies": { "chalk": "^4.1.0", "css-select": "^5.1.0", "dom-serializer": "^2.0.0", "domhandler": "^5.0.2", "htmlparser2": "^8.0.2", "postcss": "^8.4.23", "postcss-media-query-parser": "^0.2.3" } }, "sha512-ROF/tjJyyRdM8/6W0VqoN5Ql05xAGnkf5b7f3sTEl1bI5jTQQf8O918RD/V9tEb9pRY/TKcvJekDbJtniHyPtQ=="],
"cron-parser": ["cron-parser@4.9.0", "", { "dependencies": { "luxon": "^3.2.1" } }, "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q=="],
"croner": ["croner@9.1.0", "", {}, "sha512-p9nwwR4qyT5W996vBZhdvBCnMhicY5ytZkR4D1Xj0wuTDEiMnjwR57Q3RXYY/s0EpX6Ay3vgIcfaR+ewGHsi+g=="],
"cronstrue": ["cronstrue@3.3.0", "", { "bin": { "cronstrue": "bin/cli.js" } }, "sha512-iwJytzJph1hosXC09zY8F5ACDJKerr0h3/2mOxg9+5uuFObYlgK0m35uUPk4GCvhHc2abK7NfnR9oMqY0qZFAg=="],
@@ -2842,8 +2825,6 @@
"lucide-react": ["lucide-react@0.511.0", "", { "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-VK5a2ydJ7xm8GvBeKLS9mu1pVK6ucef9780JVUjw6bAjJL/QXnd4Y0p7SPeOUMC27YhzNCZvm5d/QX0Tp3rc0w=="],
"luxon": ["luxon@3.7.2", "", {}, "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew=="],
"magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="],
"magicast": ["magicast@0.3.5", "", { "dependencies": { "@babel/parser": "^7.25.4", "@babel/types": "^7.25.4", "source-map-js": "^1.2.0" } }, "sha512-L0WhttDl+2BOsybvEOLK7fW3UA0OQ0IQ2d6Zl2x/a6vVRs3bAY0ECOSHHeL5jD+SbOpOCUEi0y1DgHEn9Qn1AQ=="],
@@ -3036,10 +3017,6 @@
"ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
"msgpackr": ["msgpackr@1.11.5", "", { "optionalDependencies": { "msgpackr-extract": "^3.0.2" } }, "sha512-UjkUHN0yqp9RWKy0Lplhh+wlpdt9oQBYgULZOiFhV3VclSF1JnSQWZ5r9gORQlNYaUKQoR8itv7g7z1xDDuACA=="],
"msgpackr-extract": ["msgpackr-extract@3.0.3", "", { "dependencies": { "node-gyp-build-optional-packages": "5.2.2" }, "optionalDependencies": { "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3", "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3", "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3", "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3", "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3", "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3" }, "bin": { "download-msgpackr-prebuilds": "bin/download-prebuilds.js" } }, "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA=="],
"mustache": ["mustache@4.2.0", "", { "bin": { "mustache": "bin/mustache" } }, "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ=="],
"mute-stream": ["mute-stream@0.0.8", "", {}, "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA=="],
@@ -3080,8 +3057,6 @@
"node-abi": ["node-abi@3.89.0", "", { "dependencies": { "semver": "^7.3.5" } }, "sha512-6u9UwL0HlAl21+agMN3YAMXcKByMqwGx+pq+P76vii5f7hTPtKDp08/H9py6DY+cfDw7kQNTGEj/rly3IgbNQA=="],
"node-abort-controller": ["node-abort-controller@3.1.1", "", {}, "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ=="],
"node-domexception": ["node-domexception@1.0.0", "", {}, "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="],
"node-ensure": ["node-ensure@0.0.0", "", {}, "sha512-DRI60hzo2oKN1ma0ckc6nQWlHU69RH6xN0sjQTjMpChPfTYvKZdcQFfdYK2RWbJcKyUizSIy/l8OTGxMAM1QDw=="],
@@ -3092,8 +3067,6 @@
"node-gyp-build": ["node-gyp-build@4.8.4", "", { "bin": { "node-gyp-build": "bin.js", "node-gyp-build-optional": "optional.js", "node-gyp-build-test": "build-test.js" } }, "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ=="],
"node-gyp-build-optional-packages": ["node-gyp-build-optional-packages@5.2.2", "", { "dependencies": { "detect-libc": "^2.0.1" }, "bin": { "node-gyp-build-optional-packages": "bin.js", "node-gyp-build-optional-packages-optional": "optional.js", "node-gyp-build-optional-packages-test": "build-test.js" } }, "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw=="],
"node-int64": ["node-int64@0.4.0", "", {}, "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw=="],
"node-readable-to-web-readable-stream": ["node-readable-to-web-readable-stream@0.4.2", "", {}, "sha512-/cMZNI34v//jUTrI+UIo4ieHAB5EZRY/+7OmXZgBxaWBMcW2tGdceIw06RFxWxrKZ5Jp3sI2i5TsRo+CBhtVLQ=="],
@@ -4730,8 +4703,6 @@
"body-parser/iconv-lite": ["iconv-lite@0.7.1", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-2Tth85cXwGFHfvRgZWszZSvdo+0Xsqmw8k8ZwxScfcBneNUraK+dxRxRm24nszx80Y0TVio8kKLt5sLE7ZCLlw=="],
"bullmq/ioredis": ["ioredis@5.9.3", "", { "dependencies": { "@ioredis/commands": "1.5.0", "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", "denque": "^2.1.0", "lodash.defaults": "^4.2.0", "lodash.isarguments": "^3.1.0", "redis-errors": "^1.2.0", "redis-parser": "^3.0.0", "standard-as-callback": "^2.1.0" } }, "sha512-VI5tMCdeoxZWU5vjHWsiE/Su76JGhBvWF1MJnV9ZtGltHk9BmD48oDq8Tj8haZ85aceXZMxLNDQZRVo5QKNgXA=="],
"c12/chokidar": ["chokidar@4.0.3", "", { "dependencies": { "readdirp": "^4.0.1" } }, "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA=="],
"c12/confbox": ["confbox@0.2.4", "", {}, "sha512-ysOGlgTFbN2/Y6Cg3Iye8YKulHw+R2fNXHrgSmXISQdMnomY6eNDprVdW9R5xBguEqI954+S6709UyiO7B+6OQ=="],
@@ -5358,8 +5329,6 @@
"bl/readable-stream/string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="],
"bullmq/ioredis/@ioredis/commands": ["@ioredis/commands@1.5.0", "", {}, "sha512-eUgLqrMf8nJkZxT24JvVRrQya1vZkQh8BBeYNwGDqa5I0VUi8ACx7uFvAaLxintokpTenkK6DASvo/bvNbBGow=="],
"c12/chokidar/readdirp": ["readdirp@4.1.2", "", {}, "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg=="],
"cheerio/htmlparser2/entities": ["entities@7.0.1", "", {}, "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA=="],

30
docker-compose.local.yml
View File

@@ -67,36 +67,6 @@ services:
retries: 3
start_period: 10s
sim-worker:
build:
context: .
dockerfile: docker/app.Dockerfile
command: ['bun', 'apps/sim/worker/index.ts']
restart: unless-stopped
deploy:
resources:
limits:
memory: 4G
environment:
- NODE_ENV=development
- DATABASE_URL=postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@db:5432/${POSTGRES_DB:-simstudio}
- REDIS_URL=${REDIS_URL:-}
- ENCRYPTION_KEY=${ENCRYPTION_KEY:-dev-encryption-key-at-least-32-chars}
- API_ENCRYPTION_KEY=${API_ENCRYPTION_KEY:-}
- INTERNAL_API_SECRET=${INTERNAL_API_SECRET:-dev-internal-api-secret-min-32-chars}
- WORKER_PORT=3001
depends_on:
db:
condition: service_healthy
migrations:
condition: service_completed_successfully
healthcheck:
test: ['CMD', 'curl', '-fsS', 'http://127.0.0.1:3001/health/live']
interval: 90s
timeout: 5s
retries: 3
start_period: 10s
migrations:
build:
context: .

39
docker-compose.prod.yml
View File

@@ -24,8 +24,6 @@ services:
- SOCKET_SERVER_URL=${SOCKET_SERVER_URL:-http://realtime:3002}
- NEXT_PUBLIC_SOCKET_URL=${NEXT_PUBLIC_SOCKET_URL:-http://localhost:3002}
- ADMISSION_GATE_MAX_INFLIGHT=${ADMISSION_GATE_MAX_INFLIGHT:-500}
- DISPATCH_MAX_QUEUE_PER_WORKSPACE=${DISPATCH_MAX_QUEUE_PER_WORKSPACE:-1000}
- DISPATCH_MAX_QUEUE_GLOBAL=${DISPATCH_MAX_QUEUE_GLOBAL:-50000}
depends_on:
db:
condition: service_healthy
@@ -40,43 +38,6 @@ services:
retries: 3
start_period: 10s
sim-worker:
image: ghcr.io/simstudioai/simstudio:latest
command: ['bun', 'apps/sim/worker/index.ts']
restart: unless-stopped
deploy:
resources:
limits:
memory: 4G
environment:
- NODE_ENV=production
- DATABASE_URL=postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@db:5432/${POSTGRES_DB:-simstudio}
- REDIS_URL=${REDIS_URL:-}
- ENCRYPTION_KEY=${ENCRYPTION_KEY}
- API_ENCRYPTION_KEY=${API_ENCRYPTION_KEY:-}
- INTERNAL_API_SECRET=${INTERNAL_API_SECRET}
- WORKER_PORT=${WORKER_PORT:-3001}
- WORKER_CONCURRENCY_WORKFLOW=${WORKER_CONCURRENCY_WORKFLOW:-50}
- WORKER_CONCURRENCY_WEBHOOK=${WORKER_CONCURRENCY_WEBHOOK:-30}
- WORKER_CONCURRENCY_SCHEDULE=${WORKER_CONCURRENCY_SCHEDULE:-20}
- WORKER_CONCURRENCY_MOTHERSHIP_JOB=${WORKER_CONCURRENCY_MOTHERSHIP_JOB:-10}
- WORKER_CONCURRENCY_CONNECTOR_SYNC=${WORKER_CONCURRENCY_CONNECTOR_SYNC:-5}
- WORKER_CONCURRENCY_DOCUMENT_PROCESSING=${WORKER_CONCURRENCY_DOCUMENT_PROCESSING:-20}
- WORKER_CONCURRENCY_NOTIFICATION_DELIVERY=${WORKER_CONCURRENCY_NOTIFICATION_DELIVERY:-10}
- DISPATCH_MAX_QUEUE_PER_WORKSPACE=${DISPATCH_MAX_QUEUE_PER_WORKSPACE:-1000}
- DISPATCH_MAX_QUEUE_GLOBAL=${DISPATCH_MAX_QUEUE_GLOBAL:-50000}
depends_on:
db:
condition: service_healthy
migrations:
condition: service_completed_successfully
healthcheck:
test: ['CMD', 'curl', '-fsS', 'http://127.0.0.1:${WORKER_PORT:-3001}/health/live']
interval: 90s
timeout: 5s
retries: 3
start_period: 10s
realtime:
image: ghcr.io/simstudioai/realtime:latest
restart: unless-stopped

7
docker/app.Dockerfile
View File

@@ -104,11 +104,8 @@ COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/public ./apps/sim/public
COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/.next/standalone ./
COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/.next/static ./apps/sim/.next/static
# Copy the full dependency tree and app source so the BullMQ worker can run from source.
# The standalone server continues to use server.js; the worker uses bun on worker/index.ts.
COPY --from=deps --chown=nextjs:nodejs /app/node_modules ./node_modules
COPY --from=builder --chown=nextjs:nodejs /app/apps/sim ./apps/sim
COPY --from=builder --chown=nextjs:nodejs /app/packages ./packages
# Copy blog/author content for runtime filesystem reads (not part of the JS bundle)
COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/content ./apps/sim/content
# Copy isolated-vm native module (compiled for Node.js in deps stage)
COPY --from=deps --chown=nextjs:nodejs /app/node_modules/isolated-vm ./node_modules/isolated-vm

11
helm/sim/README.md
View File

@@ -709,16 +709,9 @@ kubectl create secret generic my-postgresql-secret \
See `examples/values-existing-secret.yaml` for more details.
### Worker and Redis
### Redis
The Helm chart enables the BullMQ worker by default so the deployment topology matches Docker Compose. If `REDIS_URL` is not configured, the worker pod will still start but remain idle and do no queue processing. This is expected.
Queue-backed API, webhook, and schedule execution requires Redis. Installs without Redis continue to use the inline execution path. If you do not want the worker pod at all, set:
```yaml
worker:
enabled: false
```
Redis is optional. When configured via `REDIS_URL`, it enables features like copilot chat stream coordination and caching. Without Redis, the application uses in-memory fallbacks.
### External Secrets Parameters

16
helm/sim/templates/_helpers.tpl
View File

@@ -117,22 +117,6 @@ Ollama selector labels
app.kubernetes.io/component: ollama
{{- end }}
{{/*
Worker specific labels
*/}}
{{- define "sim.worker.labels" -}}
{{ include "sim.labels" . }}
app.kubernetes.io/component: worker
{{- end }}
{{/*
Worker selector labels
*/}}
{{- define "sim.worker.selectorLabels" -}}
{{ include "sim.selectorLabels" . }}
app.kubernetes.io/component: worker
{{- end }}
{{/*
Migrations specific labels
*/}}

101
helm/sim/templates/deployment-worker.yaml
View File

@@ -1,101 +0,0 @@
{{- if .Values.worker.enabled }}
{{- include "sim.validateSecrets" . }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "sim.fullname" . }}-worker
namespace: {{ .Release.Namespace }}
labels:
{{- include "sim.worker.labels" . | nindent 4 }}
spec:
replicas: {{ .Values.worker.replicaCount }}
selector:
matchLabels:
{{- include "sim.worker.selectorLabels" . | nindent 6 }}
template:
metadata:
annotations:
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "sim.worker.selectorLabels" . | nindent 8 }}
{{- with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
{{- with .Values.global.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "sim.serviceAccountName" . }}
{{- include "sim.podSecurityContext" .Values.worker | nindent 6 }}
{{- include "sim.nodeSelector" .Values.worker | nindent 6 }}
{{- include "sim.tolerations" .Values | nindent 6 }}
{{- include "sim.affinity" .Values | nindent 6 }}
containers:
- name: worker
image: {{ include "sim.image" (dict "context" . "image" .Values.worker.image) }}
imagePullPolicy: {{ .Values.worker.image.pullPolicy }}
command: ["bun", "apps/sim/worker/index.ts"]
ports:
- name: health
containerPort: {{ .Values.worker.healthPort }}
protocol: TCP
env:
- name: DATABASE_URL
value: {{ include "sim.databaseUrl" . | quote }}
{{- if .Values.app.env.REDIS_URL }}
- name: REDIS_URL
value: {{ .Values.app.env.REDIS_URL | quote }}
{{- end }}
- name: WORKER_PORT
value: {{ .Values.worker.healthPort | quote }}
{{- if .Values.telemetry.enabled }}
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: "http://{{ include "sim.fullname" . }}-otel-collector:4318"
- name: OTEL_SERVICE_NAME
value: sim-worker
- name: OTEL_SERVICE_VERSION
value: {{ .Chart.AppVersion | quote }}
- name: OTEL_RESOURCE_ATTRIBUTES
value: "service.name=sim-worker,service.version={{ .Chart.AppVersion }},deployment.environment={{ .Values.worker.env.NODE_ENV }}"
{{- end }}
{{- range $key, $value := .Values.worker.env }}
{{- if ne $key "WORKER_PORT" }}
- name: {{ $key }}
value: {{ $value | quote }}
{{- end }}
{{- end }}
{{- with .Values.extraEnvVars }}
{{- toYaml . | nindent 12 }}
{{- end }}
envFrom:
- secretRef:
name: {{ include "sim.appSecretName" . }}
{{- if .Values.postgresql.enabled }}
- secretRef:
name: {{ include "sim.postgresqlSecretName" . }}
{{- else if .Values.externalDatabase.enabled }}
- secretRef:
name: {{ include "sim.externalDbSecretName" . }}
{{- end }}
livenessProbe:
httpGet:
path: /health/live
port: health
initialDelaySeconds: 10
periodSeconds: 30
timeoutSeconds: 5
failureThreshold: 3
readinessProbe:
httpGet:
path: /health
port: health
initialDelaySeconds: 10
periodSeconds: 30
timeoutSeconds: 5
failureThreshold: 3
{{- include "sim.resources" .Values.worker | nindent 10 }}
{{- include "sim.securityContext" .Values.worker | nindent 10 }}
{{- end }}

60
helm/sim/values.yaml
View File

@@ -96,7 +96,7 @@ app:
# Optional: API Key Encryption (RECOMMENDED for production)
# Generate 64-character hex string using: openssl rand -hex 32 (outputs 64 hex chars = 32 bytes)
API_ENCRYPTION_KEY: "" # OPTIONAL - encrypts API keys at rest, must be exactly 64 hex characters, if not set keys stored in plain text
REDIS_URL: "" # OPTIONAL - Redis connection string for BullMQ/workers; can also come from app secret or External Secrets
REDIS_URL: "" # OPTIONAL - Redis connection string for caching/sessions; can also come from app secret or External Secrets
# Email & Communication
EMAIL_VERIFICATION_ENABLED: "false" # Enable email verification for user registration and login (defaults to false)
@@ -139,10 +139,8 @@ app:
OLLAMA_URL: "" # Ollama local LLM server URL
ELEVENLABS_API_KEY: "" # ElevenLabs API key for text-to-speech in deployed chat
# Admission & Dispatch Queue Configuration
# Admission Gate Configuration
ADMISSION_GATE_MAX_INFLIGHT: "500" # Max concurrent in-flight execution requests per pod
DISPATCH_MAX_QUEUE_PER_WORKSPACE: "1000" # Max queued dispatch jobs per workspace
DISPATCH_MAX_QUEUE_GLOBAL: "50000" # Max queued dispatch jobs globally
# Rate Limiting Configuration (per minute)
RATE_LIMIT_WINDOW_MS: "60000" # Rate limit window duration (1 minute)
@@ -386,58 +384,6 @@ realtime:
extraVolumes: []
extraVolumeMounts: []
# BullMQ worker configuration (processes background jobs when Redis is available)
# Uses the same image as the main app with a different command.
# Enabled by default so self-hosted deployments get the same topology as compose.
# Without REDIS_URL the worker starts, logs that it is idle, and does no queue processing.
worker:
# Enable/disable the worker deployment
enabled: true
# Image configuration (defaults to same image as app)
image:
repository: simstudioai/simstudio
tag: latest
pullPolicy: Always
# Number of replicas
replicaCount: 1
# Health check port (worker exposes a lightweight HTTP health server)
healthPort: 3001
# Resource limits and requests
resources:
limits:
memory: "4Gi"
cpu: "1000m"
requests:
memory: "2Gi"
cpu: "500m"
# Node selector for pod scheduling
nodeSelector: {}
# Pod security context
podSecurityContext:
fsGroup: 1001
# Container security context
securityContext:
runAsNonRoot: true
runAsUser: 1001
# Environment variables (worker-specific tuning)
env:
NODE_ENV: "production"
WORKER_CONCURRENCY_WORKFLOW: "50"
WORKER_CONCURRENCY_WEBHOOK: "30"
WORKER_CONCURRENCY_SCHEDULE: "20"
WORKER_CONCURRENCY_MOTHERSHIP_JOB: "10"
WORKER_CONCURRENCY_CONNECTOR_SYNC: "5"
WORKER_CONCURRENCY_DOCUMENT_PROCESSING: "20"
WORKER_CONCURRENCY_NOTIFICATION_DELIVERY: "10"
# Database migrations job configuration
migrations:
# Enable/disable migrations job
@@ -1322,7 +1268,7 @@ externalSecrets:
CRON_SECRET: ""
# Path to API_ENCRYPTION_KEY in external store (optional)
API_ENCRYPTION_KEY: ""
# Path to REDIS_URL in external store (optional, required for worker when not set in app.env)
# Path to REDIS_URL in external store (optional)
REDIS_URL: ""
# PostgreSQL password (for internal PostgreSQL)