From 33d1342452239818852debc6f77beeec1d121483 Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Thu, 9 Apr 2026 19:15:02 -0700 Subject: [PATCH] remove worker code --- README.md | 10 +- apps/sim/app/api/jobs/[jobId]/route.test.ts | 64 +- apps/sim/app/api/jobs/[jobId]/route.ts | 28 +- .../[executionId]/[contextId]/route.ts | 28 +- .../app/api/schedules/execute/route.test.ts | 64 +- apps/sim/app/api/schedules/execute/route.ts | 64 +- .../app/api/webhooks/trigger/[path]/route.ts | 11 - .../[id]/execute/route.async.test.ts | 34 +- .../app/api/workflows/[id]/execute/route.ts | 234 +------ .../workspace-notification-delivery.ts | 37 +- .../lib/billing/workspace-concurrency.test.ts | 146 ----- apps/sim/lib/billing/workspace-concurrency.ts | 170 ------ .../lib/core/async-jobs/backends/bullmq.ts | 111 ---- .../sim/lib/core/async-jobs/backends/index.ts | 1 - apps/sim/lib/core/async-jobs/config.ts | 32 +- apps/sim/lib/core/async-jobs/index.ts | 1 - apps/sim/lib/core/async-jobs/types.ts | 2 +- apps/sim/lib/core/bullmq/connection.ts | 29 - apps/sim/lib/core/bullmq/index.ts | 16 - apps/sim/lib/core/bullmq/queues.ts | 209 ------- apps/sim/lib/core/config/env.ts | 8 - .../lib/core/workspace-dispatch/adapter.ts | 80 --- .../workspace-dispatch/dispatcher.test.ts | 175 ------ .../lib/core/workspace-dispatch/dispatcher.ts | 156 ----- .../lib/core/workspace-dispatch/factory.ts | 42 -- apps/sim/lib/core/workspace-dispatch/index.ts | 32 - .../workspace-dispatch/memory-store.test.ts | 65 -- .../core/workspace-dispatch/memory-store.ts | 506 --------------- .../lib/core/workspace-dispatch/planner.ts | 155 ----- .../workspace-dispatch/reconciler.test.ts | 225 ------- .../lib/core/workspace-dispatch/reconciler.ts | 226 ------- .../core/workspace-dispatch/redis-store.ts | 578 ------------------ .../core/workspace-dispatch/status.test.ts | 102 ---- .../sim/lib/core/workspace-dispatch/status.ts | 110 ---- apps/sim/lib/core/workspace-dispatch/store.ts | 193 ------ apps/sim/lib/core/workspace-dispatch/types.ts | 107 ---- .../core/workspace-dispatch/worker.test.ts | 98 --- .../sim/lib/core/workspace-dispatch/worker.ts | 104 ---- .../lib/knowledge/connectors/sync-engine.ts | 34 -- apps/sim/lib/knowledge/documents/service.ts | 42 +- apps/sim/lib/logs/events.ts | 5 - apps/sim/lib/webhooks/processor.test.ts | 24 +- apps/sim/lib/webhooks/processor.ts | 146 ++--- apps/sim/package.json | 4 +- apps/sim/scripts/load/README.md | 10 +- apps/sim/worker/health.ts | 77 --- apps/sim/worker/index.ts | 201 ------ .../processors/knowledge-connector-sync.ts | 22 - .../knowledge-document-processing.ts | 26 - .../processors/mothership-job-execution.ts | 20 - apps/sim/worker/processors/resume.ts | 22 - apps/sim/worker/processors/schedule.ts | 21 - apps/sim/worker/processors/webhook.ts | 21 - apps/sim/worker/processors/workflow.ts | 51 -- .../workspace-notification-delivery.ts | 32 - bun.lock | 31 - docker-compose.local.yml | 30 - docker-compose.prod.yml | 39 -- docker/app.Dockerfile | 7 +- helm/sim/README.md | 11 +- helm/sim/templates/_helpers.tpl | 16 - helm/sim/templates/deployment-worker.yaml | 101 --- helm/sim/values.yaml | 60 +- 63 files changed, 148 insertions(+), 5158 deletions(-) delete mode 100644 apps/sim/lib/billing/workspace-concurrency.test.ts delete mode 100644 apps/sim/lib/billing/workspace-concurrency.ts delete mode 100644 apps/sim/lib/core/async-jobs/backends/bullmq.ts delete mode 100644 apps/sim/lib/core/bullmq/connection.ts delete mode 100644 apps/sim/lib/core/bullmq/index.ts delete mode 100644 apps/sim/lib/core/bullmq/queues.ts delete mode 100644 apps/sim/lib/core/workspace-dispatch/adapter.ts delete mode 100644 apps/sim/lib/core/workspace-dispatch/dispatcher.test.ts delete mode 100644 apps/sim/lib/core/workspace-dispatch/dispatcher.ts delete mode 100644 apps/sim/lib/core/workspace-dispatch/factory.ts delete mode 100644 apps/sim/lib/core/workspace-dispatch/index.ts delete mode 100644 apps/sim/lib/core/workspace-dispatch/memory-store.test.ts delete mode 100644 apps/sim/lib/core/workspace-dispatch/memory-store.ts delete mode 100644 apps/sim/lib/core/workspace-dispatch/planner.ts delete mode 100644 apps/sim/lib/core/workspace-dispatch/reconciler.test.ts delete mode 100644 apps/sim/lib/core/workspace-dispatch/reconciler.ts delete mode 100644 apps/sim/lib/core/workspace-dispatch/redis-store.ts delete mode 100644 apps/sim/lib/core/workspace-dispatch/status.test.ts delete mode 100644 apps/sim/lib/core/workspace-dispatch/status.ts delete mode 100644 apps/sim/lib/core/workspace-dispatch/store.ts delete mode 100644 apps/sim/lib/core/workspace-dispatch/types.ts delete mode 100644 apps/sim/lib/core/workspace-dispatch/worker.test.ts delete mode 100644 apps/sim/lib/core/workspace-dispatch/worker.ts delete mode 100644 apps/sim/worker/health.ts delete mode 100644 apps/sim/worker/index.ts delete mode 100644 apps/sim/worker/processors/knowledge-connector-sync.ts delete mode 100644 apps/sim/worker/processors/knowledge-document-processing.ts delete mode 100644 apps/sim/worker/processors/mothership-job-execution.ts delete mode 100644 apps/sim/worker/processors/resume.ts delete mode 100644 apps/sim/worker/processors/schedule.ts delete mode 100644 apps/sim/worker/processors/webhook.ts delete mode 100644 apps/sim/worker/processors/workflow.ts delete mode 100644 apps/sim/worker/processors/workspace-notification-delivery.ts delete mode 100644 helm/sim/templates/deployment-worker.yaml diff --git a/README.md b/README.md index 93dbc8ba00..de6befd2a8 100644 --- a/README.md +++ b/README.md @@ -74,10 +74,6 @@ docker compose -f docker-compose.prod.yml up -d Open [http://localhost:3000](http://localhost:3000) -#### Background worker note - -The Docker Compose stack starts a dedicated worker container by default. If `REDIS_URL` is not configured, the worker will start, log that it is idle, and do no queue processing. This is expected. Queue-backed API, webhook, and schedule execution requires Redis; installs without Redis continue to use the inline execution path. - Sim also supports local models via [Ollama](https://ollama.ai) and [vLLM](https://docs.vllm.ai/) — see the [Docker self-hosting docs](https://docs.sim.ai/self-hosting/docker) for setup details. ### Self-hosted: Manual Setup @@ -123,12 +119,10 @@ cd packages/db && bun run db:migrate 5. Start development servers: ```bash -bun run dev:full # Starts Next.js app, realtime socket server, and the BullMQ worker +bun run dev:full # Starts Next.js app and realtime socket server ``` -If `REDIS_URL` is not configured, the worker will remain idle and execution continues inline. - -Or run separately: `bun run dev` (Next.js), `cd apps/sim && bun run dev:sockets` (realtime), and `cd apps/sim && bun run worker` (BullMQ worker). +Or run separately: `bun run dev` (Next.js) and `cd apps/sim && bun run dev:sockets` (realtime). ## Copilot API Keys diff --git a/apps/sim/app/api/jobs/[jobId]/route.test.ts b/apps/sim/app/api/jobs/[jobId]/route.test.ts index 050c0bee2c..6ebce09fa6 100644 --- a/apps/sim/app/api/jobs/[jobId]/route.test.ts +++ b/apps/sim/app/api/jobs/[jobId]/route.test.ts @@ -6,16 +6,16 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' const { mockCheckHybridAuth, - mockGetDispatchJobRecord, mockGetJobQueue, mockVerifyWorkflowAccess, mockGetWorkflowById, + mockGetJob, } = vi.hoisted(() => ({ mockCheckHybridAuth: vi.fn(), - mockGetDispatchJobRecord: vi.fn(), mockGetJobQueue: vi.fn(), mockVerifyWorkflowAccess: vi.fn(), mockGetWorkflowById: vi.fn(), + mockGetJob: vi.fn(), })) vi.mock('@sim/logger', () => ({ @@ -32,19 +32,9 @@ vi.mock('@/lib/auth/hybrid', () => ({ })) vi.mock('@/lib/core/async-jobs', () => ({ - JOB_STATUS: { - PENDING: 'pending', - PROCESSING: 'processing', - COMPLETED: 'completed', - FAILED: 'failed', - }, getJobQueue: mockGetJobQueue, })) -vi.mock('@/lib/core/workspace-dispatch/store', () => ({ - getDispatchJobRecord: mockGetDispatchJobRecord, -})) - vi.mock('@/lib/core/utils/request', () => ({ generateRequestId: vi.fn().mockReturnValue('request-1'), })) @@ -85,71 +75,51 @@ describe('GET /api/jobs/[jobId]', () => { }) mockGetJobQueue.mockResolvedValue({ - getJob: vi.fn().mockResolvedValue(null), + getJob: mockGetJob, }) }) - it('returns dispatcher-aware waiting status with metadata', async () => { - mockGetDispatchJobRecord.mockResolvedValue({ - id: 'dispatch-1', - workspaceId: 'workspace-1', - lane: 'runtime', - queueName: 'workflow-execution', - bullmqJobName: 'workflow-execution', - bullmqPayload: {}, + it('returns job status with metadata', async () => { + mockGetJob.mockResolvedValue({ + id: 'job-1', + status: 'pending', metadata: { workflowId: 'workflow-1', }, - priority: 10, - status: 'waiting', - createdAt: 1000, - admittedAt: 2000, }) const response = await GET(createMockRequest(), { - params: Promise.resolve({ jobId: 'dispatch-1' }), + params: Promise.resolve({ jobId: 'job-1' }), }) const body = await response.json() expect(response.status).toBe(200) - expect(body.status).toBe('waiting') - expect(body.metadata.queueName).toBe('workflow-execution') - expect(body.metadata.lane).toBe('runtime') - expect(body.metadata.workspaceId).toBe('workspace-1') + expect(body.status).toBe('pending') + expect(body.metadata.workflowId).toBe('workflow-1') }) - it('returns completed output from dispatch state', async () => { - mockGetDispatchJobRecord.mockResolvedValue({ - id: 'dispatch-2', - workspaceId: 'workspace-1', - lane: 'interactive', - queueName: 'workflow-execution', - bullmqJobName: 'direct-workflow-execution', - bullmqPayload: {}, + it('returns completed output from job', async () => { + mockGetJob.mockResolvedValue({ + id: 'job-2', + status: 'completed', metadata: { workflowId: 'workflow-1', }, - priority: 1, - status: 'completed', - createdAt: 1000, - startedAt: 2000, - completedAt: 7000, output: { success: true }, }) const response = await GET(createMockRequest(), { - params: Promise.resolve({ jobId: 'dispatch-2' }), + params: Promise.resolve({ jobId: 'job-2' }), }) const body = await response.json() expect(response.status).toBe(200) expect(body.status).toBe('completed') expect(body.output).toEqual({ success: true }) - expect(body.metadata.duration).toBe(5000) }) - it('returns 404 when neither dispatch nor BullMQ job exists', async () => { - mockGetDispatchJobRecord.mockResolvedValue(null) + it('returns 404 when job does not exist', async () => { + mockGetJob.mockResolvedValue(null) const response = await GET(createMockRequest(), { params: Promise.resolve({ jobId: 'missing-job' }), diff --git a/apps/sim/app/api/jobs/[jobId]/route.ts b/apps/sim/app/api/jobs/[jobId]/route.ts index 0ce749fa82..b096d80110 100644 --- a/apps/sim/app/api/jobs/[jobId]/route.ts +++ b/apps/sim/app/api/jobs/[jobId]/route.ts @@ -3,8 +3,6 @@ import { type NextRequest, NextResponse } from 'next/server' import { checkHybridAuth } from '@/lib/auth/hybrid' import { getJobQueue } from '@/lib/core/async-jobs' import { generateRequestId } from '@/lib/core/utils/request' -import { presentDispatchOrJobStatus } from '@/lib/core/workspace-dispatch/status' -import { getDispatchJobRecord } from '@/lib/core/workspace-dispatch/store' import { createErrorResponse } from '@/app/api/workflows/utils' const logger = createLogger('TaskStatusAPI') @@ -25,15 +23,14 @@ export async function GET( const authenticatedUserId = authResult.userId - const dispatchJob = await getDispatchJobRecord(taskId) const jobQueue = await getJobQueue() - const job = dispatchJob ? null : await jobQueue.getJob(taskId) + const job = await jobQueue.getJob(taskId) - if (!job && !dispatchJob) { + if (!job) { return createErrorResponse('Task not found', 404) } - const metadataToCheck = dispatchJob?.metadata ?? job?.metadata + const metadataToCheck = job.metadata if (metadataToCheck?.workflowId) { const { verifyWorkflowAccess } = await import('@/socket/middleware/permissions') @@ -61,25 +58,22 @@ export async function GET( return createErrorResponse('Access denied', 403) } - const presented = presentDispatchOrJobStatus(dispatchJob, job) - const response: any = { + const response: Record = { success: true, taskId, - status: presented.status, - metadata: presented.metadata, + status: job.status, + metadata: job.metadata, } - if (presented.output !== undefined) response.output = presented.output - if (presented.error !== undefined) response.error = presented.error - if (presented.estimatedDuration !== undefined) { - response.estimatedDuration = presented.estimatedDuration - } + if (job.output !== undefined) response.output = job.output + if (job.error !== undefined) response.error = job.error return NextResponse.json(response) - } catch (error: any) { + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error) logger.error(`[${requestId}] Error fetching task status:`, error) - if (error.message?.includes('not found') || error.status === 404) { + if (errorMessage?.includes('not found')) { return createErrorResponse('Task not found', 404) } diff --git a/apps/sim/app/api/resume/[workflowId]/[executionId]/[contextId]/route.ts b/apps/sim/app/api/resume/[workflowId]/[executionId]/[contextId]/route.ts index f6a33f823e..80832187d8 100644 --- a/apps/sim/app/api/resume/[workflowId]/[executionId]/[contextId]/route.ts +++ b/apps/sim/app/api/resume/[workflowId]/[executionId]/[contextId]/route.ts @@ -1,13 +1,11 @@ import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' import { AuthType } from '@/lib/auth/hybrid' -import { getJobQueue, shouldUseBullMQ } from '@/lib/core/async-jobs' -import { createBullMQJobData } from '@/lib/core/bullmq' +import { getJobQueue } from '@/lib/core/async-jobs' import { generateRequestId } from '@/lib/core/utils/request' import { SSE_HEADERS } from '@/lib/core/utils/sse' import { getBaseUrl } from '@/lib/core/utils/urls' import { generateId } from '@/lib/core/utils/uuid' -import { enqueueWorkspaceDispatch } from '@/lib/core/workspace-dispatch' import { setExecutionMeta } from '@/lib/execution/event-buffer' import { preprocessExecution } from '@/lib/execution/preprocessing' import { PauseResumeManager } from '@/lib/workflows/executor/human-in-the-loop-manager' @@ -227,26 +225,10 @@ export async function POST( let jobId: string try { - const useBullMQ = shouldUseBullMQ() - if (useBullMQ) { - jobId = await enqueueWorkspaceDispatch({ - id: enqueueResult.resumeExecutionId, - workspaceId: workflow.workspaceId, - lane: 'runtime', - queueName: 'resume-execution', - bullmqJobName: 'resume-execution', - bullmqPayload: createBullMQJobData(resumePayload, { - workflowId, - userId, - }), - metadata: { workflowId, userId }, - }) - } else { - const jobQueue = await getJobQueue() - jobId = await jobQueue.enqueue('resume-execution', resumePayload, { - metadata: { workflowId, workspaceId: workflow.workspaceId, userId }, - }) - } + const jobQueue = await getJobQueue() + jobId = await jobQueue.enqueue('resume-execution', resumePayload, { + metadata: { workflowId, workspaceId: workflow.workspaceId, userId }, + }) logger.info('Enqueued async resume execution', { jobId, resumeExecutionId: enqueueResult.resumeExecutionId, diff --git a/apps/sim/app/api/schedules/execute/route.test.ts b/apps/sim/app/api/schedules/execute/route.test.ts index 12dd33097e..aa1fba6d7b 100644 --- a/apps/sim/app/api/schedules/execute/route.test.ts +++ b/apps/sim/app/api/schedules/execute/route.test.ts @@ -14,7 +14,6 @@ const { mockDbReturning, mockDbUpdate, mockEnqueue, - mockEnqueueWorkspaceDispatch, mockStartJob, mockCompleteJob, mockMarkJobFailed, @@ -24,7 +23,6 @@ const { const mockDbSet = vi.fn().mockReturnValue({ where: mockDbWhere }) const mockDbUpdate = vi.fn().mockReturnValue({ set: mockDbSet }) const mockEnqueue = vi.fn().mockResolvedValue('job-id-1') - const mockEnqueueWorkspaceDispatch = vi.fn().mockResolvedValue('job-id-1') const mockStartJob = vi.fn().mockResolvedValue(undefined) const mockCompleteJob = vi.fn().mockResolvedValue(undefined) const mockMarkJobFailed = vi.fn().mockResolvedValue(undefined) @@ -42,7 +40,6 @@ const { mockDbReturning, mockDbUpdate, mockEnqueue, - mockEnqueueWorkspaceDispatch, mockStartJob, mockCompleteJob, mockMarkJobFailed, @@ -75,15 +72,6 @@ vi.mock('@/lib/core/async-jobs', () => ({ shouldExecuteInline: vi.fn().mockReturnValue(false), })) -vi.mock('@/lib/core/bullmq', () => ({ - isBullMQEnabled: vi.fn().mockReturnValue(true), - createBullMQJobData: vi.fn((payload: unknown) => ({ payload })), -})) - -vi.mock('@/lib/core/workspace-dispatch', () => ({ - enqueueWorkspaceDispatch: mockEnqueueWorkspaceDispatch, -})) - vi.mock('@/lib/workflows/utils', () => ({ getWorkflowById: vi.fn().mockResolvedValue({ id: 'workflow-1', @@ -175,8 +163,6 @@ const SINGLE_JOB = [ cronExpression: '0 * * * *', failedCount: 0, lastQueuedAt: undefined, - sourceUserId: 'user-1', - sourceWorkspaceId: 'workspace-1', sourceType: 'job', }, ] @@ -250,56 +236,48 @@ describe('Scheduled Workflow Execution API Route', () => { expect(data).toHaveProperty('executedCount', 2) }) - it('should queue mothership jobs to BullMQ when available', async () => { + it('should execute mothership jobs inline', async () => { mockDbReturning.mockReturnValueOnce([]).mockReturnValueOnce(SINGLE_JOB) const response = await GET(createMockRequest()) expect(response.status).toBe(200) - expect(mockEnqueueWorkspaceDispatch).toHaveBeenCalledWith( + expect(mockExecuteJobInline).toHaveBeenCalledWith( expect.objectContaining({ - workspaceId: 'workspace-1', - lane: 'runtime', - queueName: 'mothership-job-execution', - bullmqJobName: 'mothership-job-execution', - bullmqPayload: { - payload: { - scheduleId: 'job-1', - cronExpression: '0 * * * *', - failedCount: 0, - now: expect.any(String), - }, - }, + scheduleId: 'job-1', + cronExpression: '0 * * * *', + failedCount: 0, + now: expect.any(String), }) ) - expect(mockExecuteJobInline).not.toHaveBeenCalled() }) - it('should enqueue preassigned correlation metadata for schedules', async () => { - mockDbReturning.mockReturnValue(SINGLE_SCHEDULE) + it('should enqueue schedule with correlation metadata via job queue', async () => { + mockDbReturning.mockReturnValueOnce(SINGLE_SCHEDULE).mockReturnValueOnce([]) const response = await GET(createMockRequest()) expect(response.status).toBe(200) - expect(mockEnqueueWorkspaceDispatch).toHaveBeenCalledWith( + expect(mockEnqueue).toHaveBeenCalledWith( + 'schedule-execution', expect.objectContaining({ - id: 'schedule-execution-1', - workspaceId: 'workspace-1', - lane: 'runtime', - queueName: 'schedule-execution', - bullmqJobName: 'schedule-execution', - metadata: { + scheduleId: 'schedule-1', + workflowId: 'workflow-1', + executionId: 'schedule-execution-1', + requestId: 'test-request-id', + }), + expect.objectContaining({ + metadata: expect.objectContaining({ workflowId: 'workflow-1', - correlation: { + workspaceId: 'workspace-1', + correlation: expect.objectContaining({ executionId: 'schedule-execution-1', requestId: 'test-request-id', source: 'schedule', workflowId: 'workflow-1', scheduleId: 'schedule-1', - triggerType: 'schedule', - scheduledFor: '2025-01-01T00:00:00.000Z', - }, - }, + }), + }), }) ) }) diff --git a/apps/sim/app/api/schedules/execute/route.ts b/apps/sim/app/api/schedules/execute/route.ts index c89fdd26b9..4748d1f437 100644 --- a/apps/sim/app/api/schedules/execute/route.ts +++ b/apps/sim/app/api/schedules/execute/route.ts @@ -4,10 +4,8 @@ import { and, eq, isNull, lt, lte, ne, not, or, sql } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { verifyCronAuth } from '@/lib/auth/internal' import { getJobQueue, shouldExecuteInline } from '@/lib/core/async-jobs' -import { createBullMQJobData, isBullMQEnabled } from '@/lib/core/bullmq' import { generateRequestId } from '@/lib/core/utils/request' import { generateId } from '@/lib/core/utils/uuid' -import { enqueueWorkspaceDispatch } from '@/lib/core/workspace-dispatch' import { executeJobInline, executeScheduleJob, @@ -75,8 +73,6 @@ export async function GET(request: NextRequest) { cronExpression: workflowSchedule.cronExpression, failedCount: workflowSchedule.failedCount, lastQueuedAt: workflowSchedule.lastQueuedAt, - sourceWorkspaceId: workflowSchedule.sourceWorkspaceId, - sourceUserId: workflowSchedule.sourceUserId, sourceType: workflowSchedule.sourceType, }) @@ -123,38 +119,13 @@ export async function GET(request: NextRequest) { : null const resolvedWorkspaceId = resolvedWorkflow?.workspaceId - let jobId: string - if (isBullMQEnabled()) { - if (!resolvedWorkspaceId) { - throw new Error( - `Missing workspace for scheduled workflow ${schedule.workflowId}; refusing to bypass workspace admission` - ) - } - - jobId = await enqueueWorkspaceDispatch({ - id: executionId, - workspaceId: resolvedWorkspaceId, - lane: 'runtime', - queueName: 'schedule-execution', - bullmqJobName: 'schedule-execution', - bullmqPayload: createBullMQJobData(payload, { - workflowId: schedule.workflowId ?? undefined, - correlation, - }), - metadata: { - workflowId: schedule.workflowId ?? undefined, - correlation, - }, - }) - } else { - jobId = await jobQueue.enqueue('schedule-execution', payload, { - metadata: { - workflowId: schedule.workflowId ?? undefined, - workspaceId: resolvedWorkspaceId ?? undefined, - correlation, - }, - }) - } + const jobId = await jobQueue.enqueue('schedule-execution', payload, { + metadata: { + workflowId: schedule.workflowId ?? undefined, + workspaceId: resolvedWorkspaceId ?? undefined, + correlation, + }, + }) logger.info( `[${requestId}] Queued schedule execution task ${jobId} for workflow ${schedule.workflowId}` ) @@ -206,7 +177,7 @@ export async function GET(request: NextRequest) { } }) - // Mothership jobs use BullMQ when available, otherwise direct inline execution. + // Mothership jobs are executed inline directly. const jobPromises = dueJobs.map(async (job) => { const queueTime = job.lastQueuedAt ?? queuedAt const payload = { @@ -217,24 +188,7 @@ export async function GET(request: NextRequest) { } try { - if (isBullMQEnabled()) { - if (!job.sourceWorkspaceId || !job.sourceUserId) { - throw new Error(`Mothership job ${job.id} is missing workspace/user ownership`) - } - - await enqueueWorkspaceDispatch({ - workspaceId: job.sourceWorkspaceId!, - lane: 'runtime', - queueName: 'mothership-job-execution', - bullmqJobName: 'mothership-job-execution', - bullmqPayload: createBullMQJobData(payload), - metadata: { - userId: job.sourceUserId, - }, - }) - } else { - await executeJobInline(payload) - } + await executeJobInline(payload) } catch (error) { logger.error(`[${requestId}] Job execution failed for ${job.id}`, { error: error instanceof Error ? error.message : String(error), diff --git a/apps/sim/app/api/webhooks/trigger/[path]/route.ts b/apps/sim/app/api/webhooks/trigger/[path]/route.ts index 46ec98d473..0db8bd9ad7 100644 --- a/apps/sim/app/api/webhooks/trigger/[path]/route.ts +++ b/apps/sim/app/api/webhooks/trigger/[path]/route.ts @@ -2,7 +2,6 @@ import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' import { admissionRejectedResponse, tryAdmit } from '@/lib/core/admission/gate' import { generateRequestId } from '@/lib/core/utils/request' -import { DispatchQueueFullError } from '@/lib/core/workspace-dispatch' import { checkWebhookPreprocessing, findAllWebhooksForPath, @@ -167,16 +166,6 @@ async function handleWebhookPost( }) responses.push(response) } catch (error) { - if (error instanceof DispatchQueueFullError) { - return NextResponse.json( - { - error: 'Service temporarily at capacity', - message: error.message, - retryAfterSeconds: 10, - }, - { status: 503, headers: { 'Retry-After': '10' } } - ) - } throw error } } diff --git a/apps/sim/app/api/workflows/[id]/execute/route.async.test.ts b/apps/sim/app/api/workflows/[id]/execute/route.async.test.ts index 77feaf9843..81ef1a01cc 100644 --- a/apps/sim/app/api/workflows/[id]/execute/route.async.test.ts +++ b/apps/sim/app/api/workflows/[id]/execute/route.async.test.ts @@ -10,13 +10,11 @@ const { mockAuthorizeWorkflowByWorkspacePermission, mockPreprocessExecution, mockEnqueue, - mockEnqueueWorkspaceDispatch, } = vi.hoisted(() => ({ mockCheckHybridAuth: vi.fn(), mockAuthorizeWorkflowByWorkspacePermission: vi.fn(), mockPreprocessExecution: vi.fn(), mockEnqueue: vi.fn().mockResolvedValue('job-123'), - mockEnqueueWorkspaceDispatch: vi.fn().mockResolvedValue('job-123'), })) vi.mock('@/lib/auth/hybrid', () => ({ @@ -47,16 +45,6 @@ vi.mock('@/lib/core/async-jobs', () => ({ markJobFailed: vi.fn(), }), shouldExecuteInline: vi.fn().mockReturnValue(false), - shouldUseBullMQ: vi.fn().mockReturnValue(true), -})) - -vi.mock('@/lib/core/bullmq', () => ({ - createBullMQJobData: vi.fn((payload: unknown, metadata?: unknown) => ({ payload, metadata })), -})) - -vi.mock('@/lib/core/workspace-dispatch', () => ({ - enqueueWorkspaceDispatch: mockEnqueueWorkspaceDispatch, - waitForDispatchJob: vi.fn(), })) vi.mock('@/lib/core/utils/request', () => ({ @@ -150,24 +138,28 @@ describe('workflow execute async route', () => { expect(response.status).toBe(202) expect(body.executionId).toBe('execution-123') expect(body.jobId).toBe('job-123') - expect(mockEnqueueWorkspaceDispatch).toHaveBeenCalledWith( + expect(mockEnqueue).toHaveBeenCalledWith( + 'workflow-execution', expect.objectContaining({ - id: 'execution-123', + workflowId: 'workflow-1', + userId: 'actor-1', workspaceId: 'workspace-1', - lane: 'runtime', - queueName: 'workflow-execution', - bullmqJobName: 'workflow-execution', - metadata: { + executionId: 'execution-123', + executionMode: 'async', + }), + expect.objectContaining({ + metadata: expect.objectContaining({ workflowId: 'workflow-1', userId: 'actor-1', - correlation: { + workspaceId: 'workspace-1', + correlation: expect.objectContaining({ executionId: 'execution-123', requestId: 'req-12345678', source: 'workflow', workflowId: 'workflow-1', triggerType: 'manual', - }, - }, + }), + }), }) ) }) diff --git a/apps/sim/app/api/workflows/[id]/execute/route.ts b/apps/sim/app/api/workflows/[id]/execute/route.ts index c715ca2a8b..b2ec869a9b 100644 --- a/apps/sim/app/api/workflows/[id]/execute/route.ts +++ b/apps/sim/app/api/workflows/[id]/execute/route.ts @@ -3,8 +3,7 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { AuthType, checkHybridAuth, hasExternalApiCredentials } from '@/lib/auth/hybrid' import { admissionRejectedResponse, tryAdmit } from '@/lib/core/admission/gate' -import { getJobQueue, shouldExecuteInline, shouldUseBullMQ } from '@/lib/core/async-jobs' -import { createBullMQJobData } from '@/lib/core/bullmq' +import { getJobQueue, shouldExecuteInline } from '@/lib/core/async-jobs' import { createTimeoutAbortController, getTimeoutErrorMessage, @@ -14,13 +13,6 @@ import { generateRequestId } from '@/lib/core/utils/request' import { SSE_HEADERS } from '@/lib/core/utils/sse' import { getBaseUrl } from '@/lib/core/utils/urls' import { generateId, isValidUuid } from '@/lib/core/utils/uuid' -import { - DispatchQueueFullError, - enqueueWorkspaceDispatch, - type WorkspaceDispatchLane, - waitForDispatchJob, -} from '@/lib/core/workspace-dispatch' -import { createBufferedExecutionStream } from '@/lib/execution/buffered-stream' import { buildNextCallChain, parseCallChain, @@ -43,11 +35,6 @@ import { executeWorkflow } from '@/lib/workflows/executor/execute-workflow' import { executeWorkflowCore } from '@/lib/workflows/executor/execution-core' import { type ExecutionEvent, encodeSSEEvent } from '@/lib/workflows/executor/execution-events' import { handlePostExecutionPauseState } from '@/lib/workflows/executor/pause-persistence' -import { - DIRECT_WORKFLOW_JOB_NAME, - type QueuedWorkflowExecutionPayload, - type QueuedWorkflowExecutionResult, -} from '@/lib/workflows/executor/queued-workflow-execution' import { loadDeployedWorkflowState, loadWorkflowFromNormalizedTables, @@ -119,8 +106,6 @@ const ExecuteWorkflowSchema = z.object({ export const runtime = 'nodejs' export const dynamic = 'force-dynamic' -const INLINE_TRIGGER_TYPES = new Set(['manual', 'workflow']) - function resolveOutputIds( selectedOutputs: string[] | undefined, blocks: Record @@ -218,39 +203,19 @@ async function handleAsyncExecution(params: AsyncExecutionParams): Promise { try { - await inlineJobQueue.startJob(jobId) + await jobQueue.startJob(jobId) const output = await executeWorkflowJob(payload) - await inlineJobQueue.completeJob(jobId, output) + await jobQueue.completeJob(jobId, output) } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error) asyncLogger.error('Async workflow execution failed', { @@ -258,7 +223,7 @@ async function handleAsyncExecution(params: AsyncExecutionParams): Promise { - if (!isBullMQEnabled()) { - return false - } - - const lockAcquired = await acquireLock( - `workspace-notification-dispatch:${params.deliveryId}`, - params.deliveryId, - NOTIFICATION_DISPATCH_LOCK_TTL_SECONDS - ) - if (!lockAcquired) { - return false - } - - await enqueueWorkspaceDispatch({ - workspaceId: params.workspaceId, - lane: 'lightweight', - queueName: 'workspace-notification-delivery', - bullmqJobName: 'workspace-notification-delivery', - bullmqPayload: createBullMQJobData(params), - metadata: { - workflowId: params.log.workflowId ?? undefined, - }, - }) - - return true + return false } const STUCK_IN_PROGRESS_THRESHOLD_MS = 5 * 60 * 1000 export async function sweepPendingNotificationDeliveries(limit = 50): Promise { - if (!isBullMQEnabled()) { - return 0 - } - const stuckThreshold = new Date(Date.now() - STUCK_IN_PROGRESS_THRESHOLD_MS) await db diff --git a/apps/sim/lib/billing/workspace-concurrency.test.ts b/apps/sim/lib/billing/workspace-concurrency.test.ts deleted file mode 100644 index 462e24a8e0..0000000000 --- a/apps/sim/lib/billing/workspace-concurrency.test.ts +++ /dev/null @@ -1,146 +0,0 @@ -/** - * @vitest-environment node - */ -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { - mockGetHighestPrioritySubscription, - mockGetWorkspaceBilledAccountUserId, - mockFeatureFlags, - mockRedisGet, - mockRedisSet, - mockRedisDel, - mockRedisKeys, - mockGetRedisClient, -} = vi.hoisted(() => ({ - mockGetHighestPrioritySubscription: vi.fn(), - mockGetWorkspaceBilledAccountUserId: vi.fn(), - mockFeatureFlags: { - isBillingEnabled: true, - }, - mockRedisGet: vi.fn(), - mockRedisSet: vi.fn(), - mockRedisDel: vi.fn(), - mockRedisKeys: vi.fn(), - mockGetRedisClient: vi.fn(), -})) - -vi.mock('@sim/logger', () => ({ - createLogger: () => ({ - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - debug: vi.fn(), - }), -})) - -vi.mock('@/lib/billing/core/plan', () => ({ - getHighestPrioritySubscription: mockGetHighestPrioritySubscription, -})) - -vi.mock('@/lib/workspaces/utils', () => ({ - getWorkspaceBilledAccountUserId: mockGetWorkspaceBilledAccountUserId, -})) - -vi.mock('@/lib/core/config/redis', () => ({ - getRedisClient: mockGetRedisClient, -})) - -vi.mock('@/lib/core/config/feature-flags', () => mockFeatureFlags) - -import { - getWorkspaceConcurrencyLimit, - resetWorkspaceConcurrencyLimitCache, -} from '@/lib/billing/workspace-concurrency' - -describe('workspace concurrency billing', () => { - beforeEach(() => { - vi.clearAllMocks() - mockFeatureFlags.isBillingEnabled = true - - mockRedisGet.mockResolvedValue(null) - mockRedisSet.mockResolvedValue('OK') - mockRedisDel.mockResolvedValue(1) - mockRedisKeys.mockResolvedValue([]) - mockGetRedisClient.mockReturnValue({ - get: mockRedisGet, - set: mockRedisSet, - del: mockRedisDel, - keys: mockRedisKeys, - }) - }) - - it('returns free tier when no billed account exists', async () => { - mockGetWorkspaceBilledAccountUserId.mockResolvedValue(null) - - await expect(getWorkspaceConcurrencyLimit('workspace-1')).resolves.toBe(5) - }) - - it('returns pro limit for pro billing accounts', async () => { - mockGetWorkspaceBilledAccountUserId.mockResolvedValue('user-1') - mockGetHighestPrioritySubscription.mockResolvedValue({ - plan: 'pro_6000', - metadata: null, - }) - - await expect(getWorkspaceConcurrencyLimit('workspace-1')).resolves.toBe(50) - }) - - it('returns max limit for max plan tiers', async () => { - mockGetWorkspaceBilledAccountUserId.mockResolvedValue('user-1') - mockGetHighestPrioritySubscription.mockResolvedValue({ - plan: 'pro_25000', - metadata: null, - }) - - await expect(getWorkspaceConcurrencyLimit('workspace-1')).resolves.toBe(200) - }) - - it('returns max limit for legacy team plans', async () => { - mockGetWorkspaceBilledAccountUserId.mockResolvedValue('user-1') - mockGetHighestPrioritySubscription.mockResolvedValue({ - plan: 'team', - metadata: null, - }) - - await expect(getWorkspaceConcurrencyLimit('workspace-1')).resolves.toBe(200) - }) - - it('returns enterprise metadata override when present', async () => { - mockGetWorkspaceBilledAccountUserId.mockResolvedValue('user-1') - mockGetHighestPrioritySubscription.mockResolvedValue({ - plan: 'enterprise', - metadata: { - workspaceConcurrencyLimit: '350', - }, - }) - - await expect(getWorkspaceConcurrencyLimit('workspace-1')).resolves.toBe(350) - }) - - it('uses free-tier limit when billing is disabled', async () => { - mockFeatureFlags.isBillingEnabled = false - mockGetWorkspaceBilledAccountUserId.mockResolvedValue('user-1') - mockGetHighestPrioritySubscription.mockResolvedValue({ - plan: 'pro_25000', - metadata: { - workspaceConcurrencyLimit: 999, - }, - }) - - await expect(getWorkspaceConcurrencyLimit('workspace-1')).resolves.toBe(5) - }) - - it('uses redis cache when available', async () => { - mockRedisGet.mockResolvedValueOnce('123') - - await expect(getWorkspaceConcurrencyLimit('workspace-1')).resolves.toBe(123) - expect(mockGetWorkspaceBilledAccountUserId).not.toHaveBeenCalled() - }) - - it('can clear a specific workspace cache entry', async () => { - await resetWorkspaceConcurrencyLimitCache('workspace-1') - - expect(mockRedisDel).toHaveBeenCalledWith('workspace-concurrency-limit:workspace-1') - }) -}) diff --git a/apps/sim/lib/billing/workspace-concurrency.ts b/apps/sim/lib/billing/workspace-concurrency.ts deleted file mode 100644 index acb07169f9..0000000000 --- a/apps/sim/lib/billing/workspace-concurrency.ts +++ /dev/null @@ -1,170 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getHighestPrioritySubscription } from '@/lib/billing/core/plan' -import { getPlanTierCredits, isEnterprise, isPro, isTeam } from '@/lib/billing/plan-helpers' -import { parseEnterpriseWorkspaceConcurrencyMetadata } from '@/lib/billing/types' -import { env } from '@/lib/core/config/env' -import { isBillingEnabled } from '@/lib/core/config/feature-flags' -import { getRedisClient } from '@/lib/core/config/redis' -import { getWorkspaceBilledAccountUserId } from '@/lib/workspaces/utils' - -const logger = createLogger('WorkspaceConcurrencyBilling') - -const CACHE_TTL_MS = 60_000 -const CACHE_TTL_SECONDS = Math.floor(CACHE_TTL_MS / 1000) - -interface CacheEntry { - value: number - expiresAt: number -} - -const inMemoryConcurrencyCache = new Map() - -function cacheKey(workspaceId: string): string { - return `workspace-concurrency-limit:${workspaceId}` -} - -function parsePositiveLimit(value: unknown): number | null { - if (typeof value === 'number' && Number.isFinite(value) && value > 0) { - return Math.floor(value) - } - - if (typeof value === 'string') { - const parsed = Number.parseInt(value, 10) - if (Number.isFinite(parsed) && parsed > 0) { - return parsed - } - } - - return null -} - -function getFreeConcurrencyLimit(): number { - return Number.parseInt(env.WORKSPACE_CONCURRENCY_FREE, 10) || 5 -} - -function getProConcurrencyLimit(): number { - return Number.parseInt(env.WORKSPACE_CONCURRENCY_PRO, 10) || 50 -} - -function getTeamConcurrencyLimit(): number { - return Number.parseInt(env.WORKSPACE_CONCURRENCY_TEAM, 10) || 200 -} - -function getEnterpriseDefaultConcurrencyLimit(): number { - return Number.parseInt(env.WORKSPACE_CONCURRENCY_ENTERPRISE, 10) || 200 -} - -function getEnterpriseConcurrencyLimit(metadata: unknown): number { - const enterpriseMetadata = parseEnterpriseWorkspaceConcurrencyMetadata(metadata) - return enterpriseMetadata?.workspaceConcurrencyLimit ?? getEnterpriseDefaultConcurrencyLimit() -} - -function getPlanConcurrencyLimit(plan: string | null | undefined, metadata: unknown): number { - if (!isBillingEnabled) { - return getFreeConcurrencyLimit() - } - - if (!plan) { - return getFreeConcurrencyLimit() - } - - if (isEnterprise(plan)) { - return getEnterpriseConcurrencyLimit(metadata) - } - - if (isTeam(plan)) { - return getTeamConcurrencyLimit() - } - - const credits = getPlanTierCredits(plan) - if (credits >= 25_000) { - return getTeamConcurrencyLimit() - } - - if (isPro(plan)) { - return getProConcurrencyLimit() - } - - return getFreeConcurrencyLimit() -} - -export async function getWorkspaceConcurrencyLimit(workspaceId: string): Promise { - const redis = getRedisClient() - - if (redis) { - const cached = await redis.get(cacheKey(workspaceId)) - const cachedValue = parsePositiveLimit(cached) - if (cachedValue !== null) { - return cachedValue - } - } else { - const cached = inMemoryConcurrencyCache.get(workspaceId) - if (cached && cached.expiresAt > Date.now()) { - return cached.value - } - } - - try { - const billedAccountUserId = await getWorkspaceBilledAccountUserId(workspaceId) - if (!billedAccountUserId) { - if (redis) { - await redis.set( - cacheKey(workspaceId), - String(getFreeConcurrencyLimit()), - 'EX', - CACHE_TTL_SECONDS - ) - } else { - inMemoryConcurrencyCache.set(workspaceId, { - value: getFreeConcurrencyLimit(), - expiresAt: Date.now() + CACHE_TTL_MS, - }) - } - return getFreeConcurrencyLimit() - } - - const subscription = await getHighestPrioritySubscription(billedAccountUserId) - const limit = getPlanConcurrencyLimit(subscription?.plan, subscription?.metadata) - - if (redis) { - await redis.set(cacheKey(workspaceId), String(limit), 'EX', CACHE_TTL_SECONDS) - } else { - inMemoryConcurrencyCache.set(workspaceId, { - value: limit, - expiresAt: Date.now() + CACHE_TTL_MS, - }) - } - - return limit - } catch (error) { - logger.error('Failed to resolve workspace concurrency limit, using free tier', { - workspaceId, - error, - }) - - return getFreeConcurrencyLimit() - } -} - -export async function resetWorkspaceConcurrencyLimitCache(workspaceId?: string): Promise { - if (!workspaceId) { - inMemoryConcurrencyCache.clear() - } else { - inMemoryConcurrencyCache.delete(workspaceId) - } - - const redis = getRedisClient() - if (!redis) { - return - } - - if (workspaceId) { - await redis.del(cacheKey(workspaceId)) - return - } - - const keys = await redis.keys('workspace-concurrency-limit:*') - if (keys.length > 0) { - await redis.del(...keys) - } -} diff --git a/apps/sim/lib/core/async-jobs/backends/bullmq.ts b/apps/sim/lib/core/async-jobs/backends/bullmq.ts deleted file mode 100644 index f67aead4ce..0000000000 --- a/apps/sim/lib/core/async-jobs/backends/bullmq.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { createLogger } from '@sim/logger' -import type { Job as BullMQJob } from 'bullmq' -import { - type EnqueueOptions, - JOB_STATUS, - type Job, - type JobQueueBackend, - type JobStatus, - type JobType, -} from '@/lib/core/async-jobs/types' -import { type BullMQJobData, createBullMQJobData, getBullMQQueue } from '@/lib/core/bullmq' - -const logger = createLogger('BullMQJobQueue') - -function mapBullMQStatus(status: string): JobStatus { - switch (status) { - case 'active': - return JOB_STATUS.PROCESSING - case 'completed': - return JOB_STATUS.COMPLETED - case 'failed': - return JOB_STATUS.FAILED - default: - return JOB_STATUS.PENDING - } -} - -async function toJob( - queueType: JobType, - bullJob: BullMQJob> | null -): Promise { - if (!bullJob) { - return null - } - - const status = mapBullMQStatus(await bullJob.getState()) - - return { - id: bullJob.id ?? '', - type: queueType, - payload: bullJob.data.payload, - status, - createdAt: new Date(bullJob.timestamp), - startedAt: bullJob.processedOn ? new Date(bullJob.processedOn) : undefined, - completedAt: bullJob.finishedOn ? new Date(bullJob.finishedOn) : undefined, - attempts: bullJob.attemptsMade, - maxAttempts: bullJob.opts.attempts ?? 1, - error: bullJob.failedReason || undefined, - output: bullJob.returnvalue, - metadata: bullJob.data.metadata ?? {}, - } -} - -export class BullMQJobQueue implements JobQueueBackend { - async enqueue( - type: JobType, - payload: TPayload, - options?: EnqueueOptions - ): Promise { - const queue = getBullMQQueue(type) - - const job = await queue.add( - options?.name ?? type, - createBullMQJobData(payload, options?.metadata), - { - jobId: options?.jobId, - attempts: options?.maxAttempts, - priority: options?.priority, - delay: options?.delayMs, - } - ) - - logger.debug('Enqueued job via BullMQ', { - jobId: job.id, - type, - name: options?.name ?? type, - }) - - return String(job.id) - } - - async getJob(jobId: string): Promise { - const workflowJob = await getBullMQQueue('workflow-execution').getJob(jobId) - if (workflowJob) { - return toJob('workflow-execution', workflowJob) - } - - const webhookJob = await getBullMQQueue('webhook-execution').getJob(jobId) - if (webhookJob) { - return toJob('webhook-execution', webhookJob) - } - - const scheduleJob = await getBullMQQueue('schedule-execution').getJob(jobId) - if (scheduleJob) { - return toJob('schedule-execution', scheduleJob) - } - - const resumeJob = await getBullMQQueue('resume-execution').getJob(jobId) - if (resumeJob) { - return toJob('resume-execution', resumeJob) - } - - return null - } - - async startJob(_jobId: string): Promise {} - - async completeJob(_jobId: string, _output: unknown): Promise {} - - async markJobFailed(_jobId: string, _error: string): Promise {} -} diff --git a/apps/sim/lib/core/async-jobs/backends/index.ts b/apps/sim/lib/core/async-jobs/backends/index.ts index 0abb55d6af..86c7a990f1 100644 --- a/apps/sim/lib/core/async-jobs/backends/index.ts +++ b/apps/sim/lib/core/async-jobs/backends/index.ts @@ -1,3 +1,2 @@ -export { BullMQJobQueue } from './bullmq' export { DatabaseJobQueue } from './database' export { TriggerDevJobQueue } from './trigger-dev' diff --git a/apps/sim/lib/core/async-jobs/config.ts b/apps/sim/lib/core/async-jobs/config.ts index c4f0a4dcf8..b28f5453ea 100644 --- a/apps/sim/lib/core/async-jobs/config.ts +++ b/apps/sim/lib/core/async-jobs/config.ts @@ -1,6 +1,5 @@ import { createLogger } from '@sim/logger' import type { AsyncBackendType, JobQueueBackend } from '@/lib/core/async-jobs/types' -import { isBullMQEnabled } from '@/lib/core/bullmq' import { isTriggerDevEnabled } from '@/lib/core/config/feature-flags' const logger = createLogger('AsyncJobsConfig') @@ -11,17 +10,13 @@ let cachedInlineBackend: JobQueueBackend | null = null /** * Determines which async backend to use based on environment configuration. - * Follows the fallback chain: trigger.dev → bullmq → database + * Follows the fallback chain: trigger.dev → database */ export function getAsyncBackendType(): AsyncBackendType { if (isTriggerDevEnabled) { return 'trigger-dev' } - if (isBullMQEnabled()) { - return 'bullmq' - } - return 'database' } @@ -42,11 +37,6 @@ export async function getJobQueue(): Promise { cachedBackend = new TriggerDevJobQueue() break } - case 'bullmq': { - const { BullMQJobQueue } = await import('@/lib/core/async-jobs/backends/bullmq') - cachedBackend = new BullMQJobQueue() - break - } case 'database': { const { DatabaseJobQueue } = await import('@/lib/core/async-jobs/backends/database') cachedBackend = new DatabaseJobQueue() @@ -72,7 +62,7 @@ export function getCurrentBackendType(): AsyncBackendType | null { } /** - * Gets a job queue backend that bypasses Trigger.dev (BullMQ -> Database). + * Gets a job queue backend that bypasses Trigger.dev (Database only). * Used for execution paths that must avoid Trigger.dev cold starts. */ export async function getInlineJobQueue(): Promise { @@ -80,18 +70,10 @@ export async function getInlineJobQueue(): Promise { return cachedInlineBackend } - let type: string - if (isBullMQEnabled()) { - const { BullMQJobQueue } = await import('@/lib/core/async-jobs/backends/bullmq') - cachedInlineBackend = new BullMQJobQueue() - type = 'bullmq' - } else { - const { DatabaseJobQueue } = await import('@/lib/core/async-jobs/backends/database') - cachedInlineBackend = new DatabaseJobQueue() - type = 'database' - } + const { DatabaseJobQueue } = await import('@/lib/core/async-jobs/backends/database') + cachedInlineBackend = new DatabaseJobQueue() - logger.info(`Inline job backend initialized: ${type}`) + logger.info('Inline job backend initialized: database') return cachedInlineBackend } @@ -103,10 +85,6 @@ export function shouldExecuteInline(): boolean { return getAsyncBackendType() === 'database' } -export function shouldUseBullMQ(): boolean { - return isBullMQEnabled() -} - /** * Resets the cached backend (useful for testing) */ diff --git a/apps/sim/lib/core/async-jobs/index.ts b/apps/sim/lib/core/async-jobs/index.ts index 76ec707220..24e6f1e526 100644 --- a/apps/sim/lib/core/async-jobs/index.ts +++ b/apps/sim/lib/core/async-jobs/index.ts @@ -5,7 +5,6 @@ export { getJobQueue, resetJobQueueCache, shouldExecuteInline, - shouldUseBullMQ, } from './config' export type { AsyncBackendType, diff --git a/apps/sim/lib/core/async-jobs/types.ts b/apps/sim/lib/core/async-jobs/types.ts index 7b09bdb7b0..7531a46eca 100644 --- a/apps/sim/lib/core/async-jobs/types.ts +++ b/apps/sim/lib/core/async-jobs/types.ts @@ -105,4 +105,4 @@ export interface JobQueueBackend { markJobFailed(jobId: string, error: string): Promise } -export type AsyncBackendType = 'trigger-dev' | 'bullmq' | 'database' +export type AsyncBackendType = 'trigger-dev' | 'database' diff --git a/apps/sim/lib/core/bullmq/connection.ts b/apps/sim/lib/core/bullmq/connection.ts deleted file mode 100644 index e888f5772a..0000000000 --- a/apps/sim/lib/core/bullmq/connection.ts +++ /dev/null @@ -1,29 +0,0 @@ -import type { ConnectionOptions } from 'bullmq' -import { env, isTruthy } from '@/lib/core/config/env' - -export function isBullMQEnabled(): boolean { - return isTruthy(env.CONCURRENCY_CONTROL_ENABLED) && Boolean(env.REDIS_URL) -} - -export function getBullMQConnectionOptions(): ConnectionOptions { - if (!env.REDIS_URL) { - throw new Error('BullMQ requires REDIS_URL') - } - - const redisUrl = new URL(env.REDIS_URL) - const isTls = redisUrl.protocol === 'rediss:' - const port = redisUrl.port ? Number.parseInt(redisUrl.port, 10) : 6379 - const dbPath = redisUrl.pathname.replace('/', '') - const db = dbPath ? Number.parseInt(dbPath, 10) : undefined - - return { - host: redisUrl.hostname, - port, - username: redisUrl.username || undefined, - password: redisUrl.password || undefined, - db: Number.isFinite(db) ? db : undefined, - maxRetriesPerRequest: null, - enableReadyCheck: false, - ...(isTls ? { tls: {} } : {}), - } -} diff --git a/apps/sim/lib/core/bullmq/index.ts b/apps/sim/lib/core/bullmq/index.ts deleted file mode 100644 index efe937aa47..0000000000 --- a/apps/sim/lib/core/bullmq/index.ts +++ /dev/null @@ -1,16 +0,0 @@ -export { getBullMQConnectionOptions, isBullMQEnabled } from './connection' -export { - type BullMQJobData, - createBullMQJobData, - getBullMQQueue, - getBullMQQueueByName, - getKnowledgeConnectorSyncQueue, - getKnowledgeDocumentProcessingQueue, - getMothershipJobExecutionQueue, - getWorkflowQueueEvents, - getWorkspaceNotificationDeliveryQueue, - KNOWLEDGE_CONNECTOR_SYNC_QUEUE, - KNOWLEDGE_DOCUMENT_PROCESSING_QUEUE, - MOTHERSHIP_JOB_EXECUTION_QUEUE, - WORKSPACE_NOTIFICATION_DELIVERY_QUEUE, -} from './queues' diff --git a/apps/sim/lib/core/bullmq/queues.ts b/apps/sim/lib/core/bullmq/queues.ts deleted file mode 100644 index f90c2e90c8..0000000000 --- a/apps/sim/lib/core/bullmq/queues.ts +++ /dev/null @@ -1,209 +0,0 @@ -import { Queue, QueueEvents } from 'bullmq' -import type { JobMetadata, JobType } from '@/lib/core/async-jobs/types' -import { getBullMQConnectionOptions } from '@/lib/core/bullmq/connection' -import type { WorkspaceDispatchQueueName } from '@/lib/core/workspace-dispatch/types' - -export const KNOWLEDGE_CONNECTOR_SYNC_QUEUE = 'knowledge-connector-sync' as const -export const KNOWLEDGE_DOCUMENT_PROCESSING_QUEUE = 'knowledge-process-document' as const -export const MOTHERSHIP_JOB_EXECUTION_QUEUE = 'mothership-job-execution' as const -export const WORKSPACE_NOTIFICATION_DELIVERY_QUEUE = 'workspace-notification-delivery' as const - -export interface BullMQJobData { - payload: TPayload - metadata?: JobMetadata -} - -let workflowQueueInstance: Queue | null = null -let webhookQueueInstance: Queue | null = null -let scheduleQueueInstance: Queue | null = null -let resumeQueueInstance: Queue | null = null -let knowledgeConnectorSyncQueueInstance: Queue | null = null -let knowledgeDocumentProcessingQueueInstance: Queue | null = null -let mothershipJobExecutionQueueInstance: Queue | null = null -let workspaceNotificationDeliveryQueueInstance: Queue | null = null -let workflowQueueEventsInstance: QueueEvents | null = null - -function getQueueDefaultOptions(type: JobType) { - switch (type) { - case 'workflow-execution': - return { - attempts: 3, - backoff: { type: 'exponential' as const, delay: 1000 }, - removeOnComplete: { age: 24 * 60 * 60 }, - removeOnFail: { age: 7 * 24 * 60 * 60 }, - } - case 'webhook-execution': - return { - attempts: 2, - backoff: { type: 'exponential' as const, delay: 2000 }, - removeOnComplete: { age: 24 * 60 * 60 }, - removeOnFail: { age: 3 * 24 * 60 * 60 }, - } - case 'schedule-execution': - return { - attempts: 2, - backoff: { type: 'exponential' as const, delay: 5000 }, - removeOnComplete: { age: 24 * 60 * 60 }, - removeOnFail: { age: 3 * 24 * 60 * 60 }, - } - case 'resume-execution': - return { - attempts: 1, - removeOnComplete: { age: 24 * 60 * 60 }, - removeOnFail: { age: 3 * 24 * 60 * 60 }, - } - } -} - -function createQueue(type: JobType): Queue { - return new Queue(type, { - connection: getBullMQConnectionOptions(), - defaultJobOptions: getQueueDefaultOptions(type), - }) -} - -function createNamedQueue( - name: - | typeof KNOWLEDGE_CONNECTOR_SYNC_QUEUE - | typeof KNOWLEDGE_DOCUMENT_PROCESSING_QUEUE - | typeof MOTHERSHIP_JOB_EXECUTION_QUEUE - | typeof WORKSPACE_NOTIFICATION_DELIVERY_QUEUE -): Queue { - switch (name) { - case KNOWLEDGE_CONNECTOR_SYNC_QUEUE: - return new Queue(name, { - connection: getBullMQConnectionOptions(), - defaultJobOptions: { - attempts: 3, - backoff: { type: 'exponential', delay: 5000 }, - removeOnComplete: { age: 24 * 60 * 60 }, - removeOnFail: { age: 7 * 24 * 60 * 60 }, - }, - }) - case KNOWLEDGE_DOCUMENT_PROCESSING_QUEUE: - return new Queue(name, { - connection: getBullMQConnectionOptions(), - defaultJobOptions: { - attempts: 3, - backoff: { type: 'exponential', delay: 1000 }, - removeOnComplete: { age: 24 * 60 * 60 }, - removeOnFail: { age: 7 * 24 * 60 * 60 }, - }, - }) - case MOTHERSHIP_JOB_EXECUTION_QUEUE: - return new Queue(name, { - connection: getBullMQConnectionOptions(), - defaultJobOptions: { - attempts: 1, - removeOnComplete: { age: 24 * 60 * 60 }, - removeOnFail: { age: 7 * 24 * 60 * 60 }, - }, - }) - case WORKSPACE_NOTIFICATION_DELIVERY_QUEUE: - return new Queue(name, { - connection: getBullMQConnectionOptions(), - defaultJobOptions: { - attempts: 1, - removeOnComplete: { age: 24 * 60 * 60 }, - removeOnFail: { age: 7 * 24 * 60 * 60 }, - }, - }) - } -} - -export function getBullMQQueue(type: JobType): Queue { - switch (type) { - case 'workflow-execution': - if (!workflowQueueInstance) { - workflowQueueInstance = createQueue(type) - } - return workflowQueueInstance - case 'webhook-execution': - if (!webhookQueueInstance) { - webhookQueueInstance = createQueue(type) - } - return webhookQueueInstance - case 'schedule-execution': - if (!scheduleQueueInstance) { - scheduleQueueInstance = createQueue(type) - } - return scheduleQueueInstance - case 'resume-execution': - if (!resumeQueueInstance) { - resumeQueueInstance = createQueue(type) - } - return resumeQueueInstance - } -} - -export function getBullMQQueueByName(queueName: WorkspaceDispatchQueueName): Queue { - switch (queueName) { - case 'workflow-execution': - case 'webhook-execution': - case 'schedule-execution': - case 'resume-execution': - return getBullMQQueue(queueName) - case KNOWLEDGE_CONNECTOR_SYNC_QUEUE: - return getKnowledgeConnectorSyncQueue() - case KNOWLEDGE_DOCUMENT_PROCESSING_QUEUE: - return getKnowledgeDocumentProcessingQueue() - case MOTHERSHIP_JOB_EXECUTION_QUEUE: - return getMothershipJobExecutionQueue() - case WORKSPACE_NOTIFICATION_DELIVERY_QUEUE: - return getWorkspaceNotificationDeliveryQueue() - } -} - -export function getWorkflowQueueEvents(): QueueEvents { - if (!workflowQueueEventsInstance) { - workflowQueueEventsInstance = new QueueEvents('workflow-execution', { - connection: getBullMQConnectionOptions(), - }) - } - - return workflowQueueEventsInstance -} - -export function getKnowledgeConnectorSyncQueue(): Queue { - if (!knowledgeConnectorSyncQueueInstance) { - knowledgeConnectorSyncQueueInstance = createNamedQueue(KNOWLEDGE_CONNECTOR_SYNC_QUEUE) - } - - return knowledgeConnectorSyncQueueInstance -} - -export function getKnowledgeDocumentProcessingQueue(): Queue { - if (!knowledgeDocumentProcessingQueueInstance) { - knowledgeDocumentProcessingQueueInstance = createNamedQueue(KNOWLEDGE_DOCUMENT_PROCESSING_QUEUE) - } - - return knowledgeDocumentProcessingQueueInstance -} - -export function getMothershipJobExecutionQueue(): Queue { - if (!mothershipJobExecutionQueueInstance) { - mothershipJobExecutionQueueInstance = createNamedQueue(MOTHERSHIP_JOB_EXECUTION_QUEUE) - } - - return mothershipJobExecutionQueueInstance -} - -export function getWorkspaceNotificationDeliveryQueue(): Queue { - if (!workspaceNotificationDeliveryQueueInstance) { - workspaceNotificationDeliveryQueueInstance = createNamedQueue( - WORKSPACE_NOTIFICATION_DELIVERY_QUEUE - ) - } - - return workspaceNotificationDeliveryQueueInstance -} - -export function createBullMQJobData( - payload: TPayload, - metadata?: JobMetadata -): BullMQJobData { - return { - payload, - metadata: metadata ?? {}, - } -} diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index bef5712fab..b6bee85504 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -190,10 +190,7 @@ export const env = createEnv({ FREE_PLAN_LOG_RETENTION_DAYS: z.string().optional(), // Log retention days for free plan users // Admission & Burst Protection - CONCURRENCY_CONTROL_ENABLED: z.string().optional().default('false'), // Set to 'true' to enable BullMQ-based concurrency control (default: inline execution) ADMISSION_GATE_MAX_INFLIGHT: z.string().optional().default('500'), // Max concurrent in-flight execution requests per pod - DISPATCH_MAX_QUEUE_PER_WORKSPACE: z.string().optional().default('1000'), // Max queued dispatch jobs per workspace - DISPATCH_MAX_QUEUE_GLOBAL: z.string().optional().default('50000'), // Max queued dispatch jobs globally // Rate Limiting Configuration RATE_LIMIT_WINDOW_MS: z.string().optional().default('60000'), // Rate limit window duration in milliseconds (default: 1 minute) @@ -206,11 +203,6 @@ export const env = createEnv({ RATE_LIMIT_TEAM_ASYNC: z.string().optional().default('2500'), // Team tier async API executions per minute RATE_LIMIT_ENTERPRISE_SYNC: z.string().optional().default('600'), // Enterprise tier sync API executions per minute RATE_LIMIT_ENTERPRISE_ASYNC: z.string().optional().default('5000'), // Enterprise tier async API executions per minute - WORKSPACE_CONCURRENCY_FREE: z.string().optional().default('5'), // Free tier concurrent workspace executions - WORKSPACE_CONCURRENCY_PRO: z.string().optional().default('50'), // Pro tier concurrent workspace executions - WORKSPACE_CONCURRENCY_TEAM: z.string().optional().default('200'), // Team/Max tier concurrent workspace executions - WORKSPACE_CONCURRENCY_ENTERPRISE: z.string().optional().default('200'), // Enterprise default concurrent workspace executions - // Timeout Configuration EXECUTION_TIMEOUT_FREE: z.string().optional().default('300'), // 5 minutes EXECUTION_TIMEOUT_PRO: z.string().optional().default('3000'), // 50 minutes diff --git a/apps/sim/lib/core/workspace-dispatch/adapter.ts b/apps/sim/lib/core/workspace-dispatch/adapter.ts deleted file mode 100644 index 6d55a70ff3..0000000000 --- a/apps/sim/lib/core/workspace-dispatch/adapter.ts +++ /dev/null @@ -1,80 +0,0 @@ -import type { - WorkspaceDispatchClaimResult, - WorkspaceDispatchEnqueueInput, - WorkspaceDispatchJobRecord, - WorkspaceDispatchLane, -} from '@/lib/core/workspace-dispatch/types' - -export interface WorkspaceDispatchStorageAdapter { - saveDispatchJob(record: WorkspaceDispatchJobRecord): Promise - getDispatchJobRecord(jobId: string): Promise - listDispatchJobsByStatuses( - statuses: readonly WorkspaceDispatchJobRecord['status'][] - ): Promise - updateDispatchJobRecord( - jobId: string, - updater: (record: WorkspaceDispatchJobRecord) => WorkspaceDispatchJobRecord - ): Promise - enqueueWorkspaceDispatchJob( - input: WorkspaceDispatchEnqueueInput - ): Promise - restoreWorkspaceDispatchJob(record: WorkspaceDispatchJobRecord): Promise - claimWorkspaceJob( - workspaceId: string, - options: { - lanes: readonly WorkspaceDispatchLane[] - concurrencyLimit: number - leaseId: string - now: number - leaseTtlMs: number - } - ): Promise - getWorkspaceQueueDepth( - workspaceId: string, - lanes: readonly WorkspaceDispatchLane[] - ): Promise - getGlobalQueueDepth(): Promise - reconcileGlobalQueueDepth(knownCount: number): Promise - popNextWorkspaceId(): Promise - getQueuedWorkspaceCount(): Promise - hasActiveWorkspace(workspaceId: string): Promise - ensureWorkspaceActive(workspaceId: string, readyAt?: number): Promise - requeueWorkspaceId(workspaceId: string): Promise - workspaceHasPendingJobs( - workspaceId: string, - lanes: readonly WorkspaceDispatchLane[] - ): Promise - getNextWorkspaceJob( - workspaceId: string, - lanes: readonly WorkspaceDispatchLane[] - ): Promise - removeWorkspaceJobFromLane( - workspaceId: string, - lane: WorkspaceDispatchLane, - jobId: string - ): Promise - cleanupExpiredWorkspaceLeases(workspaceId: string): Promise - countActiveWorkspaceLeases(workspaceId: string): Promise - hasWorkspaceLease(workspaceId: string, leaseId: string): Promise - createWorkspaceLease(workspaceId: string, leaseId: string, ttlMs: number): Promise - refreshWorkspaceLease(workspaceId: string, leaseId: string, ttlMs: number): Promise - releaseWorkspaceLease(workspaceId: string, leaseId: string): Promise - removeWorkspaceIfIdle(workspaceId: string, lanes: readonly WorkspaceDispatchLane[]): Promise - markDispatchJobAdmitted( - jobId: string, - workspaceId: string, - leaseId: string, - leaseExpiresAt: number - ): Promise - markDispatchJobAdmitting( - jobId: string, - workspaceId: string, - leaseId: string, - leaseExpiresAt: number - ): Promise - markDispatchJobRunning(jobId: string): Promise - markDispatchJobCompleted(jobId: string, output: unknown): Promise - markDispatchJobFailed(jobId: string, error: string): Promise - clear(): Promise - dispose(): void -} diff --git a/apps/sim/lib/core/workspace-dispatch/dispatcher.test.ts b/apps/sim/lib/core/workspace-dispatch/dispatcher.test.ts deleted file mode 100644 index 6daa485f91..0000000000 --- a/apps/sim/lib/core/workspace-dispatch/dispatcher.test.ts +++ /dev/null @@ -1,175 +0,0 @@ -/** - * @vitest-environment node - */ -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { mockGetWorkspaceConcurrencyLimit, mockAcquireLock, mockReleaseLock } = vi.hoisted(() => ({ - mockGetWorkspaceConcurrencyLimit: vi.fn(), - mockAcquireLock: vi.fn(), - mockReleaseLock: vi.fn(), -})) - -vi.mock('@/lib/billing/workspace-concurrency', () => ({ - getWorkspaceConcurrencyLimit: mockGetWorkspaceConcurrencyLimit, -})) - -vi.mock('@/lib/core/config/redis', () => ({ - acquireLock: mockAcquireLock, - releaseLock: mockReleaseLock, - getRedisClient: vi.fn().mockReturnValue(null), -})) - -vi.mock('@/lib/core/bullmq', () => ({ - getBullMQQueueByName: vi.fn().mockReturnValue({ - add: vi.fn().mockResolvedValue({ id: 'bullmq-1' }), - }), -})) - -import { MemoryWorkspaceDispatchStorage } from '@/lib/core/workspace-dispatch/memory-store' -import { - DISPATCH_SCAN_RESULTS, - dispatchNextAdmissibleWorkspaceJob, -} from '@/lib/core/workspace-dispatch/planner' -import { - enqueueWorkspaceDispatchJob, - setWorkspaceDispatchStorageAdapter, -} from '@/lib/core/workspace-dispatch/store' - -describe('workspace dispatch integration (memory-backed)', () => { - let store: MemoryWorkspaceDispatchStorage - - beforeEach(async () => { - vi.clearAllMocks() - store = new MemoryWorkspaceDispatchStorage() - setWorkspaceDispatchStorageAdapter(store) - - mockGetWorkspaceConcurrencyLimit.mockResolvedValue(5) - mockAcquireLock.mockResolvedValue(true) - mockReleaseLock.mockResolvedValue(true) - }) - - async function enqueue( - workspaceId: string, - overrides: { lane?: string; delayMs?: number; priority?: number } = {} - ) { - return enqueueWorkspaceDispatchJob({ - workspaceId, - lane: (overrides.lane ?? 'runtime') as 'runtime', - queueName: 'workflow-execution', - bullmqJobName: 'workflow-execution', - bullmqPayload: { payload: { workflowId: 'wf-1' } }, - metadata: { workflowId: 'wf-1' }, - delayMs: overrides.delayMs, - priority: overrides.priority, - }) - } - - it('admits jobs round-robin across workspaces', async () => { - await enqueue('ws-a') - await enqueue('ws-b') - await enqueue('ws-a') - - const r1 = await dispatchNextAdmissibleWorkspaceJob() - const r2 = await dispatchNextAdmissibleWorkspaceJob() - const r3 = await dispatchNextAdmissibleWorkspaceJob() - - expect(r1).toBe(DISPATCH_SCAN_RESULTS.ADMITTED) - expect(r2).toBe(DISPATCH_SCAN_RESULTS.ADMITTED) - expect(r3).toBe(DISPATCH_SCAN_RESULTS.ADMITTED) - }) - - it('respects workspace concurrency limits', async () => { - mockGetWorkspaceConcurrencyLimit.mockResolvedValue(1) - - await enqueue('ws-a') - await enqueue('ws-a') - - const r1 = await dispatchNextAdmissibleWorkspaceJob() - expect(r1).toBe(DISPATCH_SCAN_RESULTS.ADMITTED) - - const r2 = await dispatchNextAdmissibleWorkspaceJob() - expect(r2).toBe(DISPATCH_SCAN_RESULTS.NO_PROGRESS) - }) - - it('skips delayed jobs and admits ready ones in same lane', async () => { - await enqueue('ws-a', { delayMs: 60_000 }) - await enqueue('ws-a', { delayMs: 0 }) - - const r1 = await dispatchNextAdmissibleWorkspaceJob() - expect(r1).toBe(DISPATCH_SCAN_RESULTS.ADMITTED) - }) - - it('returns delayed when all jobs are delayed', async () => { - await enqueue('ws-a', { delayMs: 60_000 }) - - const r1 = await dispatchNextAdmissibleWorkspaceJob() - expect(r1).toBe(DISPATCH_SCAN_RESULTS.NO_PROGRESS) - }) - - it('returns no_workspace when queue is empty', async () => { - const result = await dispatchNextAdmissibleWorkspaceJob() - expect(result).toBe(DISPATCH_SCAN_RESULTS.NO_WORKSPACE) - }) - - it('lease cleanup frees capacity for new admissions', async () => { - mockGetWorkspaceConcurrencyLimit.mockResolvedValue(1) - - const record = await enqueue('ws-a') - await enqueue('ws-a') - - const r1 = await dispatchNextAdmissibleWorkspaceJob() - expect(r1).toBe(DISPATCH_SCAN_RESULTS.ADMITTED) - - const updated = await store.getDispatchJobRecord(record.id) - if (updated?.lease) { - await store.releaseWorkspaceLease('ws-a', updated.lease.leaseId) - } - - const r2 = await dispatchNextAdmissibleWorkspaceJob() - expect(r2).toBe(DISPATCH_SCAN_RESULTS.ADMITTED) - }) - - it('expired leases are cleaned up during claim', async () => { - mockGetWorkspaceConcurrencyLimit.mockResolvedValue(1) - - await enqueue('ws-a') - await enqueue('ws-a') - - const claimResult = await store.claimWorkspaceJob('ws-a', { - lanes: ['runtime'], - concurrencyLimit: 1, - leaseId: 'old-lease', - now: Date.now(), - leaseTtlMs: 1, - }) - expect(claimResult.type).toBe('admitted') - - await new Promise((resolve) => setTimeout(resolve, 10)) - - const r2 = await dispatchNextAdmissibleWorkspaceJob() - expect(r2).toBe(DISPATCH_SCAN_RESULTS.ADMITTED) - }) - - it('recovers job to waiting via restoreWorkspaceDispatchJob', async () => { - const record = await enqueue('ws-a') - - await store.claimWorkspaceJob('ws-a', { - lanes: ['runtime'], - concurrencyLimit: 1, - leaseId: 'lease-1', - now: Date.now(), - leaseTtlMs: 1000, - }) - - await store.markDispatchJobAdmitted(record.id, 'ws-a', 'lease-1', Date.now() + 10000) - - const admitted = await store.getDispatchJobRecord(record.id) - expect(admitted).toBeDefined() - const resetRecord = { ...admitted!, status: 'waiting' as const, lease: undefined } - await store.restoreWorkspaceDispatchJob(resetRecord) - - const restored = await store.getDispatchJobRecord(record.id) - expect(restored?.status).toBe('waiting') - expect(restored?.lease).toBeUndefined() - }) -}) diff --git a/apps/sim/lib/core/workspace-dispatch/dispatcher.ts b/apps/sim/lib/core/workspace-dispatch/dispatcher.ts deleted file mode 100644 index 1122107ea4..0000000000 --- a/apps/sim/lib/core/workspace-dispatch/dispatcher.ts +++ /dev/null @@ -1,156 +0,0 @@ -import { createLogger } from '@sim/logger' -import { env } from '@/lib/core/config/env' -import { - enqueueWorkspaceDispatchJob, - getDispatchJobRecord, - getGlobalQueueDepth, - getQueuedWorkspaceCount, - getWorkspaceQueueDepth, -} from '@/lib/core/workspace-dispatch/store' -import { - WORKSPACE_DISPATCH_LANES, - type WorkspaceDispatchEnqueueInput, - type WorkspaceDispatchJobRecord, -} from '@/lib/core/workspace-dispatch/types' -import { DISPATCH_SCAN_RESULTS, dispatchNextAdmissibleWorkspaceJob } from './planner' -import { reconcileWorkspaceDispatchState } from './reconciler' - -const logger = createLogger('WorkspaceDispatcher') -const WAIT_POLL_INTERVAL_MS = 250 -const RECONCILE_INTERVAL_MS = 30_000 -const MAX_QUEUE_PER_WORKSPACE = Number.parseInt(env.DISPATCH_MAX_QUEUE_PER_WORKSPACE ?? '') || 1000 -const MAX_QUEUE_GLOBAL = Number.parseInt(env.DISPATCH_MAX_QUEUE_GLOBAL ?? '') || 50_000 - -let dispatcherRunning = false -let dispatcherWakePending = false -let lastReconcileAt = 0 - -async function runDispatcherLoop(): Promise { - if (dispatcherRunning) { - dispatcherWakePending = true - return - } - - dispatcherRunning = true - - try { - const now = Date.now() - if (now - lastReconcileAt >= RECONCILE_INTERVAL_MS) { - await reconcileWorkspaceDispatchState() - lastReconcileAt = now - } - - do { - dispatcherWakePending = false - const queuedWorkspaces = await getQueuedWorkspaceCount() - if (queuedWorkspaces === 0) { - continue - } - - let admitted = 0 - let scanned = 0 - const loopStartMs = Date.now() - - for (let index = 0; index < queuedWorkspaces; index++) { - scanned++ - const result = await dispatchNextAdmissibleWorkspaceJob() - if (result === DISPATCH_SCAN_RESULTS.ADMITTED) { - admitted++ - } - if (result === DISPATCH_SCAN_RESULTS.NO_WORKSPACE) { - break - } - } - - if (admitted > 0) { - dispatcherWakePending = true - } - - if (admitted > 0 || scanned > 0) { - logger.info('Dispatcher pass', { - admitted, - scanned, - queuedWorkspaces, - durationMs: Date.now() - loopStartMs, - }) - } - } while (dispatcherWakePending) - } catch (error) { - logger.error('Workspace dispatcher loop failed', { error }) - } finally { - dispatcherRunning = false - } -} - -export class DispatchQueueFullError extends Error { - readonly statusCode = 503 - - constructor( - readonly scope: 'workspace' | 'global', - readonly depth: number, - readonly limit: number - ) { - super( - scope === 'workspace' - ? `Workspace queue is at capacity (${depth}/${limit})` - : `Global dispatch queue is at capacity (${depth}/${limit})` - ) - this.name = 'DispatchQueueFullError' - } -} - -export async function enqueueWorkspaceDispatch( - input: WorkspaceDispatchEnqueueInput -): Promise { - const [workspaceDepth, globalDepth] = await Promise.all([ - getWorkspaceQueueDepth(input.workspaceId, WORKSPACE_DISPATCH_LANES), - getGlobalQueueDepth(), - ]) - - if (workspaceDepth >= MAX_QUEUE_PER_WORKSPACE) { - logger.warn('Workspace dispatch queue at capacity', { - workspaceId: input.workspaceId, - depth: workspaceDepth, - limit: MAX_QUEUE_PER_WORKSPACE, - }) - throw new DispatchQueueFullError('workspace', workspaceDepth, MAX_QUEUE_PER_WORKSPACE) - } - - if (globalDepth >= MAX_QUEUE_GLOBAL) { - logger.warn('Global dispatch queue at capacity', { - depth: globalDepth, - limit: MAX_QUEUE_GLOBAL, - }) - throw new DispatchQueueFullError('global', globalDepth, MAX_QUEUE_GLOBAL) - } - - const record = await enqueueWorkspaceDispatchJob(input) - void runDispatcherLoop() - return record.id -} - -export async function wakeWorkspaceDispatcher(): Promise { - await runDispatcherLoop() -} - -export async function waitForDispatchJob( - dispatchJobId: string, - timeoutMs: number -): Promise { - const deadline = Date.now() + timeoutMs - - while (Date.now() < deadline) { - const record = await getDispatchJobRecord(dispatchJobId) - if (!record) { - throw new Error(`Dispatch job not found: ${dispatchJobId}`) - } - - if (record.status === 'completed' || record.status === 'failed') { - return record - } - - await new Promise((resolve) => setTimeout(resolve, WAIT_POLL_INTERVAL_MS)) - } - - throw new Error(`Timed out waiting for dispatch job ${dispatchJobId}`) -} diff --git a/apps/sim/lib/core/workspace-dispatch/factory.ts b/apps/sim/lib/core/workspace-dispatch/factory.ts deleted file mode 100644 index 3a07c68cf0..0000000000 --- a/apps/sim/lib/core/workspace-dispatch/factory.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getRedisClient } from '@/lib/core/config/redis' -import type { WorkspaceDispatchStorageAdapter } from '@/lib/core/workspace-dispatch/adapter' -import { MemoryWorkspaceDispatchStorage } from '@/lib/core/workspace-dispatch/memory-store' -import { RedisWorkspaceDispatchStorage } from '@/lib/core/workspace-dispatch/redis-store' - -const logger = createLogger('WorkspaceDispatchFactory') - -let cachedAdapter: WorkspaceDispatchStorageAdapter | null = null - -export function createWorkspaceDispatchStorageAdapter(): WorkspaceDispatchStorageAdapter { - if (cachedAdapter) { - return cachedAdapter - } - - const redis = getRedisClient() - - if (redis) { - logger.info('Workspace dispatcher: Using Redis storage') - const adapter = new RedisWorkspaceDispatchStorage(redis) - cachedAdapter = adapter - return adapter - } - - logger.warn( - 'Workspace dispatcher: Using in-memory storage; distributed fairness is disabled in multi-process deployments' - ) - const adapter = new MemoryWorkspaceDispatchStorage() - cachedAdapter = adapter - return adapter -} - -export function setWorkspaceDispatchStorageAdapter(adapter: WorkspaceDispatchStorageAdapter): void { - cachedAdapter = adapter -} - -export function resetWorkspaceDispatchStorageAdapter(): void { - if (cachedAdapter) { - cachedAdapter.dispose() - cachedAdapter = null - } -} diff --git a/apps/sim/lib/core/workspace-dispatch/index.ts b/apps/sim/lib/core/workspace-dispatch/index.ts deleted file mode 100644 index 74645372c9..0000000000 --- a/apps/sim/lib/core/workspace-dispatch/index.ts +++ /dev/null @@ -1,32 +0,0 @@ -export type { WorkspaceDispatchStorageAdapter } from './adapter' -export { - DispatchQueueFullError, - enqueueWorkspaceDispatch, - waitForDispatchJob, - wakeWorkspaceDispatcher, -} from './dispatcher' -export { - createWorkspaceDispatchStorageAdapter, - resetWorkspaceDispatchStorageAdapter, -} from './factory' -export { - markDispatchJobAdmitted, - markDispatchJobAdmitting, - markDispatchJobCompleted, - markDispatchJobFailed, - markDispatchJobRunning, - refreshWorkspaceLease, - releaseWorkspaceLease, -} from './store' -export { - WORKSPACE_DISPATCH_LANES, - WORKSPACE_DISPATCH_STATUSES, - type WorkspaceDispatchEnqueueInput, - type WorkspaceDispatchJobContext, - type WorkspaceDispatchJobRecord, - type WorkspaceDispatchLane, - type WorkspaceDispatchLeaseInfo, - type WorkspaceDispatchQueueName, - type WorkspaceDispatchStatus, -} from './types' -export { getDispatchRuntimeMetadata, runDispatchedJob } from './worker' diff --git a/apps/sim/lib/core/workspace-dispatch/memory-store.test.ts b/apps/sim/lib/core/workspace-dispatch/memory-store.test.ts deleted file mode 100644 index 87a54de26d..0000000000 --- a/apps/sim/lib/core/workspace-dispatch/memory-store.test.ts +++ /dev/null @@ -1,65 +0,0 @@ -/** - * @vitest-environment node - */ -import { afterEach, describe, expect, it } from 'vitest' -import { MemoryWorkspaceDispatchStorage } from '@/lib/core/workspace-dispatch/memory-store' - -describe('memory workspace dispatch storage', () => { - const store = new MemoryWorkspaceDispatchStorage() - - afterEach(async () => { - await store.clear() - }) - - it('claims a runnable job and marks it admitting with a lease', async () => { - const record = await store.enqueueWorkspaceDispatchJob({ - workspaceId: 'workspace-1', - lane: 'runtime', - queueName: 'workflow-execution', - bullmqJobName: 'workflow-execution', - bullmqPayload: { payload: { workflowId: 'workflow-1' } }, - metadata: { - workflowId: 'workflow-1', - }, - }) - - const result = await store.claimWorkspaceJob('workspace-1', { - lanes: ['runtime'], - concurrencyLimit: 1, - leaseId: 'lease-1', - now: Date.now(), - leaseTtlMs: 1000, - }) - - expect(result.type).toBe('admitted') - if (result.type === 'admitted') { - expect(result.record.id).toBe(record.id) - expect(result.record.status).toBe('admitting') - expect(result.record.lease?.leaseId).toBe('lease-1') - } - }) - - it('returns delayed when only delayed jobs exist', async () => { - await store.enqueueWorkspaceDispatchJob({ - workspaceId: 'workspace-1', - lane: 'runtime', - queueName: 'workflow-execution', - bullmqJobName: 'workflow-execution', - bullmqPayload: { payload: { workflowId: 'workflow-1' } }, - metadata: { - workflowId: 'workflow-1', - }, - delayMs: 5000, - }) - - const result = await store.claimWorkspaceJob('workspace-1', { - lanes: ['runtime'], - concurrencyLimit: 1, - leaseId: 'lease-2', - now: Date.now(), - leaseTtlMs: 1000, - }) - - expect(result.type).toBe('delayed') - }) -}) diff --git a/apps/sim/lib/core/workspace-dispatch/memory-store.ts b/apps/sim/lib/core/workspace-dispatch/memory-store.ts deleted file mode 100644 index 52295172bb..0000000000 --- a/apps/sim/lib/core/workspace-dispatch/memory-store.ts +++ /dev/null @@ -1,506 +0,0 @@ -import { createLogger } from '@sim/logger' -import { generateId } from '@/lib/core/utils/uuid' -import type { WorkspaceDispatchStorageAdapter } from '@/lib/core/workspace-dispatch/adapter' -import { - WORKSPACE_DISPATCH_CLAIM_RESULTS, - type WorkspaceDispatchClaimResult, - type WorkspaceDispatchEnqueueInput, - type WorkspaceDispatchJobRecord, - type WorkspaceDispatchLane, -} from '@/lib/core/workspace-dispatch/types' - -const logger = createLogger('WorkspaceDispatchMemoryStore') -const JOB_TTL_MS = 48 * 60 * 60 * 1000 - -export class MemoryWorkspaceDispatchStorage implements WorkspaceDispatchStorageAdapter { - private jobs = new Map() - private workspaceOrder: string[] = [] - private laneQueues = new Map() - private leases = new Map>() - private cleanupInterval: NodeJS.Timeout | null = null - - constructor() { - this.cleanupInterval = setInterval(() => { - void this.clearExpiredState() - }, 60_000) - this.cleanupInterval.unref() - } - - private queueKey(workspaceId: string, lane: WorkspaceDispatchLane): string { - return `${workspaceId}:${lane}` - } - - private ensureWorkspaceQueued(workspaceId: string): void { - if (!this.workspaceOrder.includes(workspaceId)) { - this.workspaceOrder.push(workspaceId) - } - } - - private getLaneQueue(workspaceId: string, lane: WorkspaceDispatchLane): string[] { - const key = this.queueKey(workspaceId, lane) - const existing = this.laneQueues.get(key) - if (existing) { - return existing - } - - const queue: string[] = [] - this.laneQueues.set(key, queue) - return queue - } - - private sortQueue(queue: string[]): void { - queue.sort((leftId, rightId) => { - const left = this.jobs.get(leftId) - const right = this.jobs.get(rightId) - if (!left || !right) { - return 0 - } - - if (left.priority !== right.priority) { - return left.priority - right.priority - } - - return left.createdAt - right.createdAt - }) - } - - private getLeaseMap(workspaceId: string): Map { - const existing = this.leases.get(workspaceId) - if (existing) { - return existing - } - - const leaseMap = new Map() - this.leases.set(workspaceId, leaseMap) - return leaseMap - } - - private async clearExpiredState(): Promise { - const now = Date.now() - - for (const [jobId, record] of this.jobs.entries()) { - if ( - (record.status === 'completed' || record.status === 'failed') && - record.completedAt && - now - record.completedAt > JOB_TTL_MS - ) { - this.jobs.delete(jobId) - } - } - - for (const [workspaceId, leaseMap] of this.leases.entries()) { - for (const [leaseId, expiresAt] of leaseMap.entries()) { - if (expiresAt <= now) { - leaseMap.delete(leaseId) - } - } - if (leaseMap.size === 0) { - this.leases.delete(workspaceId) - } - } - } - - async saveDispatchJob(record: WorkspaceDispatchJobRecord): Promise { - this.jobs.set(record.id, record) - } - - async getDispatchJobRecord(jobId: string): Promise { - return this.jobs.get(jobId) ?? null - } - - async listDispatchJobsByStatuses( - statuses: readonly WorkspaceDispatchJobRecord['status'][] - ): Promise { - return Array.from(this.jobs.values()).filter((record) => statuses.includes(record.status)) - } - - private static readonly TERMINAL_STATUSES = new Set(['completed', 'failed']) - - async updateDispatchJobRecord( - jobId: string, - updater: (record: WorkspaceDispatchJobRecord) => WorkspaceDispatchJobRecord - ): Promise { - const current = this.jobs.get(jobId) - if (!current) { - return null - } - - const updated = updater(current) - if ( - MemoryWorkspaceDispatchStorage.TERMINAL_STATUSES.has(current.status) && - !MemoryWorkspaceDispatchStorage.TERMINAL_STATUSES.has(updated.status) - ) { - return current - } - - this.jobs.set(jobId, updated) - return updated - } - - async enqueueWorkspaceDispatchJob( - input: WorkspaceDispatchEnqueueInput - ): Promise { - const id = input.id ?? `dispatch_${generateId().replace(/-/g, '').slice(0, 20)}` - const createdAt = Date.now() - - const record: WorkspaceDispatchJobRecord = { - id, - workspaceId: input.workspaceId, - lane: input.lane, - queueName: input.queueName, - bullmqJobName: input.bullmqJobName, - bullmqPayload: input.bullmqPayload, - metadata: input.metadata, - priority: input.priority ?? 100, - maxAttempts: input.maxAttempts, - delayMs: input.delayMs, - status: 'waiting', - createdAt, - } - - this.jobs.set(id, record) - const queue = this.getLaneQueue(record.workspaceId, record.lane) - queue.push(id) - this.sortQueue(queue) - this.ensureWorkspaceQueued(record.workspaceId) - return record - } - - async restoreWorkspaceDispatchJob(record: WorkspaceDispatchJobRecord): Promise { - this.jobs.set(record.id, record) - const queue = this.getLaneQueue(record.workspaceId, record.lane) - if (!queue.includes(record.id)) { - queue.push(record.id) - this.sortQueue(queue) - } - this.ensureWorkspaceQueued(record.workspaceId) - } - - async claimWorkspaceJob( - workspaceId: string, - options: { - lanes: readonly WorkspaceDispatchLane[] - concurrencyLimit: number - leaseId: string - now: number - leaseTtlMs: number - } - ): Promise { - await this.cleanupExpiredWorkspaceLeases(workspaceId) - if (this.getLeaseMap(workspaceId).size >= options.concurrencyLimit) { - this.ensureWorkspaceQueued(workspaceId) - return { type: WORKSPACE_DISPATCH_CLAIM_RESULTS.LIMIT_REACHED } - } - - let selectedRecord: WorkspaceDispatchJobRecord | null = null - let selectedLane: WorkspaceDispatchLane | null = null - let nextReadyAt: number | null = null - - for (const lane of options.lanes) { - const queue = this.getLaneQueue(workspaceId, lane) - for (let scanIndex = 0; scanIndex < queue.length && scanIndex < 20; ) { - const jobId = queue[scanIndex] - const record = this.jobs.get(jobId) - if (!record) { - queue.splice(scanIndex, 1) - continue - } - - const readyAt = record.createdAt + (record.delayMs ?? 0) - if (readyAt <= options.now) { - selectedRecord = record - selectedLane = lane - queue.splice(scanIndex, 1) - break - } - - nextReadyAt = nextReadyAt ? Math.min(nextReadyAt, readyAt) : readyAt - scanIndex++ - } - - if (selectedRecord) { - break - } - } - - if (!selectedRecord || !selectedLane) { - const hasPending = await this.workspaceHasPendingJobs(workspaceId, options.lanes) - if (!hasPending) { - this.workspaceOrder = this.workspaceOrder.filter((value) => value !== workspaceId) - return { type: WORKSPACE_DISPATCH_CLAIM_RESULTS.EMPTY } - } - - this.ensureWorkspaceQueued(workspaceId) - return { - type: WORKSPACE_DISPATCH_CLAIM_RESULTS.DELAYED, - nextReadyAt: nextReadyAt ?? options.now, - } - } - - const leaseExpiresAt = options.now + options.leaseTtlMs - this.getLeaseMap(workspaceId).set(options.leaseId, leaseExpiresAt) - - const updatedRecord: WorkspaceDispatchJobRecord = { - ...selectedRecord, - status: 'admitting', - lease: { - workspaceId, - leaseId: options.leaseId, - }, - metadata: { - ...selectedRecord.metadata, - dispatchLeaseExpiresAt: leaseExpiresAt, - }, - } - this.jobs.set(updatedRecord.id, updatedRecord) - - const hasPending = await this.workspaceHasPendingJobs(workspaceId, options.lanes) - if (hasPending) { - this.ensureWorkspaceQueued(workspaceId) - } else { - this.workspaceOrder = this.workspaceOrder.filter((value) => value !== workspaceId) - } - - return { - type: WORKSPACE_DISPATCH_CLAIM_RESULTS.ADMITTED, - record: updatedRecord, - leaseId: options.leaseId, - leaseExpiresAt, - } - } - - async getWorkspaceQueueDepth( - workspaceId: string, - lanes: readonly WorkspaceDispatchLane[] - ): Promise { - let depth = 0 - for (const lane of lanes) { - depth += this.getLaneQueue(workspaceId, lane).length - } - return depth - } - - async getGlobalQueueDepth(): Promise { - const terminalStatuses = new Set(['completed', 'failed']) - let count = 0 - for (const job of this.jobs.values()) { - if (!terminalStatuses.has(job.status)) { - count++ - } - } - return count - } - - async reconcileGlobalQueueDepth(_knownCount: number): Promise { - // no-op: memory store computes depth on the fly - } - - async popNextWorkspaceId(): Promise { - const now = Date.now() - const maxScans = this.workspaceOrder.length - for (let i = 0; i < maxScans; i++) { - const id = this.workspaceOrder.shift() - if (!id) return null - const readyAt = this.workspaceReadyAt.get(id) - if (readyAt && readyAt > now) { - this.workspaceOrder.push(id) - continue - } - this.workspaceReadyAt.delete(id) - return id - } - return null - } - - async getQueuedWorkspaceCount(): Promise { - return this.workspaceOrder.length - } - - async hasActiveWorkspace(workspaceId: string): Promise { - return this.workspaceOrder.includes(workspaceId) - } - - private workspaceReadyAt = new Map() - - async ensureWorkspaceActive(workspaceId: string, readyAt?: number): Promise { - if (readyAt && readyAt > Date.now()) { - this.workspaceReadyAt.set(workspaceId, readyAt) - } - this.ensureWorkspaceQueued(workspaceId) - } - - async requeueWorkspaceId(workspaceId: string): Promise { - this.ensureWorkspaceQueued(workspaceId) - } - - async workspaceHasPendingJobs( - workspaceId: string, - lanes: readonly WorkspaceDispatchLane[] - ): Promise { - return lanes.some((lane) => this.getLaneQueue(workspaceId, lane).length > 0) - } - - async getNextWorkspaceJob( - workspaceId: string, - lanes: readonly WorkspaceDispatchLane[] - ): Promise { - for (const lane of lanes) { - const queue = this.getLaneQueue(workspaceId, lane) - while (queue.length > 0) { - const jobId = queue[0] - const job = this.jobs.get(jobId) - if (job) { - return job - } - queue.shift() - } - } - - return null - } - - async removeWorkspaceJobFromLane( - workspaceId: string, - lane: WorkspaceDispatchLane, - jobId: string - ): Promise { - const queue = this.getLaneQueue(workspaceId, lane) - const index = queue.indexOf(jobId) - if (index >= 0) { - queue.splice(index, 1) - } - } - - async cleanupExpiredWorkspaceLeases(workspaceId: string): Promise { - const leaseMap = this.getLeaseMap(workspaceId) - const now = Date.now() - for (const [leaseId, expiresAt] of leaseMap.entries()) { - if (expiresAt <= now) { - leaseMap.delete(leaseId) - } - } - } - - async countActiveWorkspaceLeases(workspaceId: string): Promise { - await this.cleanupExpiredWorkspaceLeases(workspaceId) - return this.getLeaseMap(workspaceId).size - } - - async hasWorkspaceLease(workspaceId: string, leaseId: string): Promise { - await this.cleanupExpiredWorkspaceLeases(workspaceId) - return this.getLeaseMap(workspaceId).has(leaseId) - } - - async createWorkspaceLease(workspaceId: string, leaseId: string, ttlMs: number): Promise { - const expiresAt = Date.now() + ttlMs - this.getLeaseMap(workspaceId).set(leaseId, expiresAt) - return expiresAt - } - - async refreshWorkspaceLease( - workspaceId: string, - leaseId: string, - ttlMs: number - ): Promise { - return this.createWorkspaceLease(workspaceId, leaseId, ttlMs) - } - - async releaseWorkspaceLease(workspaceId: string, leaseId: string): Promise { - this.getLeaseMap(workspaceId).delete(leaseId) - } - - async removeWorkspaceIfIdle( - workspaceId: string, - lanes: readonly WorkspaceDispatchLane[] - ): Promise { - const hasPending = await this.workspaceHasPendingJobs(workspaceId, lanes) - if (!hasPending) { - this.workspaceOrder = this.workspaceOrder.filter((value) => value !== workspaceId) - } - } - - async markDispatchJobAdmitted( - jobId: string, - workspaceId: string, - leaseId: string, - leaseExpiresAt: number - ): Promise { - await this.updateDispatchJobRecord(jobId, (record) => ({ - ...record, - status: 'admitted', - admittedAt: Date.now(), - lease: { - workspaceId, - leaseId, - }, - metadata: { - ...record.metadata, - dispatchLeaseExpiresAt: leaseExpiresAt, - }, - })) - } - - async markDispatchJobAdmitting( - jobId: string, - workspaceId: string, - leaseId: string, - leaseExpiresAt: number - ): Promise { - await this.updateDispatchJobRecord(jobId, (record) => ({ - ...record, - status: 'admitting', - lease: { - workspaceId, - leaseId, - }, - metadata: { - ...record.metadata, - dispatchLeaseExpiresAt: leaseExpiresAt, - }, - })) - } - - async markDispatchJobRunning(jobId: string): Promise { - await this.updateDispatchJobRecord(jobId, (record) => ({ - ...record, - status: 'running', - startedAt: record.startedAt ?? Date.now(), - })) - } - - async markDispatchJobCompleted(jobId: string, output: unknown): Promise { - await this.updateDispatchJobRecord(jobId, (record) => ({ - ...record, - status: 'completed', - completedAt: Date.now(), - output, - })) - } - - async markDispatchJobFailed(jobId: string, error: string): Promise { - await this.updateDispatchJobRecord(jobId, (record) => ({ - ...record, - status: 'failed', - completedAt: Date.now(), - error, - })) - } - - async clear(): Promise { - this.jobs.clear() - this.workspaceOrder = [] - this.laneQueues.clear() - this.leases.clear() - this.workspaceReadyAt.clear() - } - - dispose(): void { - if (this.cleanupInterval) { - clearInterval(this.cleanupInterval) - this.cleanupInterval = null - } - void this.clear().catch((error) => { - logger.error('Failed to clear memory workspace dispatch storage', { error }) - }) - } -} diff --git a/apps/sim/lib/core/workspace-dispatch/planner.ts b/apps/sim/lib/core/workspace-dispatch/planner.ts deleted file mode 100644 index 9912b22aba..0000000000 --- a/apps/sim/lib/core/workspace-dispatch/planner.ts +++ /dev/null @@ -1,155 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getWorkspaceConcurrencyLimit } from '@/lib/billing/workspace-concurrency' -import { type BullMQJobData, getBullMQQueueByName } from '@/lib/core/bullmq' -import { acquireLock, releaseLock } from '@/lib/core/config/redis' -import { generateId } from '@/lib/core/utils/uuid' -import { - claimWorkspaceJob, - markDispatchJobAdmitted, - popNextWorkspaceId, - releaseWorkspaceLease, - removeWorkspaceIfIdle, - requeueWorkspaceId, -} from '@/lib/core/workspace-dispatch/store' -import { - WORKSPACE_DISPATCH_CLAIM_RESULTS, - WORKSPACE_DISPATCH_LANES, - type WorkspaceDispatchJobRecord, -} from '@/lib/core/workspace-dispatch/types' - -const logger = createLogger('WorkspaceDispatchPlanner') - -const LEASE_TTL_MS = 15 * 60 * 1000 -const WORKSPACE_CLAIM_LOCK_TTL_SECONDS = 10 - -export const DISPATCH_SCAN_RESULTS = { - NO_WORKSPACE: 'no_workspace', - NO_PROGRESS: 'no_progress', - ADMITTED: 'admitted', -} as const - -export type DispatchScanResult = (typeof DISPATCH_SCAN_RESULTS)[keyof typeof DISPATCH_SCAN_RESULTS] - -function attachDispatchMetadata( - bullmqPayload: unknown, - record: WorkspaceDispatchJobRecord, - leaseId: string, - leaseExpiresAt: number -): BullMQJobData { - if ( - bullmqPayload && - typeof bullmqPayload === 'object' && - 'payload' in bullmqPayload && - 'metadata' in bullmqPayload - ) { - const data = bullmqPayload as BullMQJobData - return { - payload: data.payload, - metadata: { - ...(data.metadata ?? {}), - dispatchJobId: record.id, - dispatchWorkspaceId: record.workspaceId, - dispatchLeaseId: leaseId, - dispatchLeaseExpiresAt: leaseExpiresAt, - }, - } - } - - return { - payload: bullmqPayload, - metadata: { - ...record.metadata, - dispatchJobId: record.id, - dispatchWorkspaceId: record.workspaceId, - dispatchLeaseId: leaseId, - dispatchLeaseExpiresAt: leaseExpiresAt, - }, - } -} - -async function finalizeAdmittedJob( - record: WorkspaceDispatchJobRecord, - leaseId: string, - leaseExpiresAt: number -): Promise { - try { - await getBullMQQueueByName(record.queueName).add( - record.bullmqJobName, - attachDispatchMetadata(record.bullmqPayload, record, leaseId, leaseExpiresAt), - { - jobId: record.id, - attempts: record.maxAttempts, - priority: record.priority, - } - ) - - await markDispatchJobAdmitted(record.id, record.workspaceId, leaseId, leaseExpiresAt) - } catch (error) { - await releaseWorkspaceLease(record.workspaceId, leaseId).catch(() => undefined) - throw error - } -} - -export async function dispatchNextAdmissibleWorkspaceJob(): Promise { - const workspaceId = await popNextWorkspaceId() - if (!workspaceId) { - return DISPATCH_SCAN_RESULTS.NO_WORKSPACE - } - - const lockValue = `lock_${generateId()}` - try { - const lockKey = `workspace-dispatch:claim-lock:${workspaceId}` - const acquired = await acquireLock(lockKey, lockValue, WORKSPACE_CLAIM_LOCK_TTL_SECONDS) - if (!acquired) { - await requeueWorkspaceId(workspaceId) - return DISPATCH_SCAN_RESULTS.NO_PROGRESS - } - - const limit = await getWorkspaceConcurrencyLimit(workspaceId) - const leaseId = `lease_${generateId()}` - const claimResult = await claimWorkspaceJob(workspaceId, { - lanes: WORKSPACE_DISPATCH_LANES, - concurrencyLimit: limit, - leaseId, - now: Date.now(), - leaseTtlMs: LEASE_TTL_MS, - }) - - switch (claimResult.type) { - case WORKSPACE_DISPATCH_CLAIM_RESULTS.LIMIT_REACHED: - logger.debug('Workspace concurrency limit reached', { workspaceId, limit }) - await requeueWorkspaceId(workspaceId) - return DISPATCH_SCAN_RESULTS.NO_PROGRESS - case WORKSPACE_DISPATCH_CLAIM_RESULTS.DELAYED: - logger.debug('Workspace has only delayed jobs', { - workspaceId, - nextReadyAt: claimResult.nextReadyAt, - }) - return DISPATCH_SCAN_RESULTS.NO_PROGRESS - case WORKSPACE_DISPATCH_CLAIM_RESULTS.EMPTY: - await removeWorkspaceIfIdle(workspaceId, WORKSPACE_DISPATCH_LANES) - return DISPATCH_SCAN_RESULTS.NO_PROGRESS - case WORKSPACE_DISPATCH_CLAIM_RESULTS.ADMITTED: - logger.info('Admitting workspace job', { - workspaceId, - dispatchJobId: claimResult.record.id, - lane: claimResult.record.lane, - queueName: claimResult.record.queueName, - }) - await finalizeAdmittedJob( - claimResult.record, - claimResult.leaseId, - claimResult.leaseExpiresAt - ) - return DISPATCH_SCAN_RESULTS.ADMITTED - } - } catch (error) { - logger.error('Failed to dispatch workspace job', { workspaceId, error }) - await requeueWorkspaceId(workspaceId) - return DISPATCH_SCAN_RESULTS.NO_PROGRESS - } finally { - await releaseLock(`workspace-dispatch:claim-lock:${workspaceId}`, lockValue).catch( - () => undefined - ) - } -} diff --git a/apps/sim/lib/core/workspace-dispatch/reconciler.test.ts b/apps/sim/lib/core/workspace-dispatch/reconciler.test.ts deleted file mode 100644 index a61d0dc4d1..0000000000 --- a/apps/sim/lib/core/workspace-dispatch/reconciler.test.ts +++ /dev/null @@ -1,225 +0,0 @@ -/** - * @vitest-environment node - */ -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { - mockGetBullMQQueueByName, - mockHasActiveWorkspace, - mockEnsureWorkspaceActive, - mockHasWorkspaceLease, - mockListDispatchJobsByStatuses, - mockMarkDispatchJobAdmitted, - mockMarkDispatchJobCompleted, - mockMarkDispatchJobFailed, - mockRefreshWorkspaceLease, - mockReleaseWorkspaceLease, - mockRemoveWorkspaceJobFromLane, - mockRestoreWorkspaceDispatchJob, - mockWakeWorkspaceDispatcher, -} = vi.hoisted(() => ({ - mockGetBullMQQueueByName: vi.fn(), - mockHasActiveWorkspace: vi.fn(), - mockEnsureWorkspaceActive: vi.fn(), - mockHasWorkspaceLease: vi.fn(), - mockListDispatchJobsByStatuses: vi.fn(), - mockMarkDispatchJobAdmitted: vi.fn(), - mockMarkDispatchJobCompleted: vi.fn(), - mockMarkDispatchJobFailed: vi.fn(), - mockRefreshWorkspaceLease: vi.fn(), - mockReleaseWorkspaceLease: vi.fn(), - mockRemoveWorkspaceJobFromLane: vi.fn(), - mockRestoreWorkspaceDispatchJob: vi.fn(), - mockWakeWorkspaceDispatcher: vi.fn(), -})) - -vi.mock('@sim/logger', () => ({ - createLogger: () => ({ - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - debug: vi.fn(), - }), -})) - -vi.mock('@/lib/core/bullmq', () => ({ - getBullMQQueueByName: mockGetBullMQQueueByName, -})) - -vi.mock('@/lib/core/workspace-dispatch/store', () => ({ - ensureWorkspaceActive: mockEnsureWorkspaceActive, - hasActiveWorkspace: mockHasActiveWorkspace, - hasWorkspaceLease: mockHasWorkspaceLease, - listDispatchJobsByStatuses: mockListDispatchJobsByStatuses, - markDispatchJobAdmitted: mockMarkDispatchJobAdmitted, - markDispatchJobCompleted: mockMarkDispatchJobCompleted, - markDispatchJobFailed: mockMarkDispatchJobFailed, - reconcileGlobalQueueDepth: vi.fn().mockResolvedValue(undefined), - refreshWorkspaceLease: mockRefreshWorkspaceLease, - releaseWorkspaceLease: mockReleaseWorkspaceLease, - removeWorkspaceJobFromLane: mockRemoveWorkspaceJobFromLane, - restoreWorkspaceDispatchJob: mockRestoreWorkspaceDispatchJob, -})) - -vi.mock('@/lib/core/workspace-dispatch/dispatcher', () => ({ - wakeWorkspaceDispatcher: mockWakeWorkspaceDispatcher, -})) - -import { reconcileWorkspaceDispatchState } from '@/lib/core/workspace-dispatch/reconciler' - -describe('workspace dispatch reconciler', () => { - beforeEach(() => { - vi.clearAllMocks() - mockHasActiveWorkspace.mockResolvedValue(true) - mockRemoveWorkspaceJobFromLane.mockResolvedValue(undefined) - }) - - it('marks dispatch job completed when BullMQ job is completed', async () => { - mockListDispatchJobsByStatuses.mockResolvedValue([ - { - id: 'dispatch-1', - workspaceId: 'workspace-1', - lane: 'runtime', - queueName: 'workflow-execution', - bullmqJobName: 'workflow-execution', - bullmqPayload: {}, - metadata: {}, - priority: 10, - status: 'running', - createdAt: 1, - lease: { - workspaceId: 'workspace-1', - leaseId: 'lease-1', - }, - }, - ]) - mockGetBullMQQueueByName.mockReturnValue({ - getJob: vi.fn().mockResolvedValue({ - getState: vi.fn().mockResolvedValue('completed'), - returnvalue: { ok: true }, - }), - }) - - await reconcileWorkspaceDispatchState() - - expect(mockMarkDispatchJobCompleted).toHaveBeenCalledWith('dispatch-1', { ok: true }) - expect(mockReleaseWorkspaceLease).toHaveBeenCalledWith('workspace-1', 'lease-1') - expect(mockWakeWorkspaceDispatcher).toHaveBeenCalled() - }) - - it('restores admitted jobs to waiting when lease and BullMQ job are gone', async () => { - mockListDispatchJobsByStatuses.mockResolvedValue([ - { - id: 'dispatch-2', - workspaceId: 'workspace-2', - lane: 'runtime', - queueName: 'workflow-execution', - bullmqJobName: 'workflow-execution', - bullmqPayload: {}, - metadata: {}, - priority: 10, - status: 'admitted', - createdAt: 1, - admittedAt: 2, - lease: { - workspaceId: 'workspace-2', - leaseId: 'lease-2', - }, - }, - ]) - mockGetBullMQQueueByName.mockReturnValue({ - getJob: vi.fn().mockResolvedValue(null), - }) - mockHasWorkspaceLease.mockResolvedValue(false) - - await reconcileWorkspaceDispatchState() - - expect(mockRestoreWorkspaceDispatchJob).toHaveBeenCalledWith( - expect.objectContaining({ - id: 'dispatch-2', - status: 'waiting', - lease: undefined, - }) - ) - expect(mockWakeWorkspaceDispatcher).toHaveBeenCalled() - }) - - it('reacquires the lease for a live admitting BullMQ job', async () => { - mockListDispatchJobsByStatuses.mockResolvedValue([ - { - id: 'dispatch-3', - workspaceId: 'workspace-3', - lane: 'runtime', - queueName: 'workflow-execution', - bullmqJobName: 'workflow-execution', - bullmqPayload: {}, - metadata: { - dispatchLeaseExpiresAt: 12345, - }, - priority: 10, - status: 'admitting', - createdAt: 1, - lease: { - workspaceId: 'workspace-3', - leaseId: 'lease-3', - }, - }, - ]) - mockGetBullMQQueueByName.mockReturnValue({ - getJob: vi.fn().mockResolvedValue({ - getState: vi.fn().mockResolvedValue('active'), - }), - }) - mockHasWorkspaceLease.mockResolvedValue(false) - - await reconcileWorkspaceDispatchState() - - expect(mockRefreshWorkspaceLease).toHaveBeenCalledWith('workspace-3', 'lease-3', 15 * 60 * 1000) - expect(mockMarkDispatchJobAdmitted).toHaveBeenCalledWith( - 'dispatch-3', - 'workspace-3', - 'lease-3', - 12345 - ) - expect(mockRemoveWorkspaceJobFromLane).toHaveBeenCalledWith( - 'workspace-3', - 'runtime', - 'dispatch-3' - ) - }) - - it('releases leaked lease and restores waiting when BullMQ job is gone but lease remains', async () => { - mockListDispatchJobsByStatuses.mockResolvedValue([ - { - id: 'dispatch-4', - workspaceId: 'workspace-4', - lane: 'runtime', - queueName: 'workflow-execution', - bullmqJobName: 'workflow-execution', - bullmqPayload: {}, - metadata: {}, - priority: 10, - status: 'running', - createdAt: 1, - lease: { - workspaceId: 'workspace-4', - leaseId: 'lease-4', - }, - }, - ]) - mockGetBullMQQueueByName.mockReturnValue({ - getJob: vi.fn().mockResolvedValue(null), - }) - mockHasWorkspaceLease.mockResolvedValue(true) - - await reconcileWorkspaceDispatchState() - - expect(mockReleaseWorkspaceLease).toHaveBeenCalledWith('workspace-4', 'lease-4') - expect(mockRestoreWorkspaceDispatchJob).toHaveBeenCalledWith( - expect.objectContaining({ - id: 'dispatch-4', - status: 'waiting', - }) - ) - }) -}) diff --git a/apps/sim/lib/core/workspace-dispatch/reconciler.ts b/apps/sim/lib/core/workspace-dispatch/reconciler.ts deleted file mode 100644 index 739d75533b..0000000000 --- a/apps/sim/lib/core/workspace-dispatch/reconciler.ts +++ /dev/null @@ -1,226 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getBullMQQueueByName } from '@/lib/core/bullmq' -import { - ensureWorkspaceActive, - hasActiveWorkspace, - hasWorkspaceLease, - listDispatchJobsByStatuses, - markDispatchJobAdmitted, - markDispatchJobCompleted, - markDispatchJobFailed, - markDispatchJobRunning, - reconcileGlobalQueueDepth, - refreshWorkspaceLease, - releaseWorkspaceLease, - removeWorkspaceJobFromLane, - restoreWorkspaceDispatchJob, -} from '@/lib/core/workspace-dispatch/store' -import type { WorkspaceDispatchJobRecord } from '@/lib/core/workspace-dispatch/types' -import { wakeWorkspaceDispatcher } from './dispatcher' - -const logger = createLogger('WorkspaceDispatchReconciler') -const LEASE_TTL_MS = 15 * 60 * 1000 - -function resetToWaiting(record: WorkspaceDispatchJobRecord): WorkspaceDispatchJobRecord { - return { - ...record, - status: 'waiting', - admittedAt: undefined, - startedAt: undefined, - completedAt: undefined, - output: undefined, - error: undefined, - lease: undefined, - } -} - -async function reconcileTerminalBullMQState(record: WorkspaceDispatchJobRecord): Promise { - const queue = getBullMQQueueByName(record.queueName) - const job = await queue.getJob(record.id) - if (!job) { - return false - } - - const state = await job.getState() - if (state === 'completed') { - await markDispatchJobCompleted(record.id, job.returnvalue) - if (record.lease) { - await releaseWorkspaceLease(record.lease.workspaceId, record.lease.leaseId) - } - return true - } - - if (state === 'failed' && job.attemptsMade >= (job.opts.attempts ?? 1)) { - await markDispatchJobFailed(record.id, job.failedReason || 'Job failed') - if (record.lease) { - await releaseWorkspaceLease(record.lease.workspaceId, record.lease.leaseId) - } - return true - } - - return false -} - -async function reconcileStrandedDispatchJob(record: WorkspaceDispatchJobRecord): Promise { - if (!record.lease && record.status !== 'waiting') { - await restoreWorkspaceDispatchJob(resetToWaiting(record)) - return true - } - - if (!record.lease) { - return false - } - - const hasLease = await hasWorkspaceLease(record.lease.workspaceId, record.lease.leaseId) - const queue = getBullMQQueueByName(record.queueName) - const job = await queue.getJob(record.id) - if (hasLease) { - if (!job) { - await releaseWorkspaceLease(record.lease.workspaceId, record.lease.leaseId) - await restoreWorkspaceDispatchJob(resetToWaiting(record)) - return true - } - - return false - } - - if (job) { - if (record.status === 'admitting') { - await refreshWorkspaceLease(record.lease.workspaceId, record.lease.leaseId, LEASE_TTL_MS) - await markDispatchJobAdmitted( - record.id, - record.lease.workspaceId, - record.lease.leaseId, - (record.metadata as { dispatchLeaseExpiresAt?: number }).dispatchLeaseExpiresAt ?? - Date.now() - ) - await removeWorkspaceJobFromLane(record.workspaceId, record.lane, record.id).catch( - () => undefined - ) - return true - } - await refreshWorkspaceLease(record.lease.workspaceId, record.lease.leaseId, LEASE_TTL_MS) - if (record.status === 'admitted') { - await markDispatchJobRunning(record.id) - return true - } - return false - } - - await restoreWorkspaceDispatchJob(resetToWaiting(record)) - return true -} - -async function reconcileTerminalDispatchLease( - record: WorkspaceDispatchJobRecord -): Promise { - if ((record.status !== 'completed' && record.status !== 'failed') || !record.lease) { - return false - } - - const hasLease = await hasWorkspaceLease(record.lease.workspaceId, record.lease.leaseId) - if (!hasLease) { - return false - } - - await releaseWorkspaceLease(record.lease.workspaceId, record.lease.leaseId) - return true -} - -async function reconcileWaitingWorkspaceTracking( - waitingJobs: WorkspaceDispatchJobRecord[] -): Promise { - let changed = false - const earliestByWorkspace = new Map() - - for (const record of waitingJobs) { - const readyAt = record.createdAt + (record.delayMs ?? 0) - const current = earliestByWorkspace.get(record.workspaceId) - if (current === undefined || readyAt < current) { - earliestByWorkspace.set(record.workspaceId, readyAt) - } - } - - for (const [workspaceId, nextReadyAt] of earliestByWorkspace.entries()) { - const active = await hasActiveWorkspace(workspaceId) - if (!active) { - await ensureWorkspaceActive(workspaceId, nextReadyAt) - changed = true - } - } - - return changed -} - -export async function reconcileWorkspaceDispatchState(): Promise { - const allJobs = await listDispatchJobsByStatuses([ - 'waiting', - 'admitting', - 'admitted', - 'running', - 'completed', - 'failed', - ]) - - const activeJobs: WorkspaceDispatchJobRecord[] = [] - const waitingJobs: WorkspaceDispatchJobRecord[] = [] - const terminalJobs: WorkspaceDispatchJobRecord[] = [] - let nonTerminalCount = 0 - - for (const job of allJobs) { - switch (job.status) { - case 'admitting': - case 'admitted': - case 'running': - activeJobs.push(job) - nonTerminalCount++ - break - case 'waiting': - waitingJobs.push(job) - nonTerminalCount++ - break - case 'completed': - case 'failed': - terminalJobs.push(job) - break - } - } - - let changed = false - - for (const record of activeJobs) { - const terminal = await reconcileTerminalBullMQState(record) - if (terminal) { - changed = true - continue - } - - const restored = await reconcileStrandedDispatchJob(record) - if (restored) { - changed = true - } - } - - if (await reconcileWaitingWorkspaceTracking(waitingJobs)) { - changed = true - } - - for (const record of terminalJobs) { - if (await reconcileTerminalDispatchLease(record)) { - changed = true - } - } - - await reconcileGlobalQueueDepth(nonTerminalCount).catch((error) => { - logger.error('Failed to reconcile global queue depth', { error }) - }) - - if (changed) { - logger.info('Workspace dispatch reconciliation updated state', { - activeJobsInspected: activeJobs.length, - waitingJobsInspected: waitingJobs.length, - terminalJobsInspected: terminalJobs.length, - }) - await wakeWorkspaceDispatcher() - } -} diff --git a/apps/sim/lib/core/workspace-dispatch/redis-store.ts b/apps/sim/lib/core/workspace-dispatch/redis-store.ts deleted file mode 100644 index e7ce5a8769..0000000000 --- a/apps/sim/lib/core/workspace-dispatch/redis-store.ts +++ /dev/null @@ -1,578 +0,0 @@ -import { createLogger } from '@sim/logger' -import type Redis from 'ioredis' -import { generateId } from '@/lib/core/utils/uuid' -import type { WorkspaceDispatchStorageAdapter } from '@/lib/core/workspace-dispatch/adapter' -import { - WORKSPACE_DISPATCH_CLAIM_RESULTS, - type WorkspaceDispatchClaimResult, - type WorkspaceDispatchEnqueueInput, - type WorkspaceDispatchJobRecord, - type WorkspaceDispatchLane, -} from '@/lib/core/workspace-dispatch/types' - -const logger = createLogger('WorkspaceDispatchRedisStore') - -const DISPATCH_PREFIX = 'workspace-dispatch:v1' -const JOB_TTL_SECONDS = 48 * 60 * 60 -const SEQUENCE_KEY = `${DISPATCH_PREFIX}:sequence` -const ACTIVE_WORKSPACES_KEY = `${DISPATCH_PREFIX}:workspaces` -const GLOBAL_DEPTH_KEY = `${DISPATCH_PREFIX}:global-depth` -const CLAIM_JOB_SCRIPT = ` -local workspaceId = ARGV[1] -local now = tonumber(ARGV[2]) -local concurrencyLimit = tonumber(ARGV[3]) -local leaseId = ARGV[4] -local leaseExpiresAt = tonumber(ARGV[5]) -local lanes = cjson.decode(ARGV[6]) -local sequenceKey = ARGV[7] -local activeWorkspacesKey = ARGV[8] -local jobPrefix = ARGV[9] -local workspacePrefix = ARGV[10] -local jobTtlSeconds = tonumber(ARGV[11]) - -local function laneKey(lane) - return workspacePrefix .. workspaceId .. ':lane:' .. lane -end - -local function leaseKey() - return workspacePrefix .. workspaceId .. ':leases' -end - -local function workspaceHasPending() - local minReadyAt = nil - local hasPending = false - - for _, lane in ipairs(lanes) do - local ids = redis.call('ZRANGE', laneKey(lane), 0, 0) - if #ids > 0 then - local raw = redis.call('GET', jobPrefix .. ids[1]) - if raw then - hasPending = true - local record = cjson.decode(raw) - local readyAt = (record.createdAt or 0) + (record.delayMs or 0) - if (minReadyAt == nil) or (readyAt < minReadyAt) then - minReadyAt = readyAt - end - else - redis.call('ZREM', laneKey(lane), ids[1]) - end - end - end - - return hasPending, minReadyAt -end - -redis.call('ZREMRANGEBYSCORE', leaseKey(), 0, now) -local activeLeaseCount = redis.call('ZCARD', leaseKey()) -if activeLeaseCount >= concurrencyLimit then - return cjson.encode({ type = 'limit_reached' }) -end - -local selectedId = nil -local selectedLane = nil -local selectedRecord = nil -local delayedNextReadyAt = nil - -local maxScanPerLane = 20 - -for _, lane in ipairs(lanes) do - local ids = redis.call('ZRANGE', laneKey(lane), 0, maxScanPerLane - 1) - for _, candidateId in ipairs(ids) do - local raw = redis.call('GET', jobPrefix .. candidateId) - if raw then - local record = cjson.decode(raw) - local readyAt = (record.createdAt or 0) + (record.delayMs or 0) - if readyAt <= now then - selectedId = candidateId - selectedLane = lane - selectedRecord = record - break - end - - if (delayedNextReadyAt == nil) or (readyAt < delayedNextReadyAt) then - delayedNextReadyAt = readyAt - end - else - redis.call('ZREM', laneKey(lane), candidateId) - end - end - - if selectedRecord then - break - end -end - -if selectedRecord == nil then - local hasPending, minReadyAt = workspaceHasPending() - if not hasPending then - return cjson.encode({ type = 'empty' }) - end - - local sequence = redis.call('INCR', sequenceKey) - local score = sequence - if minReadyAt ~= nil and minReadyAt > now then - score = minReadyAt * 1000000 + sequence - end - redis.call('ZADD', activeWorkspacesKey, score, workspaceId) - - return cjson.encode({ - type = 'delayed', - nextReadyAt = delayedNextReadyAt or minReadyAt or now - }) -end - -redis.call('ZADD', leaseKey(), leaseExpiresAt, leaseId) -selectedRecord.status = 'admitting' -selectedRecord.lease = { - workspaceId = workspaceId, - leaseId = leaseId -} -if selectedRecord.metadata == nil then - selectedRecord.metadata = {} -end -selectedRecord.metadata.dispatchLeaseExpiresAt = leaseExpiresAt - -redis.call('SET', jobPrefix .. selectedId, cjson.encode(selectedRecord), 'EX', jobTtlSeconds) -redis.call('ZREM', laneKey(selectedLane), selectedId) - -local hasPending, minReadyAt = workspaceHasPending() -if hasPending then - local sequence = redis.call('INCR', sequenceKey) - local score = sequence - if minReadyAt ~= nil and minReadyAt > now then - score = minReadyAt * 1000000 + sequence - end - redis.call('ZADD', activeWorkspacesKey, score, workspaceId) -end - -return cjson.encode({ - type = 'admitted', - record = selectedRecord, - leaseId = leaseId, - leaseExpiresAt = leaseExpiresAt -}) -` - -function jobKey(jobId: string): string { - return `${DISPATCH_PREFIX}:job:${jobId}` -} - -function workspaceLaneKey(workspaceId: string, lane: WorkspaceDispatchLane): string { - return `${DISPATCH_PREFIX}:workspace:${workspaceId}:lane:${lane}` -} - -function workspaceLeaseKey(workspaceId: string): string { - return `${DISPATCH_PREFIX}:workspace:${workspaceId}:leases` -} - -function createPriorityScore(priority: number, sequence: number): number { - return priority * 1_000_000_000_000 + sequence -} - -export class RedisWorkspaceDispatchStorage implements WorkspaceDispatchStorageAdapter { - constructor(private redis: Redis) {} - - private async nextSequence(): Promise { - return this.redis.incr(SEQUENCE_KEY) - } - - async saveDispatchJob(record: WorkspaceDispatchJobRecord): Promise { - await this.redis.set(jobKey(record.id), JSON.stringify(record), 'EX', JOB_TTL_SECONDS) - } - - async getDispatchJobRecord(jobId: string): Promise { - const raw = await this.redis.get(jobKey(jobId)) - if (!raw) { - return null - } - - try { - return JSON.parse(raw) as WorkspaceDispatchJobRecord - } catch (error) { - logger.warn('Corrupted dispatch job record, deleting', { jobId, error }) - await this.redis.del(jobKey(jobId)) - return null - } - } - - async listDispatchJobsByStatuses( - statuses: readonly WorkspaceDispatchJobRecord['status'][] - ): Promise { - let cursor = '0' - const jobs: WorkspaceDispatchJobRecord[] = [] - - do { - const [nextCursor, keys] = await this.redis.scan( - cursor, - 'MATCH', - `${DISPATCH_PREFIX}:job:*`, - 'COUNT', - 100 - ) - cursor = nextCursor - - if (keys.length === 0) { - continue - } - - const values = await this.redis.mget(...keys) - for (const value of values) { - if (!value) { - continue - } - try { - const record = JSON.parse(value) as WorkspaceDispatchJobRecord - if (statuses.includes(record.status)) { - jobs.push(record) - } - } catch { - // Best effort during reconciliation scans. - } - } - } while (cursor !== '0') - - return jobs - } - - private static readonly TERMINAL_STATUSES = new Set(['completed', 'failed']) - - async updateDispatchJobRecord( - jobId: string, - updater: (record: WorkspaceDispatchJobRecord) => WorkspaceDispatchJobRecord - ): Promise { - const current = await this.getDispatchJobRecord(jobId) - if (!current) { - return null - } - - const updated = updater(current) - if ( - RedisWorkspaceDispatchStorage.TERMINAL_STATUSES.has(current.status) && - !RedisWorkspaceDispatchStorage.TERMINAL_STATUSES.has(updated.status) - ) { - return current - } - - await this.saveDispatchJob(updated) - return updated - } - - async enqueueWorkspaceDispatchJob( - input: WorkspaceDispatchEnqueueInput - ): Promise { - const id = input.id ?? `dispatch_${generateId().replace(/-/g, '').slice(0, 20)}` - const createdAt = Date.now() - const sequence = await this.nextSequence() - - const record: WorkspaceDispatchJobRecord = { - id, - workspaceId: input.workspaceId, - lane: input.lane, - queueName: input.queueName, - bullmqJobName: input.bullmqJobName, - bullmqPayload: input.bullmqPayload, - metadata: input.metadata, - priority: input.priority ?? 100, - maxAttempts: input.maxAttempts, - delayMs: input.delayMs, - status: 'waiting', - createdAt, - } - - const score = createPriorityScore(record.priority, sequence) - const pipeline = this.redis.pipeline() - pipeline.set(jobKey(id), JSON.stringify(record), 'EX', JOB_TTL_SECONDS) - pipeline.zadd(workspaceLaneKey(record.workspaceId, record.lane), score, id) - pipeline.zadd(ACTIVE_WORKSPACES_KEY, 'NX', sequence, record.workspaceId) - pipeline.incr(GLOBAL_DEPTH_KEY) - await pipeline.exec() - - return record - } - - async restoreWorkspaceDispatchJob(record: WorkspaceDispatchJobRecord): Promise { - const sequence = await this.nextSequence() - const score = createPriorityScore(record.priority, sequence) - const pipeline = this.redis.pipeline() - pipeline.set(jobKey(record.id), JSON.stringify(record), 'EX', JOB_TTL_SECONDS) - pipeline.zadd(workspaceLaneKey(record.workspaceId, record.lane), score, record.id) - pipeline.zadd(ACTIVE_WORKSPACES_KEY, 'NX', sequence, record.workspaceId) - await pipeline.exec() - } - - async claimWorkspaceJob( - workspaceId: string, - options: { - lanes: readonly WorkspaceDispatchLane[] - concurrencyLimit: number - leaseId: string - now: number - leaseTtlMs: number - } - ): Promise { - const raw = await this.redis.eval( - CLAIM_JOB_SCRIPT, - 0, - workspaceId, - String(options.now), - String(options.concurrencyLimit), - options.leaseId, - String(options.now + options.leaseTtlMs), - JSON.stringify(options.lanes), - SEQUENCE_KEY, - ACTIVE_WORKSPACES_KEY, - `${DISPATCH_PREFIX}:job:`, - `${DISPATCH_PREFIX}:workspace:`, - String(JOB_TTL_SECONDS) - ) - - const parsed = JSON.parse(String(raw)) as WorkspaceDispatchClaimResult - switch (parsed.type) { - case WORKSPACE_DISPATCH_CLAIM_RESULTS.ADMITTED: - case WORKSPACE_DISPATCH_CLAIM_RESULTS.DELAYED: - case WORKSPACE_DISPATCH_CLAIM_RESULTS.LIMIT_REACHED: - case WORKSPACE_DISPATCH_CLAIM_RESULTS.EMPTY: - return parsed - default: - throw new Error( - `Unknown dispatch claim result: ${String((parsed as { type?: string }).type)}` - ) - } - } - - async getWorkspaceQueueDepth( - workspaceId: string, - lanes: readonly WorkspaceDispatchLane[] - ): Promise { - if (lanes.length === 0) return 0 - const pipeline = this.redis.pipeline() - for (const lane of lanes) { - pipeline.zcard(workspaceLaneKey(workspaceId, lane)) - } - const results = await pipeline.exec() - let depth = 0 - for (const result of results ?? []) { - if (result && !result[0]) { - depth += (result[1] as number) ?? 0 - } - } - return depth - } - - async getGlobalQueueDepth(): Promise { - const count = await this.redis.get(GLOBAL_DEPTH_KEY) - return count ? Math.max(0, Number.parseInt(count, 10)) : 0 - } - - async reconcileGlobalQueueDepth(knownCount: number): Promise { - await this.redis.set(GLOBAL_DEPTH_KEY, knownCount) - } - - async popNextWorkspaceId(): Promise { - const result = await this.redis.zpopmin(ACTIVE_WORKSPACES_KEY) - if (!result || result.length === 0) { - return null - } - - return result[0] ?? null - } - - async getQueuedWorkspaceCount(): Promise { - return this.redis.zcard(ACTIVE_WORKSPACES_KEY) - } - - async hasActiveWorkspace(workspaceId: string): Promise { - return (await this.redis.zscore(ACTIVE_WORKSPACES_KEY, workspaceId)) !== null - } - - async ensureWorkspaceActive(workspaceId: string, readyAt?: number): Promise { - const sequence = await this.nextSequence() - const score = readyAt && readyAt > Date.now() ? readyAt * 1_000_000 + sequence : sequence - await this.redis.zadd(ACTIVE_WORKSPACES_KEY, 'NX', score, workspaceId) - } - - async requeueWorkspaceId(workspaceId: string): Promise { - const sequence = await this.nextSequence() - await this.redis.zadd(ACTIVE_WORKSPACES_KEY, sequence, workspaceId) - } - - async workspaceHasPendingJobs( - workspaceId: string, - lanes: readonly WorkspaceDispatchLane[] - ): Promise { - for (const lane of lanes) { - const count = await this.redis.zcard(workspaceLaneKey(workspaceId, lane)) - if (count > 0) { - return true - } - } - - return false - } - - async getNextWorkspaceJob( - workspaceId: string, - lanes: readonly WorkspaceDispatchLane[] - ): Promise { - for (const lane of lanes) { - const ids = await this.redis.zrange(workspaceLaneKey(workspaceId, lane), 0, 0) - if (ids.length === 0) { - continue - } - - const record = await this.getDispatchJobRecord(ids[0]) - if (!record) { - await this.redis.zrem(workspaceLaneKey(workspaceId, lane), ids[0]) - continue - } - - return record - } - - return null - } - - async removeWorkspaceJobFromLane( - workspaceId: string, - lane: WorkspaceDispatchLane, - jobId: string - ): Promise { - await this.redis.zrem(workspaceLaneKey(workspaceId, lane), jobId) - } - - async cleanupExpiredWorkspaceLeases(workspaceId: string): Promise { - await this.redis.zremrangebyscore(workspaceLeaseKey(workspaceId), 0, Date.now()) - } - - async countActiveWorkspaceLeases(workspaceId: string): Promise { - await this.cleanupExpiredWorkspaceLeases(workspaceId) - return this.redis.zcard(workspaceLeaseKey(workspaceId)) - } - - async hasWorkspaceLease(workspaceId: string, leaseId: string): Promise { - await this.cleanupExpiredWorkspaceLeases(workspaceId) - return (await this.redis.zscore(workspaceLeaseKey(workspaceId), leaseId)) !== null - } - - async createWorkspaceLease(workspaceId: string, leaseId: string, ttlMs: number): Promise { - const expiresAt = Date.now() + ttlMs - await this.redis.zadd(workspaceLeaseKey(workspaceId), expiresAt, leaseId) - return expiresAt - } - - async refreshWorkspaceLease( - workspaceId: string, - leaseId: string, - ttlMs: number - ): Promise { - return this.createWorkspaceLease(workspaceId, leaseId, ttlMs) - } - - async releaseWorkspaceLease(workspaceId: string, leaseId: string): Promise { - await this.redis.zrem(workspaceLeaseKey(workspaceId), leaseId) - } - - async removeWorkspaceIfIdle( - workspaceId: string, - lanes: readonly WorkspaceDispatchLane[] - ): Promise { - const hasPendingJobs = await this.workspaceHasPendingJobs(workspaceId, lanes) - if (!hasPendingJobs) { - await this.redis.zrem(ACTIVE_WORKSPACES_KEY, workspaceId) - } - } - - async markDispatchJobAdmitted( - jobId: string, - workspaceId: string, - leaseId: string, - leaseExpiresAt: number - ): Promise { - await this.updateDispatchJobRecord(jobId, (record) => ({ - ...record, - status: 'admitted', - admittedAt: Date.now(), - lease: { - workspaceId, - leaseId, - }, - metadata: { - ...record.metadata, - dispatchLeaseExpiresAt: leaseExpiresAt, - }, - })) - } - - async markDispatchJobAdmitting( - jobId: string, - workspaceId: string, - leaseId: string, - leaseExpiresAt: number - ): Promise { - await this.updateDispatchJobRecord(jobId, (record) => ({ - ...record, - status: 'admitting', - lease: { - workspaceId, - leaseId, - }, - metadata: { - ...record.metadata, - dispatchLeaseExpiresAt: leaseExpiresAt, - }, - })) - } - - async markDispatchJobRunning(jobId: string): Promise { - await this.updateDispatchJobRecord(jobId, (record) => ({ - ...record, - status: 'running', - startedAt: record.startedAt ?? Date.now(), - })) - } - - async markDispatchJobCompleted(jobId: string, output: unknown): Promise { - await this.updateDispatchJobRecord(jobId, (record) => ({ - ...record, - status: 'completed', - completedAt: Date.now(), - output, - })) - await this.redis.decr(GLOBAL_DEPTH_KEY).catch(() => undefined) - } - - async markDispatchJobFailed(jobId: string, error: string): Promise { - await this.updateDispatchJobRecord(jobId, (record) => ({ - ...record, - status: 'failed', - completedAt: Date.now(), - error, - })) - await this.redis.decr(GLOBAL_DEPTH_KEY).catch(() => undefined) - } - - async clear(): Promise { - let cursor = '0' - const keys: string[] = [] - - do { - const [nextCursor, foundKeys] = await this.redis.scan( - cursor, - 'MATCH', - `${DISPATCH_PREFIX}:*`, - 'COUNT', - 100 - ) - cursor = nextCursor - keys.push(...foundKeys) - } while (cursor !== '0') - - if (keys.length > 0) { - await this.redis.del(...keys) - } - } - - dispose(): void { - logger.info('Redis workspace dispatch storage disposed') - } -} diff --git a/apps/sim/lib/core/workspace-dispatch/status.test.ts b/apps/sim/lib/core/workspace-dispatch/status.test.ts deleted file mode 100644 index e72e210b18..0000000000 --- a/apps/sim/lib/core/workspace-dispatch/status.test.ts +++ /dev/null @@ -1,102 +0,0 @@ -/** - * @vitest-environment node - */ -import { describe, expect, it } from 'vitest' -import { presentDispatchOrJobStatus } from '@/lib/core/workspace-dispatch/status' - -describe('workspace dispatch status presentation', () => { - it('presents waiting dispatch jobs with queue metadata', () => { - const result = presentDispatchOrJobStatus( - { - id: 'dispatch-1', - workspaceId: 'workspace-1', - lane: 'runtime', - queueName: 'workflow-execution', - bullmqJobName: 'workflow-execution', - bullmqPayload: {}, - metadata: { workflowId: 'workflow-1' }, - priority: 10, - status: 'waiting', - createdAt: 1000, - }, - null - ) - - expect(result).toEqual({ - status: 'waiting', - metadata: { - createdAt: new Date(1000), - admittedAt: undefined, - startedAt: undefined, - completedAt: undefined, - queueName: 'workflow-execution', - lane: 'runtime', - workspaceId: 'workspace-1', - }, - estimatedDuration: 300000, - }) - }) - - it('presents admitting dispatch jobs distinctly', () => { - const result = presentDispatchOrJobStatus( - { - id: 'dispatch-1a', - workspaceId: 'workspace-1', - lane: 'runtime', - queueName: 'workflow-execution', - bullmqJobName: 'workflow-execution', - bullmqPayload: {}, - metadata: { workflowId: 'workflow-1' }, - priority: 10, - status: 'admitting', - createdAt: 1000, - }, - null - ) - - expect(result.status).toBe('admitting') - expect(result.estimatedDuration).toBe(300000) - }) - - it('presents completed dispatch jobs with output and duration', () => { - const result = presentDispatchOrJobStatus( - { - id: 'dispatch-2', - workspaceId: 'workspace-1', - lane: 'interactive', - queueName: 'workflow-execution', - bullmqJobName: 'direct-workflow-execution', - bullmqPayload: {}, - metadata: { workflowId: 'workflow-1' }, - priority: 1, - status: 'completed', - createdAt: 1000, - admittedAt: 1500, - startedAt: 2000, - completedAt: 7000, - output: { success: true }, - }, - null - ) - - expect(result.status).toBe('completed') - expect(result.output).toEqual({ success: true }) - expect(result.metadata.duration).toBe(5000) - }) - - it('falls back to legacy job status when no dispatch record exists', () => { - const result = presentDispatchOrJobStatus(null, { - id: 'job-1', - type: 'workflow-execution', - payload: {}, - status: 'pending', - createdAt: new Date(1000), - attempts: 0, - maxAttempts: 3, - metadata: {}, - }) - - expect(result.status).toBe('queued') - expect(result.estimatedDuration).toBe(300000) - }) -}) diff --git a/apps/sim/lib/core/workspace-dispatch/status.ts b/apps/sim/lib/core/workspace-dispatch/status.ts deleted file mode 100644 index fc5d934434..0000000000 --- a/apps/sim/lib/core/workspace-dispatch/status.ts +++ /dev/null @@ -1,110 +0,0 @@ -import type { Job, JobStatus } from '@/lib/core/async-jobs/types' -import type { WorkspaceDispatchJobRecord } from '@/lib/core/workspace-dispatch/types' - -export type DispatchPresentedStatus = - | 'waiting' - | 'admitting' - | 'admitted' - | 'running' - | 'completed' - | 'failed' - | 'queued' - | JobStatus - -export interface DispatchStatusPresentation { - status: DispatchPresentedStatus - metadata: { - createdAt?: Date - admittedAt?: Date - startedAt?: Date - completedAt?: Date - queueName?: string - lane?: string - workspaceId?: string - duration?: number - } - output?: unknown - error?: string - estimatedDuration?: number -} - -export function presentDispatchOrJobStatus( - dispatchJob: WorkspaceDispatchJobRecord | null, - job: Job | null -): DispatchStatusPresentation { - if (dispatchJob) { - const startedAt = dispatchJob.startedAt ? new Date(dispatchJob.startedAt) : undefined - const completedAt = dispatchJob.completedAt ? new Date(dispatchJob.completedAt) : undefined - - const response: DispatchStatusPresentation = { - status: dispatchJob.status, - metadata: { - createdAt: new Date(dispatchJob.createdAt), - admittedAt: dispatchJob.admittedAt ? new Date(dispatchJob.admittedAt) : undefined, - startedAt, - completedAt, - queueName: dispatchJob.queueName, - lane: dispatchJob.lane, - workspaceId: dispatchJob.workspaceId, - }, - } - - if (startedAt && completedAt) { - response.metadata.duration = completedAt.getTime() - startedAt.getTime() - } - - if (dispatchJob.status === 'completed') { - response.output = dispatchJob.output - } - - if (dispatchJob.status === 'failed') { - response.error = dispatchJob.error - } - - if ( - dispatchJob.status === 'waiting' || - dispatchJob.status === 'admitting' || - dispatchJob.status === 'admitted' || - dispatchJob.status === 'running' - ) { - response.estimatedDuration = 300000 - } - - return response - } - - if (!job) { - return { - status: 'queued', - metadata: {}, - } - } - - const mappedStatus = job.status === 'pending' ? 'queued' : job.status - const response: DispatchStatusPresentation = { - status: mappedStatus, - metadata: { - createdAt: job.createdAt, - startedAt: job.startedAt, - completedAt: job.completedAt, - }, - } - - if (job.startedAt && job.completedAt) { - response.metadata.duration = job.completedAt.getTime() - job.startedAt.getTime() - } - - if (job.status === 'completed') { - response.output = job.output - } - - if (job.status === 'failed') { - response.error = job.error - } - - if (job.status === 'processing' || job.status === 'pending') { - response.estimatedDuration = 300000 - } - - return response -} diff --git a/apps/sim/lib/core/workspace-dispatch/store.ts b/apps/sim/lib/core/workspace-dispatch/store.ts deleted file mode 100644 index 86c1c3951a..0000000000 --- a/apps/sim/lib/core/workspace-dispatch/store.ts +++ /dev/null @@ -1,193 +0,0 @@ -import type { WorkspaceDispatchStorageAdapter } from '@/lib/core/workspace-dispatch/adapter' -import { - setWorkspaceDispatchStorageAdapter as _setAdapter, - createWorkspaceDispatchStorageAdapter, -} from '@/lib/core/workspace-dispatch/factory' -import type { - WorkspaceDispatchClaimResult, - WorkspaceDispatchEnqueueInput, - WorkspaceDispatchJobRecord, - WorkspaceDispatchLane, -} from '@/lib/core/workspace-dispatch/types' - -function getAdapter() { - return createWorkspaceDispatchStorageAdapter() -} - -export function setWorkspaceDispatchStorageAdapter(adapter: WorkspaceDispatchStorageAdapter): void { - _setAdapter(adapter) -} - -export async function saveDispatchJob(record: WorkspaceDispatchJobRecord): Promise { - return getAdapter().saveDispatchJob(record) -} - -export async function getDispatchJobRecord( - jobId: string -): Promise { - return getAdapter().getDispatchJobRecord(jobId) -} - -export async function listDispatchJobsByStatuses( - statuses: readonly WorkspaceDispatchJobRecord['status'][] -): Promise { - return getAdapter().listDispatchJobsByStatuses(statuses) -} - -export async function updateDispatchJobRecord( - jobId: string, - updater: (record: WorkspaceDispatchJobRecord) => WorkspaceDispatchJobRecord -): Promise { - return getAdapter().updateDispatchJobRecord(jobId, updater) -} - -export async function enqueueWorkspaceDispatchJob( - input: WorkspaceDispatchEnqueueInput -): Promise { - return getAdapter().enqueueWorkspaceDispatchJob(input) -} - -export async function restoreWorkspaceDispatchJob( - record: WorkspaceDispatchJobRecord -): Promise { - return getAdapter().restoreWorkspaceDispatchJob(record) -} - -export async function claimWorkspaceJob( - workspaceId: string, - options: { - lanes: readonly WorkspaceDispatchLane[] - concurrencyLimit: number - leaseId: string - now: number - leaseTtlMs: number - } -): Promise { - return getAdapter().claimWorkspaceJob(workspaceId, options) -} - -export async function getWorkspaceQueueDepth( - workspaceId: string, - lanes: readonly WorkspaceDispatchLane[] -): Promise { - return getAdapter().getWorkspaceQueueDepth(workspaceId, lanes) -} - -export async function getGlobalQueueDepth(): Promise { - return getAdapter().getGlobalQueueDepth() -} - -export async function reconcileGlobalQueueDepth(knownCount: number): Promise { - return getAdapter().reconcileGlobalQueueDepth(knownCount) -} - -export async function popNextWorkspaceId(): Promise { - return getAdapter().popNextWorkspaceId() -} - -export async function getQueuedWorkspaceCount(): Promise { - return getAdapter().getQueuedWorkspaceCount() -} - -export async function hasActiveWorkspace(workspaceId: string): Promise { - return getAdapter().hasActiveWorkspace(workspaceId) -} - -export async function ensureWorkspaceActive(workspaceId: string, readyAt?: number): Promise { - return getAdapter().ensureWorkspaceActive(workspaceId, readyAt) -} - -export async function requeueWorkspaceId(workspaceId: string): Promise { - return getAdapter().requeueWorkspaceId(workspaceId) -} - -export async function workspaceHasPendingJobs( - workspaceId: string, - lanes: readonly WorkspaceDispatchLane[] -): Promise { - return getAdapter().workspaceHasPendingJobs(workspaceId, lanes) -} - -export async function getNextWorkspaceJob( - workspaceId: string, - lanes: readonly WorkspaceDispatchLane[] -): Promise { - return getAdapter().getNextWorkspaceJob(workspaceId, lanes) -} - -export async function removeWorkspaceJobFromLane( - workspaceId: string, - lane: WorkspaceDispatchLane, - jobId: string -): Promise { - return getAdapter().removeWorkspaceJobFromLane(workspaceId, lane, jobId) -} - -export async function cleanupExpiredWorkspaceLeases(workspaceId: string): Promise { - return getAdapter().cleanupExpiredWorkspaceLeases(workspaceId) -} - -export async function countActiveWorkspaceLeases(workspaceId: string): Promise { - return getAdapter().countActiveWorkspaceLeases(workspaceId) -} - -export async function hasWorkspaceLease(workspaceId: string, leaseId: string): Promise { - return getAdapter().hasWorkspaceLease(workspaceId, leaseId) -} - -export async function createWorkspaceLease( - workspaceId: string, - leaseId: string, - ttlMs: number -): Promise { - return getAdapter().createWorkspaceLease(workspaceId, leaseId, ttlMs) -} - -export async function refreshWorkspaceLease( - workspaceId: string, - leaseId: string, - ttlMs: number -): Promise { - return getAdapter().refreshWorkspaceLease(workspaceId, leaseId, ttlMs) -} - -export async function releaseWorkspaceLease(workspaceId: string, leaseId: string): Promise { - return getAdapter().releaseWorkspaceLease(workspaceId, leaseId) -} - -export async function removeWorkspaceIfIdle( - workspaceId: string, - lanes: readonly WorkspaceDispatchLane[] -): Promise { - return getAdapter().removeWorkspaceIfIdle(workspaceId, lanes) -} - -export async function markDispatchJobAdmitted( - jobId: string, - workspaceId: string, - leaseId: string, - leaseExpiresAt: number -): Promise { - return getAdapter().markDispatchJobAdmitted(jobId, workspaceId, leaseId, leaseExpiresAt) -} - -export async function markDispatchJobAdmitting( - jobId: string, - workspaceId: string, - leaseId: string, - leaseExpiresAt: number -): Promise { - return getAdapter().markDispatchJobAdmitting(jobId, workspaceId, leaseId, leaseExpiresAt) -} - -export async function markDispatchJobRunning(jobId: string): Promise { - return getAdapter().markDispatchJobRunning(jobId) -} - -export async function markDispatchJobCompleted(jobId: string, output: unknown): Promise { - return getAdapter().markDispatchJobCompleted(jobId, output) -} - -export async function markDispatchJobFailed(jobId: string, error: string): Promise { - return getAdapter().markDispatchJobFailed(jobId, error) -} diff --git a/apps/sim/lib/core/workspace-dispatch/types.ts b/apps/sim/lib/core/workspace-dispatch/types.ts deleted file mode 100644 index 8721895664..0000000000 --- a/apps/sim/lib/core/workspace-dispatch/types.ts +++ /dev/null @@ -1,107 +0,0 @@ -import type { JobMetadata, JobType } from '@/lib/core/async-jobs/types' -import type { - KNOWLEDGE_CONNECTOR_SYNC_QUEUE, - KNOWLEDGE_DOCUMENT_PROCESSING_QUEUE, - MOTHERSHIP_JOB_EXECUTION_QUEUE, - WORKSPACE_NOTIFICATION_DELIVERY_QUEUE, -} from '@/lib/core/bullmq/queues' - -export const WORKSPACE_DISPATCH_LANES = [ - 'interactive', - 'runtime', - 'knowledge', - 'lightweight', -] as const - -export type WorkspaceDispatchLane = (typeof WORKSPACE_DISPATCH_LANES)[number] - -export type WorkspaceDispatchQueueName = - | JobType - | typeof KNOWLEDGE_CONNECTOR_SYNC_QUEUE - | typeof KNOWLEDGE_DOCUMENT_PROCESSING_QUEUE - | typeof MOTHERSHIP_JOB_EXECUTION_QUEUE - | typeof WORKSPACE_NOTIFICATION_DELIVERY_QUEUE - -export const WORKSPACE_DISPATCH_STATUSES = { - WAITING: 'waiting', - ADMITTING: 'admitting', - ADMITTED: 'admitted', - RUNNING: 'running', - COMPLETED: 'completed', - FAILED: 'failed', -} as const - -export type WorkspaceDispatchStatus = - (typeof WORKSPACE_DISPATCH_STATUSES)[keyof typeof WORKSPACE_DISPATCH_STATUSES] - -export interface WorkspaceDispatchLeaseInfo { - workspaceId: string - leaseId: string -} - -export interface WorkspaceDispatchJobContext { - dispatchJobId: string - workspaceId: string - lane: WorkspaceDispatchLane - queueName: WorkspaceDispatchQueueName - bullmqJobName: string - priority: number -} - -export interface WorkspaceDispatchJobRecord { - id: string - workspaceId: string - lane: WorkspaceDispatchLane - queueName: WorkspaceDispatchQueueName - bullmqJobName: string - bullmqPayload: unknown - metadata: JobMetadata - priority: number - maxAttempts?: number - delayMs?: number - status: WorkspaceDispatchStatus - createdAt: number - admittedAt?: number - startedAt?: number - completedAt?: number - output?: unknown - error?: string - lease?: WorkspaceDispatchLeaseInfo -} - -export interface WorkspaceDispatchEnqueueInput { - id?: string - workspaceId: string - lane: WorkspaceDispatchLane - queueName: WorkspaceDispatchQueueName - bullmqJobName: string - bullmqPayload: unknown - metadata: JobMetadata - priority?: number - maxAttempts?: number - delayMs?: number -} - -export const WORKSPACE_DISPATCH_CLAIM_RESULTS = { - ADMITTED: 'admitted', - LIMIT_REACHED: 'limit_reached', - DELAYED: 'delayed', - EMPTY: 'empty', -} as const - -export type WorkspaceDispatchClaimResult = - | { - type: typeof WORKSPACE_DISPATCH_CLAIM_RESULTS.ADMITTED - record: WorkspaceDispatchJobRecord - leaseId: string - leaseExpiresAt: number - } - | { - type: - | typeof WORKSPACE_DISPATCH_CLAIM_RESULTS.LIMIT_REACHED - | typeof WORKSPACE_DISPATCH_CLAIM_RESULTS.EMPTY - } - | { - type: typeof WORKSPACE_DISPATCH_CLAIM_RESULTS.DELAYED - nextReadyAt: number - } diff --git a/apps/sim/lib/core/workspace-dispatch/worker.test.ts b/apps/sim/lib/core/workspace-dispatch/worker.test.ts deleted file mode 100644 index 1833b128cd..0000000000 --- a/apps/sim/lib/core/workspace-dispatch/worker.test.ts +++ /dev/null @@ -1,98 +0,0 @@ -/** - * @vitest-environment node - */ -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { - mockMarkDispatchJobCompleted, - mockMarkDispatchJobFailed, - mockMarkDispatchJobRunning, - mockReleaseWorkspaceLease, - mockWakeWorkspaceDispatcher, -} = vi.hoisted(() => ({ - mockMarkDispatchJobCompleted: vi.fn(), - mockMarkDispatchJobFailed: vi.fn(), - mockMarkDispatchJobRunning: vi.fn(), - mockReleaseWorkspaceLease: vi.fn(), - mockWakeWorkspaceDispatcher: vi.fn(), -})) - -vi.mock('@sim/logger', () => ({ - createLogger: () => ({ - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - debug: vi.fn(), - }), -})) - -vi.mock('@/lib/core/workspace-dispatch', () => ({ - markDispatchJobCompleted: mockMarkDispatchJobCompleted, - markDispatchJobFailed: mockMarkDispatchJobFailed, - markDispatchJobRunning: mockMarkDispatchJobRunning, - releaseWorkspaceLease: mockReleaseWorkspaceLease, - wakeWorkspaceDispatcher: mockWakeWorkspaceDispatcher, -})) - -import { getDispatchRuntimeMetadata, runDispatchedJob } from '@/lib/core/workspace-dispatch/worker' - -describe('workspace dispatch worker lifecycle', () => { - beforeEach(() => { - vi.clearAllMocks() - }) - - it('returns null for missing metadata', () => { - expect(getDispatchRuntimeMetadata(undefined)).toBeNull() - }) - - it('extracts dispatch runtime metadata when all fields are present', () => { - expect( - getDispatchRuntimeMetadata({ - dispatchJobId: 'dispatch-1', - dispatchWorkspaceId: 'workspace-1', - dispatchLeaseId: 'lease-1', - }) - ).toEqual({ - dispatchJobId: 'dispatch-1', - dispatchWorkspaceId: 'workspace-1', - dispatchLeaseId: 'lease-1', - }) - }) - - it('marks running, completed, releases lease, and wakes dispatcher on success', async () => { - const result = await runDispatchedJob( - { - dispatchJobId: 'dispatch-1', - dispatchWorkspaceId: 'workspace-1', - dispatchLeaseId: 'lease-1', - }, - async () => ({ success: true }) - ) - - expect(result).toEqual({ success: true }) - expect(mockMarkDispatchJobRunning).toHaveBeenCalledWith('dispatch-1') - expect(mockMarkDispatchJobCompleted).toHaveBeenCalledWith('dispatch-1', { success: true }) - expect(mockReleaseWorkspaceLease).toHaveBeenCalledWith('workspace-1', 'lease-1') - expect(mockWakeWorkspaceDispatcher).toHaveBeenCalled() - }) - - it('marks failed and still releases lease on error', async () => { - await expect( - runDispatchedJob( - { - dispatchJobId: 'dispatch-2', - dispatchWorkspaceId: 'workspace-2', - dispatchLeaseId: 'lease-2', - }, - async () => { - throw new Error('boom') - } - ) - ).rejects.toThrow('boom') - - expect(mockMarkDispatchJobRunning).toHaveBeenCalledWith('dispatch-2') - expect(mockMarkDispatchJobFailed).toHaveBeenCalledWith('dispatch-2', 'boom') - expect(mockReleaseWorkspaceLease).toHaveBeenCalledWith('workspace-2', 'lease-2') - expect(mockWakeWorkspaceDispatcher).toHaveBeenCalled() - }) -}) diff --git a/apps/sim/lib/core/workspace-dispatch/worker.ts b/apps/sim/lib/core/workspace-dispatch/worker.ts deleted file mode 100644 index ced31a599a..0000000000 --- a/apps/sim/lib/core/workspace-dispatch/worker.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { createLogger } from '@sim/logger' -import { - markDispatchJobCompleted, - markDispatchJobFailed, - markDispatchJobRunning, - refreshWorkspaceLease, - releaseWorkspaceLease, - wakeWorkspaceDispatcher, -} from '@/lib/core/workspace-dispatch' - -const logger = createLogger('WorkspaceDispatchWorker') - -interface DispatchRuntimeMetadata { - dispatchJobId: string - dispatchWorkspaceId: string - dispatchLeaseId: string -} - -interface RunDispatchedJobOptions { - isFinalAttempt?: boolean - leaseTtlMs?: number -} - -const DEFAULT_LEASE_TTL_MS = 15 * 60 * 1000 -const LEASE_HEARTBEAT_INTERVAL_MS = 60_000 - -export function getDispatchRuntimeMetadata(metadata: unknown): DispatchRuntimeMetadata | null { - if (!metadata || typeof metadata !== 'object') { - return null - } - - const value = metadata as Partial - if (!value.dispatchJobId || !value.dispatchWorkspaceId || !value.dispatchLeaseId) { - return null - } - - return { - dispatchJobId: value.dispatchJobId, - dispatchWorkspaceId: value.dispatchWorkspaceId, - dispatchLeaseId: value.dispatchLeaseId, - } -} - -export async function runDispatchedJob( - metadata: unknown, - run: () => Promise, - options: RunDispatchedJobOptions = {} -): Promise { - const dispatchMetadata = getDispatchRuntimeMetadata(metadata) - - if (!dispatchMetadata) { - return run() - } - - const leaseTtlMs = options.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS - const isFinalAttempt = options.isFinalAttempt ?? true - - await markDispatchJobRunning(dispatchMetadata.dispatchJobId) - - let heartbeatTimer: NodeJS.Timeout | null = setInterval(() => { - void refreshWorkspaceLease( - dispatchMetadata.dispatchWorkspaceId, - dispatchMetadata.dispatchLeaseId, - leaseTtlMs - ).catch((error) => { - logger.error('Failed to refresh dispatch lease', { error, dispatchMetadata }) - }) - }, LEASE_HEARTBEAT_INTERVAL_MS) - heartbeatTimer.unref() - - let succeeded = false - try { - const result = await run() - succeeded = true - await markDispatchJobCompleted(dispatchMetadata.dispatchJobId, result) - return result - } catch (error) { - if (isFinalAttempt && !succeeded) { - await markDispatchJobFailed( - dispatchMetadata.dispatchJobId, - error instanceof Error ? error.message : String(error) - ) - } - throw error - } finally { - if (heartbeatTimer) { - clearInterval(heartbeatTimer) - heartbeatTimer = null - } - - const shouldReleaseLease = succeeded || isFinalAttempt - if (shouldReleaseLease) { - try { - await releaseWorkspaceLease( - dispatchMetadata.dispatchWorkspaceId, - dispatchMetadata.dispatchLeaseId - ) - await wakeWorkspaceDispatcher() - } catch (error) { - logger.error('Failed to release dispatch lease', { error, dispatchMetadata }) - } - } - } -} diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.ts b/apps/sim/lib/knowledge/connectors/sync-engine.ts index 24ab9210f5..e2551d5b03 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.ts @@ -9,10 +9,8 @@ import { import { createLogger } from '@sim/logger' import { and, eq, gt, inArray, isNull, lt, ne, or, sql } from 'drizzle-orm' import { decryptApiKey } from '@/lib/api-key/crypto' -import { createBullMQJobData, isBullMQEnabled } from '@/lib/core/bullmq' import { getInternalApiBaseUrl } from '@/lib/core/utils/urls' import { generateId } from '@/lib/core/utils/uuid' -import { enqueueWorkspaceDispatch } from '@/lib/core/workspace-dispatch' import type { DocumentData } from '@/lib/knowledge/documents/service' import { hardDeleteDocuments, @@ -180,38 +178,6 @@ export async function dispatchSync( { tags } ) logger.info(`Dispatched connector sync to Trigger.dev`, { connectorId, requestId }) - } else if (isBullMQEnabled()) { - const connectorRows = await db - .select({ - workspaceId: knowledgeBase.workspaceId, - userId: knowledgeBase.userId, - }) - .from(knowledgeConnector) - .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) - .where(eq(knowledgeConnector.id, connectorId)) - .limit(1) - - const workspaceId = connectorRows[0]?.workspaceId - const userId = connectorRows[0]?.userId - if (!workspaceId || !userId) { - throw new Error(`No workspace found for connector ${connectorId}`) - } - - await enqueueWorkspaceDispatch({ - workspaceId, - lane: 'knowledge', - queueName: 'knowledge-connector-sync', - bullmqJobName: 'knowledge-connector-sync', - bullmqPayload: createBullMQJobData({ - connectorId, - fullSync: options?.fullSync, - requestId, - }), - metadata: { - userId, - }, - }) - logger.info(`Dispatched connector sync to BullMQ`, { connectorId, requestId }) } else { executeSync(connectorId, { fullSync: options?.fullSync }).catch((error) => { logger.error(`Sync failed for connector ${connectorId}`, { diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index ff613b4e8c..10c7ee639c 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -27,11 +27,9 @@ import { } from 'drizzle-orm' import { recordUsage } from '@/lib/billing/core/usage-log' import { checkAndBillOverageThreshold } from '@/lib/billing/threshold-billing' -import { createBullMQJobData, isBullMQEnabled } from '@/lib/core/bullmq' import { env } from '@/lib/core/config/env' import { getCostMultiplier, isTriggerDevEnabled } from '@/lib/core/config/feature-flags' import { generateId } from '@/lib/core/utils/uuid' -import { enqueueWorkspaceDispatch } from '@/lib/core/workspace-dispatch' import { processDocument } from '@/lib/knowledge/documents/document-processor' import type { DocumentSortField, SortOrder } from '@/lib/knowledge/documents/types' import { generateEmbeddings } from '@/lib/knowledge/embeddings' @@ -85,15 +83,8 @@ const PROCESSING_CONFIG = { delayBetweenDocuments: (env.KB_CONFIG_DELAY_BETWEEN_DOCUMENTS || 50) * 2, } -const REDIS_PROCESSING_CONFIG = { - maxConcurrentDocuments: env.KB_CONFIG_CONCURRENCY_LIMIT || 20, - batchSize: env.KB_CONFIG_BATCH_SIZE || 20, - delayBetweenBatches: env.KB_CONFIG_DELAY_BETWEEN_BATCHES || 100, - delayBetweenDocuments: env.KB_CONFIG_DELAY_BETWEEN_DOCUMENTS || 50, -} - export function getProcessingConfig() { - return isBullMQEnabled() ? REDIS_PROCESSING_CONFIG : PROCESSING_CONFIG + return PROCESSING_CONFIG } export interface DocumentData { @@ -130,35 +121,6 @@ export async function dispatchDocumentProcessingJob(payload: DocumentJobData): P return } - if (isBullMQEnabled()) { - const workspaceRows = await db - .select({ - workspaceId: knowledgeBase.workspaceId, - userId: knowledgeBase.userId, - }) - .from(knowledgeBase) - .where(and(eq(knowledgeBase.id, payload.knowledgeBaseId), isNull(knowledgeBase.deletedAt))) - .limit(1) - - const workspaceId = workspaceRows[0]?.workspaceId - const userId = workspaceRows[0]?.userId - if (!workspaceId || !userId) { - throw new Error(`Knowledge base not found: ${payload.knowledgeBaseId}`) - } - - await enqueueWorkspaceDispatch({ - workspaceId, - lane: 'knowledge', - queueName: 'knowledge-process-document', - bullmqJobName: 'knowledge-process-document', - bullmqPayload: createBullMQJobData(payload), - metadata: { - userId, - }, - }) - return - } - await processDocumentAsync( payload.knowledgeBaseId, payload.documentId, @@ -379,7 +341,7 @@ export async function processDocumentsWithQueue( logger.info( `[${requestId}] Dispatching background processing for ${jobPayloads.length} documents`, { - backend: isTriggerAvailable() ? 'trigger-dev' : isBullMQEnabled() ? 'bullmq' : 'direct', + backend: isTriggerAvailable() ? 'trigger-dev' : 'direct', } ) diff --git a/apps/sim/lib/logs/events.ts b/apps/sim/lib/logs/events.ts index b08b7dc131..c1f54d64da 100644 --- a/apps/sim/lib/logs/events.ts +++ b/apps/sim/lib/logs/events.ts @@ -12,7 +12,6 @@ import { } from '@/lib/notifications/alert-rules' import { getActiveWorkflowContext } from '@/lib/workflows/active-context' import { - enqueueNotificationDeliveryDispatch, executeNotificationDelivery, workspaceNotificationDeliveryTask, } from '@/background/workspace-notification-delivery' @@ -149,10 +148,6 @@ export async function emitWorkflowExecutionCompleted(log: WorkflowExecutionLog): logger.info( `Enqueued ${subscription.notificationType} notification ${deliveryId} via Trigger.dev` ) - } else if (await enqueueNotificationDeliveryDispatch(payload)) { - logger.info( - `Enqueued ${subscription.notificationType} notification ${deliveryId} via BullMQ` - ) } else { void executeNotificationDelivery(payload).catch((error) => { logger.error(`Direct notification delivery failed for ${deliveryId}`, { error }) diff --git a/apps/sim/lib/webhooks/processor.test.ts b/apps/sim/lib/webhooks/processor.test.ts index 3f543780f7..6d644092e8 100644 --- a/apps/sim/lib/webhooks/processor.test.ts +++ b/apps/sim/lib/webhooks/processor.test.ts @@ -9,14 +9,12 @@ const { mockGenerateId, mockPreprocessExecution, mockEnqueue, - mockEnqueueWorkspaceDispatch, mockGetJobQueue, mockShouldExecuteInline, } = vi.hoisted(() => ({ mockGenerateId: vi.fn(), mockPreprocessExecution: vi.fn(), mockEnqueue: vi.fn(), - mockEnqueueWorkspaceDispatch: vi.fn(), mockGetJobQueue: vi.fn(), mockShouldExecuteInline: vi.fn(), })) @@ -68,15 +66,6 @@ vi.mock('@/lib/core/async-jobs', () => ({ shouldExecuteInline: mockShouldExecuteInline, })) -vi.mock('@/lib/core/bullmq', () => ({ - isBullMQEnabled: vi.fn().mockReturnValue(true), - createBullMQJobData: vi.fn((payload: unknown, metadata?: unknown) => ({ payload, metadata })), -})) - -vi.mock('@/lib/core/workspace-dispatch', () => ({ - enqueueWorkspaceDispatch: mockEnqueueWorkspaceDispatch, -})) - vi.mock('@/lib/core/config/feature-flags', () => ({ isProd: false, })) @@ -150,7 +139,6 @@ describe('webhook processor execution identity', () => { actorUserId: 'actor-user-1', }) mockEnqueue.mockResolvedValue('job-1') - mockEnqueueWorkspaceDispatch.mockResolvedValue('job-1') mockGetJobQueue.mockResolvedValue({ enqueue: mockEnqueue }) mockShouldExecuteInline.mockReturnValue(false) mockGenerateId.mockReturnValue('generated-execution-id') @@ -211,14 +199,16 @@ describe('webhook processor execution identity', () => { ) expect(mockGenerateId).toHaveBeenCalledTimes(1) - expect(mockEnqueueWorkspaceDispatch).toHaveBeenCalledWith( + expect(mockEnqueue).toHaveBeenCalledWith( + 'webhook-execution', + expect.objectContaining({ + workflowId: 'workflow-1', + provider: 'gmail', + }), expect.objectContaining({ - id: 'generated-execution-id', - workspaceId: 'workspace-1', - lane: 'runtime', - queueName: 'webhook-execution', metadata: expect.objectContaining({ workflowId: 'workflow-1', + workspaceId: 'workspace-1', userId: 'actor-user-1', correlation: preprocessingResult.correlation, }), diff --git a/apps/sim/lib/webhooks/processor.ts b/apps/sim/lib/webhooks/processor.ts index ba20a6c4cb..6b7aca3f20 100644 --- a/apps/sim/lib/webhooks/processor.ts +++ b/apps/sim/lib/webhooks/processor.ts @@ -7,10 +7,8 @@ import { isOrganizationOnTeamOrEnterprisePlan } from '@/lib/billing/core/subscri import { tryAdmit } from '@/lib/core/admission/gate' import { getInlineJobQueue, getJobQueue, shouldExecuteInline } from '@/lib/core/async-jobs' import type { AsyncExecutionCorrelation } from '@/lib/core/async-jobs/types' -import { createBullMQJobData, isBullMQEnabled } from '@/lib/core/bullmq' import { isProd } from '@/lib/core/config/feature-flags' import { generateId } from '@/lib/core/utils/uuid' -import { DispatchQueueFullError, enqueueWorkspaceDispatch } from '@/lib/core/workspace-dispatch' import { getEffectiveDecryptedEnv } from '@/lib/environment/utils' import { preprocessExecution } from '@/lib/execution/preprocessing' import { @@ -568,68 +566,32 @@ export async function queueWebhookExecution( const isPolling = isPollingWebhookProvider(payload.provider) if (isPolling && !shouldExecuteInline()) { - const jobId = isBullMQEnabled() - ? await enqueueWorkspaceDispatch({ - id: executionId, - workspaceId: foundWorkflow.workspaceId, - lane: 'runtime', - queueName: 'webhook-execution', - bullmqJobName: 'webhook-execution', - bullmqPayload: createBullMQJobData(payload, { - workflowId: foundWorkflow.id, - userId: actorUserId, - correlation, - }), - metadata: { - workflowId: foundWorkflow.id, - userId: actorUserId, - correlation, - }, - }) - : await (await getJobQueue()).enqueue('webhook-execution', payload, { - metadata: { - workflowId: foundWorkflow.id, - workspaceId: foundWorkflow.workspaceId, - userId: actorUserId, - correlation, - }, - }) + const jobId = await (await getJobQueue()).enqueue('webhook-execution', payload, { + metadata: { + workflowId: foundWorkflow.id, + workspaceId: foundWorkflow.workspaceId, + userId: actorUserId, + correlation, + }, + }) logger.info( `[${options.requestId}] Queued polling webhook execution task ${jobId} for ${foundWebhook.provider} webhook via job queue` ) } else { const jobQueue = await getInlineJobQueue() - const jobId = isBullMQEnabled() - ? await enqueueWorkspaceDispatch({ - id: executionId, - workspaceId: foundWorkflow.workspaceId, - lane: 'runtime', - queueName: 'webhook-execution', - bullmqJobName: 'webhook-execution', - bullmqPayload: createBullMQJobData(payload, { - workflowId: foundWorkflow.id, - userId: actorUserId, - correlation, - }), - metadata: { - workflowId: foundWorkflow.id, - userId: actorUserId, - correlation, - }, - }) - : await jobQueue.enqueue('webhook-execution', payload, { - metadata: { - workflowId: foundWorkflow.id, - workspaceId: foundWorkflow.workspaceId, - userId: actorUserId, - correlation, - }, - }) + const jobId = await jobQueue.enqueue('webhook-execution', payload, { + metadata: { + workflowId: foundWorkflow.id, + workspaceId: foundWorkflow.workspaceId, + userId: actorUserId, + correlation, + }, + }) logger.info( `[${options.requestId}] Queued ${foundWebhook.provider} webhook execution ${jobId} via inline backend` ) - if (!isBullMQEnabled()) { + if (shouldExecuteInline()) { void (async () => { try { await jobQueue.startJob(jobId) @@ -795,66 +757,30 @@ export async function processPolledWebhookEvent( } if (isPollingWebhookProvider(payload.provider) && !shouldExecuteInline()) { - const jobId = isBullMQEnabled() - ? await enqueueWorkspaceDispatch({ - id: executionId, - workspaceId: foundWorkflow.workspaceId, - lane: 'runtime', - queueName: 'webhook-execution', - bullmqJobName: 'webhook-execution', - bullmqPayload: createBullMQJobData(payload, { - workflowId: foundWorkflow.id, - userId: actorUserId, - correlation, - }), - metadata: { - workflowId: foundWorkflow.id, - userId: actorUserId, - correlation, - }, - }) - : await (await getJobQueue()).enqueue('webhook-execution', payload, { - metadata: { - workflowId: foundWorkflow.id, - workspaceId: foundWorkflow.workspaceId, - userId: actorUserId, - correlation, - }, - }) + const jobId = await (await getJobQueue()).enqueue('webhook-execution', payload, { + metadata: { + workflowId: foundWorkflow.id, + workspaceId: foundWorkflow.workspaceId, + userId: actorUserId, + correlation, + }, + }) logger.info( `[${requestId}] Queued polling webhook execution task ${jobId} for ${provider} webhook via job queue` ) } else { const jobQueue = await getInlineJobQueue() - const jobId = isBullMQEnabled() - ? await enqueueWorkspaceDispatch({ - id: executionId, - workspaceId: foundWorkflow.workspaceId, - lane: 'runtime', - queueName: 'webhook-execution', - bullmqJobName: 'webhook-execution', - bullmqPayload: createBullMQJobData(payload, { - workflowId: foundWorkflow.id, - userId: actorUserId, - correlation, - }), - metadata: { - workflowId: foundWorkflow.id, - userId: actorUserId, - correlation, - }, - }) - : await jobQueue.enqueue('webhook-execution', payload, { - metadata: { - workflowId: foundWorkflow.id, - workspaceId: foundWorkflow.workspaceId, - userId: actorUserId, - correlation, - }, - }) + const jobId = await jobQueue.enqueue('webhook-execution', payload, { + metadata: { + workflowId: foundWorkflow.id, + workspaceId: foundWorkflow.workspaceId, + userId: actorUserId, + correlation, + }, + }) logger.info(`[${requestId}] Queued ${provider} webhook execution ${jobId} via inline backend`) - if (!isBullMQEnabled()) { + if (shouldExecuteInline()) { void (async () => { try { await jobQueue.startJob(jobId) @@ -884,10 +810,6 @@ export async function processPolledWebhookEvent( return { success: true } } catch (error: unknown) { - if (error instanceof DispatchQueueFullError) { - logger.warn(`[${requestId}] Dispatch queue full for polled webhook: ${error.message}`) - return { success: false, error: 'Service temporarily at capacity', statusCode: 503 } - } logger.error(`[${requestId}] Failed to process polled webhook event:`, error) return { success: false, error: 'Internal server error', statusCode: 500 } } finally { diff --git a/apps/sim/package.json b/apps/sim/package.json index 525c8e6b97..77b362b0e4 100644 --- a/apps/sim/package.json +++ b/apps/sim/package.json @@ -11,7 +11,7 @@ "dev": "next dev --port 3000", "dev:webpack": "next dev --webpack", "dev:sockets": "bun run socket/index.ts", - "dev:full": "bunx concurrently -n \"App,Realtime,Worker\" -c \"cyan,magenta,yellow\" \"bun run dev\" \"bun run dev:sockets\" \"bun run worker\"", + "dev:full": "bunx concurrently -n \"App,Realtime\" -c \"cyan,magenta\" \"bun run dev\" \"bun run dev:sockets\"", "load:workflow": "bun run load:workflow:baseline", "load:workflow:baseline": "BASE_URL=${BASE_URL:-http://localhost:3000} WARMUP_DURATION=${WARMUP_DURATION:-10} WARMUP_RATE=${WARMUP_RATE:-2} PEAK_RATE=${PEAK_RATE:-8} HOLD_DURATION=${HOLD_DURATION:-20} bunx artillery run scripts/load/workflow-concurrency.yml", "load:workflow:waves": "BASE_URL=${BASE_URL:-http://localhost:3000} WAVE_ONE_DURATION=${WAVE_ONE_DURATION:-10} WAVE_ONE_RATE=${WAVE_ONE_RATE:-6} QUIET_DURATION=${QUIET_DURATION:-5} WAVE_TWO_DURATION=${WAVE_TWO_DURATION:-15} WAVE_TWO_RATE=${WAVE_TWO_RATE:-8} WAVE_THREE_DURATION=${WAVE_THREE_DURATION:-20} WAVE_THREE_RATE=${WAVE_THREE_RATE:-10} bunx artillery run scripts/load/workflow-waves.yml", @@ -20,7 +20,6 @@ "build:pptx-worker": "bun build ./lib/execution/pptx-worker.cjs --target=node --format=cjs --outfile ./dist/pptx-worker.cjs", "build:doc-worker": "bun build ./lib/execution/doc-worker.cjs --target=node --format=cjs --outfile ./dist/doc-worker.cjs", "start": "next start", - "worker": "NODE_ENV=production bun run worker/index.ts", "prepare": "cd ../.. && bun husky", "test": "vitest run", "test:watch": "vitest", @@ -102,7 +101,6 @@ "better-auth-harmony": "1.3.1", "binary-extensions": "^2.0.0", "browser-image-compression": "^2.0.2", - "bullmq": "5.71.0", "chalk": "5.6.2", "chart.js": "4.5.1", "cheerio": "1.1.2", diff --git a/apps/sim/scripts/load/README.md b/apps/sim/scripts/load/README.md index 926f84fad5..43a0897643 100644 --- a/apps/sim/scripts/load/README.md +++ b/apps/sim/scripts/load/README.md @@ -11,14 +11,8 @@ These local-only Artillery scenarios exercise `POST /api/workflows/[id]/execute` The default rates are tuned for these local limits: - `ADMISSION_GATE_MAX_INFLIGHT=500` -- `DISPATCH_MAX_QUEUE_PER_WORKSPACE=1000` -- `DISPATCH_MAX_QUEUE_GLOBAL=50000` -- `WORKSPACE_CONCURRENCY_FREE=5` -- `WORKSPACE_CONCURRENCY_PRO=50` -- `WORKSPACE_CONCURRENCY_TEAM=200` -- `WORKSPACE_CONCURRENCY_ENTERPRISE=200` -That means the defaults are intentionally aimed at forcing queueing for a Free workspace without overwhelming a single local dev server process. +The admission gate caps total in-flight requests per pod. ## Baseline Concurrency @@ -109,5 +103,5 @@ Optional variables: - `load:workflow` is an alias for `load:workflow:baseline` - All scenarios send `x-execution-mode: async` - Artillery output will show request counts and response codes, which is usually enough for quick local verification -- At these defaults, you should observe queueing behavior before you approach `ADMISSION_GATE_MAX_INFLIGHT=500` or `DISPATCH_MAX_QUEUE_PER_WORKSPACE=1000` +- At these defaults, you should see `429` responses when approaching `ADMISSION_GATE_MAX_INFLIGHT=500` - If you still see lots of `429` or `ETIMEDOUT` responses locally, lower the rates again before increasing durations diff --git a/apps/sim/worker/health.ts b/apps/sim/worker/health.ts deleted file mode 100644 index f2a5fea3c1..0000000000 --- a/apps/sim/worker/health.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { createServer } from 'http' -import { createLogger } from '@sim/logger' -import { getRedisClient } from '@/lib/core/config/redis' - -const logger = createLogger('BullMQWorkerHealth') - -export interface WorkerHealthServer { - close: () => Promise -} - -interface WorkerHealthCheck { - redisConnected: boolean - dispatcherLastWakeAt: number -} - -let healthState: WorkerHealthCheck = { - redisConnected: false, - dispatcherLastWakeAt: 0, -} - -export function updateWorkerHealthState(update: Partial): void { - healthState = { ...healthState, ...update } -} - -export function startWorkerHealthServer(port: number): WorkerHealthServer { - const server = createServer((req, res) => { - if (req.method === 'GET' && req.url === '/health') { - const redis = getRedisClient() - const redisConnected = redis !== null - const dispatcherActive = - healthState.dispatcherLastWakeAt > 0 && - Date.now() - healthState.dispatcherLastWakeAt < 30_000 - - const healthy = redisConnected && dispatcherActive - - res.writeHead(healthy ? 200 : 503, { 'Content-Type': 'application/json' }) - res.end( - JSON.stringify({ - ok: healthy, - redis: redisConnected, - dispatcher: dispatcherActive, - lastWakeAgoMs: healthState.dispatcherLastWakeAt - ? Date.now() - healthState.dispatcherLastWakeAt - : null, - }) - ) - return - } - - if (req.method === 'GET' && req.url === '/health/live') { - res.writeHead(200, { 'Content-Type': 'application/json' }) - res.end(JSON.stringify({ ok: true })) - return - } - - res.writeHead(404, { 'Content-Type': 'application/json' }) - res.end(JSON.stringify({ error: 'Not found' })) - }) - - server.listen(port, '0.0.0.0', () => { - logger.info(`Worker health server listening on port ${port}`) - }) - - return { - close: () => - new Promise((resolve, reject) => { - server.close((error) => { - if (error) { - reject(error) - return - } - - resolve() - }) - }), - } -} diff --git a/apps/sim/worker/index.ts b/apps/sim/worker/index.ts deleted file mode 100644 index e5459af3f3..0000000000 --- a/apps/sim/worker/index.ts +++ /dev/null @@ -1,201 +0,0 @@ -import { createLogger } from '@sim/logger' -import { Worker } from 'bullmq' -import { - getBullMQConnectionOptions, - isBullMQEnabled, - KNOWLEDGE_CONNECTOR_SYNC_QUEUE, - KNOWLEDGE_DOCUMENT_PROCESSING_QUEUE, - MOTHERSHIP_JOB_EXECUTION_QUEUE, - WORKSPACE_NOTIFICATION_DELIVERY_QUEUE, -} from '@/lib/core/bullmq' -import { wakeWorkspaceDispatcher } from '@/lib/core/workspace-dispatch' -import { sweepPendingNotificationDeliveries } from '@/background/workspace-notification-delivery' -import { startWorkerHealthServer, updateWorkerHealthState } from '@/worker/health' -import { processKnowledgeConnectorSync } from '@/worker/processors/knowledge-connector-sync' -import { processKnowledgeDocument } from '@/worker/processors/knowledge-document-processing' -import { processMothershipJobExecution } from '@/worker/processors/mothership-job-execution' -import { processResume } from '@/worker/processors/resume' -import { processSchedule } from '@/worker/processors/schedule' -import { processWebhook } from '@/worker/processors/webhook' -import { processWorkflow } from '@/worker/processors/workflow' -import { processWorkspaceNotificationDelivery } from '@/worker/processors/workspace-notification-delivery' - -const logger = createLogger('BullMQWorker') - -const DEFAULT_WORKER_PORT = 3001 -const DEFAULT_WORKFLOW_CONCURRENCY = 50 -const DEFAULT_WEBHOOK_CONCURRENCY = 30 -const DEFAULT_SCHEDULE_CONCURRENCY = 20 -const DEFAULT_RESUME_CONCURRENCY = 20 -const DEFAULT_MOTHERSHIP_JOB_CONCURRENCY = 10 -const DEFAULT_CONNECTOR_SYNC_CONCURRENCY = 5 -const DEFAULT_DOCUMENT_PROCESSING_CONCURRENCY = 20 -const DEFAULT_NOTIFICATION_DELIVERY_CONCURRENCY = 10 -const DISPATCHER_WAKE_INTERVAL_MS = 5_000 -const NOTIFICATION_SWEEPER_INTERVAL_MS = 10_000 - -function parseWorkerNumber(value: string | undefined, fallback: number): number { - const parsed = Number.parseInt(value ?? '', 10) - return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback -} - -async function main() { - const workerPort = parseWorkerNumber(process.env.WORKER_PORT, DEFAULT_WORKER_PORT) - const healthServer = startWorkerHealthServer(workerPort) - - if (!isBullMQEnabled()) { - logger.warn('BullMQ worker started without REDIS_URL; worker will remain idle') - - const shutdownWithoutRedis = async () => { - await healthServer.close() - process.exit(0) - } - - process.on('SIGINT', shutdownWithoutRedis) - process.on('SIGTERM', shutdownWithoutRedis) - return - } - - const connection = getBullMQConnectionOptions() - - const workflowWorker = new Worker('workflow-execution', processWorkflow, { - connection, - concurrency: parseWorkerNumber( - process.env.WORKER_CONCURRENCY_WORKFLOW, - DEFAULT_WORKFLOW_CONCURRENCY - ), - }) - - const webhookWorker = new Worker('webhook-execution', processWebhook, { - connection, - concurrency: parseWorkerNumber( - process.env.WORKER_CONCURRENCY_WEBHOOK, - DEFAULT_WEBHOOK_CONCURRENCY - ), - }) - - const scheduleWorker = new Worker('schedule-execution', processSchedule, { - connection, - concurrency: parseWorkerNumber( - process.env.WORKER_CONCURRENCY_SCHEDULE, - DEFAULT_SCHEDULE_CONCURRENCY - ), - }) - - const resumeWorker = new Worker('resume-execution', processResume, { - connection, - concurrency: parseWorkerNumber( - process.env.WORKER_CONCURRENCY_RESUME, - DEFAULT_RESUME_CONCURRENCY - ), - }) - - const mothershipJobWorker = new Worker( - MOTHERSHIP_JOB_EXECUTION_QUEUE, - processMothershipJobExecution, - { - connection, - concurrency: parseWorkerNumber( - process.env.WORKER_CONCURRENCY_MOTHERSHIP_JOB, - DEFAULT_MOTHERSHIP_JOB_CONCURRENCY - ), - } - ) - - const connectorSyncWorker = new Worker( - KNOWLEDGE_CONNECTOR_SYNC_QUEUE, - processKnowledgeConnectorSync, - { - connection, - concurrency: parseWorkerNumber( - process.env.WORKER_CONCURRENCY_CONNECTOR_SYNC, - DEFAULT_CONNECTOR_SYNC_CONCURRENCY - ), - } - ) - - const documentProcessingWorker = new Worker( - KNOWLEDGE_DOCUMENT_PROCESSING_QUEUE, - processKnowledgeDocument, - { - connection, - concurrency: parseWorkerNumber( - process.env.WORKER_CONCURRENCY_DOCUMENT_PROCESSING, - DEFAULT_DOCUMENT_PROCESSING_CONCURRENCY - ), - } - ) - - const notificationDeliveryWorker = new Worker( - WORKSPACE_NOTIFICATION_DELIVERY_QUEUE, - processWorkspaceNotificationDelivery, - { - connection, - concurrency: parseWorkerNumber( - process.env.WORKER_CONCURRENCY_NOTIFICATION_DELIVERY, - DEFAULT_NOTIFICATION_DELIVERY_CONCURRENCY - ), - } - ) - - const workers = [ - workflowWorker, - webhookWorker, - scheduleWorker, - resumeWorker, - mothershipJobWorker, - connectorSyncWorker, - documentProcessingWorker, - notificationDeliveryWorker, - ] - - for (const worker of workers) { - worker.on('failed', (job, error) => { - logger.error('BullMQ job failed', { - queue: worker.name, - jobId: job?.id, - name: job?.name, - error: error.message, - }) - }) - } - - const dispatcherWakeInterval = setInterval(() => { - void wakeWorkspaceDispatcher() - .then(() => { - updateWorkerHealthState({ dispatcherLastWakeAt: Date.now() }) - }) - .catch((error) => { - logger.error('Periodic workspace dispatcher wake failed', { error }) - }) - }, DISPATCHER_WAKE_INTERVAL_MS) - dispatcherWakeInterval.unref() - - const notificationSweeperInterval = setInterval(() => { - void sweepPendingNotificationDeliveries().catch((error) => { - logger.error('Pending notification sweeper failed', { error }) - }) - }, NOTIFICATION_SWEEPER_INTERVAL_MS) - notificationSweeperInterval.unref() - - const shutdown = async () => { - logger.info('Shutting down BullMQ worker') - - clearInterval(dispatcherWakeInterval) - clearInterval(notificationSweeperInterval) - await Promise.allSettled(workers.map((worker) => worker.close())) - await healthServer.close() - - process.exit(0) - } - - process.on('SIGINT', shutdown) - process.on('SIGTERM', shutdown) -} - -main().catch((error) => { - logger.error('Failed to start BullMQ worker', { - error: error instanceof Error ? error.message : String(error), - }) - process.exit(1) -}) diff --git a/apps/sim/worker/processors/knowledge-connector-sync.ts b/apps/sim/worker/processors/knowledge-connector-sync.ts deleted file mode 100644 index 9a504ebde0..0000000000 --- a/apps/sim/worker/processors/knowledge-connector-sync.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { createLogger } from '@sim/logger' -import type { Job } from 'bullmq' -import type { BullMQJobData } from '@/lib/core/bullmq' -import { runDispatchedJob } from '@/lib/core/workspace-dispatch' -import { executeSync } from '@/lib/knowledge/connectors/sync-engine' -import type { ConnectorSyncPayload } from '@/background/knowledge-connector-sync' - -const logger = createLogger('BullMQKnowledgeConnectorSync') - -export async function processKnowledgeConnectorSync(job: Job>) { - const { connectorId, fullSync } = job.data.payload - const isFinalAttempt = job.attemptsMade + 1 >= (job.opts.attempts ?? 1) - - logger.info('Processing connector sync job', { - jobId: job.id, - connectorId, - }) - - return runDispatchedJob(job.data.metadata, () => executeSync(connectorId, { fullSync }), { - isFinalAttempt, - }) -} diff --git a/apps/sim/worker/processors/knowledge-document-processing.ts b/apps/sim/worker/processors/knowledge-document-processing.ts deleted file mode 100644 index 74fff94fb7..0000000000 --- a/apps/sim/worker/processors/knowledge-document-processing.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { createLogger } from '@sim/logger' -import type { Job } from 'bullmq' -import type { BullMQJobData } from '@/lib/core/bullmq' -import { runDispatchedJob } from '@/lib/core/workspace-dispatch' -import { type DocumentJobData, processDocumentAsync } from '@/lib/knowledge/documents/service' - -const logger = createLogger('BullMQKnowledgeDocumentProcessing') - -export async function processKnowledgeDocument(job: Job>) { - const { knowledgeBaseId, documentId, docData, processingOptions } = job.data.payload - const isFinalAttempt = job.attemptsMade + 1 >= (job.opts.attempts ?? 1) - - logger.info('Processing knowledge document job', { - jobId: job.id, - knowledgeBaseId, - documentId, - }) - - await runDispatchedJob( - job.data.metadata, - () => processDocumentAsync(knowledgeBaseId, documentId, docData, processingOptions), - { - isFinalAttempt, - } - ) -} diff --git a/apps/sim/worker/processors/mothership-job-execution.ts b/apps/sim/worker/processors/mothership-job-execution.ts deleted file mode 100644 index d598039438..0000000000 --- a/apps/sim/worker/processors/mothership-job-execution.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { createLogger } from '@sim/logger' -import type { Job } from 'bullmq' -import type { BullMQJobData } from '@/lib/core/bullmq' -import { runDispatchedJob } from '@/lib/core/workspace-dispatch' -import { executeJobInline, type JobExecutionPayload } from '@/background/schedule-execution' - -const logger = createLogger('BullMQMothershipJobExecution') - -export async function processMothershipJobExecution(job: Job>) { - const isFinalAttempt = job.attemptsMade + 1 >= (job.opts.attempts ?? 1) - - logger.info('Processing mothership scheduled job', { - jobId: job.id, - scheduleId: job.data.payload.scheduleId, - }) - - await runDispatchedJob(job.data.metadata, () => executeJobInline(job.data.payload), { - isFinalAttempt, - }) -} diff --git a/apps/sim/worker/processors/resume.ts b/apps/sim/worker/processors/resume.ts deleted file mode 100644 index d5dba569bb..0000000000 --- a/apps/sim/worker/processors/resume.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { createLogger } from '@sim/logger' -import type { Job } from 'bullmq' -import type { BullMQJobData } from '@/lib/core/bullmq' -import { runDispatchedJob } from '@/lib/core/workspace-dispatch' -import { executeResumeJob, type ResumeExecutionPayload } from '@/background/resume-execution' - -const logger = createLogger('BullMQResumeProcessor') - -export async function processResume(job: Job>) { - const { payload } = job.data - const isFinalAttempt = job.attemptsMade + 1 >= (job.opts.attempts ?? 1) - - logger.info('Processing resume execution job', { - jobId: job.id, - resumeExecutionId: payload.resumeExecutionId, - workflowId: payload.workflowId, - }) - - return runDispatchedJob(job.data.metadata, () => executeResumeJob(payload), { - isFinalAttempt, - }) -} diff --git a/apps/sim/worker/processors/schedule.ts b/apps/sim/worker/processors/schedule.ts deleted file mode 100644 index 78f4cde7d7..0000000000 --- a/apps/sim/worker/processors/schedule.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { createLogger } from '@sim/logger' -import type { Job } from 'bullmq' -import type { BullMQJobData } from '@/lib/core/bullmq' -import { runDispatchedJob } from '@/lib/core/workspace-dispatch' -import { executeScheduleJob, type ScheduleExecutionPayload } from '@/background/schedule-execution' - -const logger = createLogger('BullMQScheduleProcessor') - -export async function processSchedule(job: Job>) { - const { payload } = job.data - const isFinalAttempt = job.attemptsMade + 1 >= (job.opts.attempts ?? 1) - - logger.info('Processing schedule job', { - jobId: job.id, - name: job.name, - }) - - return runDispatchedJob(job.data.metadata, () => executeScheduleJob(payload), { - isFinalAttempt, - }) -} diff --git a/apps/sim/worker/processors/webhook.ts b/apps/sim/worker/processors/webhook.ts deleted file mode 100644 index da61aede1c..0000000000 --- a/apps/sim/worker/processors/webhook.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { createLogger } from '@sim/logger' -import type { Job } from 'bullmq' -import type { BullMQJobData } from '@/lib/core/bullmq' -import { runDispatchedJob } from '@/lib/core/workspace-dispatch' -import { executeWebhookJob, type WebhookExecutionPayload } from '@/background/webhook-execution' - -const logger = createLogger('BullMQWebhookProcessor') - -export async function processWebhook(job: Job>) { - const { payload } = job.data - const isFinalAttempt = job.attemptsMade + 1 >= (job.opts.attempts ?? 1) - - logger.info('Processing webhook job', { - jobId: job.id, - name: job.name, - }) - - return runDispatchedJob(job.data.metadata, () => executeWebhookJob(payload), { - isFinalAttempt, - }) -} diff --git a/apps/sim/worker/processors/workflow.ts b/apps/sim/worker/processors/workflow.ts deleted file mode 100644 index 8648e76b55..0000000000 --- a/apps/sim/worker/processors/workflow.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { createLogger } from '@sim/logger' -import type { Job } from 'bullmq' -import type { BullMQJobData } from '@/lib/core/bullmq' -import { runDispatchedJob } from '@/lib/core/workspace-dispatch' -import { - DIRECT_WORKFLOW_JOB_NAME, - executeQueuedWorkflowJob, - type QueuedWorkflowExecutionPayload, -} from '@/lib/workflows/executor/queued-workflow-execution' -import { executeWorkflowJob, type WorkflowExecutionPayload } from '@/background/workflow-execution' - -const logger = createLogger('BullMQWorkflowProcessor') - -type WorkflowQueueJobData = - | BullMQJobData - | BullMQJobData - -function isDirectWorkflowJob( - job: Job -): job is Job> { - return job.name === DIRECT_WORKFLOW_JOB_NAME -} - -function isBackgroundWorkflowJob( - job: Job -): job is Job> { - return job.name !== DIRECT_WORKFLOW_JOB_NAME -} - -export async function processWorkflow(job: Job) { - const isFinalAttempt = job.attemptsMade + 1 >= (job.opts.attempts ?? 1) - - logger.info('Processing workflow job', { - jobId: job.id, - name: job.name, - }) - - if (isDirectWorkflowJob(job)) { - return runDispatchedJob(job.data.metadata, () => executeQueuedWorkflowJob(job.data.payload), { - isFinalAttempt, - }) - } - - if (isBackgroundWorkflowJob(job)) { - return runDispatchedJob(job.data.metadata, () => executeWorkflowJob(job.data.payload), { - isFinalAttempt, - }) - } - - throw new Error(`Unsupported workflow job type: ${job.name}`) -} diff --git a/apps/sim/worker/processors/workspace-notification-delivery.ts b/apps/sim/worker/processors/workspace-notification-delivery.ts deleted file mode 100644 index 7b6dbbc6da..0000000000 --- a/apps/sim/worker/processors/workspace-notification-delivery.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { createLogger } from '@sim/logger' -import type { Job } from 'bullmq' -import type { BullMQJobData } from '@/lib/core/bullmq' -import { runDispatchedJob } from '@/lib/core/workspace-dispatch' -import { - executeNotificationDelivery, - type NotificationDeliveryParams, -} from '@/background/workspace-notification-delivery' - -const logger = createLogger('BullMQWorkspaceNotificationDelivery') - -export async function processWorkspaceNotificationDelivery( - job: Job> -) { - const isFinalAttempt = job.attemptsMade + 1 >= (job.opts.attempts ?? 1) - - logger.info('Processing workspace notification delivery job', { - jobId: job.id, - deliveryId: job.data.payload.deliveryId, - }) - - const result = await runDispatchedJob( - job.data.metadata, - () => executeNotificationDelivery(job.data.payload), - { - isFinalAttempt, - } - ) - - // Retry scheduling is persisted in the notification delivery row and - // rehydrated by the periodic sweeper, which makes retries crash-safe. -} diff --git a/bun.lock b/bun.lock index 8335f0b768..beb05b2ffc 100644 --- a/bun.lock +++ b/bun.lock @@ -124,7 +124,6 @@ "better-auth-harmony": "1.3.1", "binary-extensions": "^2.0.0", "browser-image-compression": "^2.0.2", - "bullmq": "5.71.0", "chalk": "5.6.2", "chart.js": "4.5.1", "cheerio": "1.1.2", @@ -870,18 +869,6 @@ "@mongodb-js/saslprep": ["@mongodb-js/saslprep@1.4.6", "", { "dependencies": { "sparse-bitfield": "^3.0.3" } }, "sha512-y+x3H1xBZd38n10NZF/rEBlvDOOMQ6LKUTHqr8R9VkJ+mmQOYtJFxIlkkK8fZrtOiL6VixbOBWMbZGBdal3Z1g=="], - "@msgpackr-extract/msgpackr-extract-darwin-arm64": ["@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3", "", { "os": "darwin", "cpu": "arm64" }, "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw=="], - - "@msgpackr-extract/msgpackr-extract-darwin-x64": ["@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3", "", { "os": "darwin", "cpu": "x64" }, "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw=="], - - "@msgpackr-extract/msgpackr-extract-linux-arm": ["@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3", "", { "os": "linux", "cpu": "arm" }, "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw=="], - - "@msgpackr-extract/msgpackr-extract-linux-arm64": ["@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3", "", { "os": "linux", "cpu": "arm64" }, "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg=="], - - "@msgpackr-extract/msgpackr-extract-linux-x64": ["@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3", "", { "os": "linux", "cpu": "x64" }, "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg=="], - - "@msgpackr-extract/msgpackr-extract-win32-x64": ["@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3", "", { "os": "win32", "cpu": "x64" }, "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ=="], - "@napi-rs/canvas": ["@napi-rs/canvas@0.1.97", "", { "optionalDependencies": { "@napi-rs/canvas-android-arm64": "0.1.97", "@napi-rs/canvas-darwin-arm64": "0.1.97", "@napi-rs/canvas-darwin-x64": "0.1.97", "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.97", "@napi-rs/canvas-linux-arm64-gnu": "0.1.97", "@napi-rs/canvas-linux-arm64-musl": "0.1.97", "@napi-rs/canvas-linux-riscv64-gnu": "0.1.97", "@napi-rs/canvas-linux-x64-gnu": "0.1.97", "@napi-rs/canvas-linux-x64-musl": "0.1.97", "@napi-rs/canvas-win32-arm64-msvc": "0.1.97", "@napi-rs/canvas-win32-x64-msvc": "0.1.97" } }, "sha512-8cFniXvrIEnVwuNSRCW9wirRZbHvrD3JVujdS2P5n5xiJZNZMOZcfOvJ1pb66c7jXMKHHglJEDVJGbm8XWFcXQ=="], "@napi-rs/canvas-android-arm64": ["@napi-rs/canvas-android-arm64@0.1.97", "", { "os": "android", "cpu": "arm64" }, "sha512-V1c/WVw+NzH8vk7ZK/O8/nyBSCQimU8sfMsB/9qeSvdkGKNU7+mxy/bIF0gTgeBFmHpj30S4E9WHMSrxXGQuVQ=="], @@ -1890,8 +1877,6 @@ "buildcheck": ["buildcheck@0.0.7", "", {}, "sha512-lHblz4ahamxpTmnsk+MNTRWsjYKv965MwOrSJyeD588rR3Jcu7swE+0wN5F+PbL5cjgu/9ObkhfzEPuofEMwLA=="], - "bullmq": ["bullmq@5.71.0", "", { "dependencies": { "cron-parser": "4.9.0", "ioredis": "5.9.3", "msgpackr": "1.11.5", "node-abort-controller": "3.1.1", "semver": "7.7.4", "tslib": "2.8.1", "uuid": "11.1.0" } }, "sha512-aeNWh4drsafSKnAJeiNH/nZP/5O8ZdtdMbnOPZmpjXj7NZUP5YC901U3bIH41iZValm7d1i3c34ojv7q31m30w=="], - "bytes": ["bytes@3.1.2", "", {}, "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="], "c12": ["c12@3.1.0", "", { "dependencies": { "chokidar": "^4.0.3", "confbox": "^0.2.2", "defu": "^6.1.4", "dotenv": "^16.6.1", "exsolve": "^1.0.7", "giget": "^2.0.0", "jiti": "^2.4.2", "ohash": "^2.0.11", "pathe": "^2.0.3", "perfect-debounce": "^1.0.0", "pkg-types": "^2.2.0", "rc9": "^2.1.2" }, "peerDependencies": { "magicast": "^0.3.5" }, "optionalPeers": ["magicast"] }, "sha512-uWoS8OU1MEIsOv8p/5a82c3H31LsWVR5qiyXVfBNOzfffjUWtPnhAb4BYI2uG2HfGmZmFjCtui5XNWaps+iFuw=="], @@ -2036,8 +2021,6 @@ "critters": ["critters@0.0.25", "", { "dependencies": { "chalk": "^4.1.0", "css-select": "^5.1.0", "dom-serializer": "^2.0.0", "domhandler": "^5.0.2", "htmlparser2": "^8.0.2", "postcss": "^8.4.23", "postcss-media-query-parser": "^0.2.3" } }, "sha512-ROF/tjJyyRdM8/6W0VqoN5Ql05xAGnkf5b7f3sTEl1bI5jTQQf8O918RD/V9tEb9pRY/TKcvJekDbJtniHyPtQ=="], - "cron-parser": ["cron-parser@4.9.0", "", { "dependencies": { "luxon": "^3.2.1" } }, "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q=="], - "croner": ["croner@9.1.0", "", {}, "sha512-p9nwwR4qyT5W996vBZhdvBCnMhicY5ytZkR4D1Xj0wuTDEiMnjwR57Q3RXYY/s0EpX6Ay3vgIcfaR+ewGHsi+g=="], "cronstrue": ["cronstrue@3.3.0", "", { "bin": { "cronstrue": "bin/cli.js" } }, "sha512-iwJytzJph1hosXC09zY8F5ACDJKerr0h3/2mOxg9+5uuFObYlgK0m35uUPk4GCvhHc2abK7NfnR9oMqY0qZFAg=="], @@ -2842,8 +2825,6 @@ "lucide-react": ["lucide-react@0.511.0", "", { "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-VK5a2ydJ7xm8GvBeKLS9mu1pVK6ucef9780JVUjw6bAjJL/QXnd4Y0p7SPeOUMC27YhzNCZvm5d/QX0Tp3rc0w=="], - "luxon": ["luxon@3.7.2", "", {}, "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew=="], - "magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="], "magicast": ["magicast@0.3.5", "", { "dependencies": { "@babel/parser": "^7.25.4", "@babel/types": "^7.25.4", "source-map-js": "^1.2.0" } }, "sha512-L0WhttDl+2BOsybvEOLK7fW3UA0OQ0IQ2d6Zl2x/a6vVRs3bAY0ECOSHHeL5jD+SbOpOCUEi0y1DgHEn9Qn1AQ=="], @@ -3036,10 +3017,6 @@ "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], - "msgpackr": ["msgpackr@1.11.5", "", { "optionalDependencies": { "msgpackr-extract": "^3.0.2" } }, "sha512-UjkUHN0yqp9RWKy0Lplhh+wlpdt9oQBYgULZOiFhV3VclSF1JnSQWZ5r9gORQlNYaUKQoR8itv7g7z1xDDuACA=="], - - "msgpackr-extract": ["msgpackr-extract@3.0.3", "", { "dependencies": { "node-gyp-build-optional-packages": "5.2.2" }, "optionalDependencies": { "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3", "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3", "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3", "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3", "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3", "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3" }, "bin": { "download-msgpackr-prebuilds": "bin/download-prebuilds.js" } }, "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA=="], - "mustache": ["mustache@4.2.0", "", { "bin": { "mustache": "bin/mustache" } }, "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ=="], "mute-stream": ["mute-stream@0.0.8", "", {}, "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA=="], @@ -3080,8 +3057,6 @@ "node-abi": ["node-abi@3.89.0", "", { "dependencies": { "semver": "^7.3.5" } }, "sha512-6u9UwL0HlAl21+agMN3YAMXcKByMqwGx+pq+P76vii5f7hTPtKDp08/H9py6DY+cfDw7kQNTGEj/rly3IgbNQA=="], - "node-abort-controller": ["node-abort-controller@3.1.1", "", {}, "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ=="], - "node-domexception": ["node-domexception@1.0.0", "", {}, "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="], "node-ensure": ["node-ensure@0.0.0", "", {}, "sha512-DRI60hzo2oKN1ma0ckc6nQWlHU69RH6xN0sjQTjMpChPfTYvKZdcQFfdYK2RWbJcKyUizSIy/l8OTGxMAM1QDw=="], @@ -3092,8 +3067,6 @@ "node-gyp-build": ["node-gyp-build@4.8.4", "", { "bin": { "node-gyp-build": "bin.js", "node-gyp-build-optional": "optional.js", "node-gyp-build-test": "build-test.js" } }, "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ=="], - "node-gyp-build-optional-packages": ["node-gyp-build-optional-packages@5.2.2", "", { "dependencies": { "detect-libc": "^2.0.1" }, "bin": { "node-gyp-build-optional-packages": "bin.js", "node-gyp-build-optional-packages-optional": "optional.js", "node-gyp-build-optional-packages-test": "build-test.js" } }, "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw=="], - "node-int64": ["node-int64@0.4.0", "", {}, "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw=="], "node-readable-to-web-readable-stream": ["node-readable-to-web-readable-stream@0.4.2", "", {}, "sha512-/cMZNI34v//jUTrI+UIo4ieHAB5EZRY/+7OmXZgBxaWBMcW2tGdceIw06RFxWxrKZ5Jp3sI2i5TsRo+CBhtVLQ=="], @@ -4730,8 +4703,6 @@ "body-parser/iconv-lite": ["iconv-lite@0.7.1", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-2Tth85cXwGFHfvRgZWszZSvdo+0Xsqmw8k8ZwxScfcBneNUraK+dxRxRm24nszx80Y0TVio8kKLt5sLE7ZCLlw=="], - "bullmq/ioredis": ["ioredis@5.9.3", "", { "dependencies": { "@ioredis/commands": "1.5.0", "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", "denque": "^2.1.0", "lodash.defaults": "^4.2.0", "lodash.isarguments": "^3.1.0", "redis-errors": "^1.2.0", "redis-parser": "^3.0.0", "standard-as-callback": "^2.1.0" } }, "sha512-VI5tMCdeoxZWU5vjHWsiE/Su76JGhBvWF1MJnV9ZtGltHk9BmD48oDq8Tj8haZ85aceXZMxLNDQZRVo5QKNgXA=="], - "c12/chokidar": ["chokidar@4.0.3", "", { "dependencies": { "readdirp": "^4.0.1" } }, "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA=="], "c12/confbox": ["confbox@0.2.4", "", {}, "sha512-ysOGlgTFbN2/Y6Cg3Iye8YKulHw+R2fNXHrgSmXISQdMnomY6eNDprVdW9R5xBguEqI954+S6709UyiO7B+6OQ=="], @@ -5358,8 +5329,6 @@ "bl/readable-stream/string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], - "bullmq/ioredis/@ioredis/commands": ["@ioredis/commands@1.5.0", "", {}, "sha512-eUgLqrMf8nJkZxT24JvVRrQya1vZkQh8BBeYNwGDqa5I0VUi8ACx7uFvAaLxintokpTenkK6DASvo/bvNbBGow=="], - "c12/chokidar/readdirp": ["readdirp@4.1.2", "", {}, "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg=="], "cheerio/htmlparser2/entities": ["entities@7.0.1", "", {}, "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA=="], diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 8b874acee8..335fb0c33a 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -67,36 +67,6 @@ services: retries: 3 start_period: 10s - sim-worker: - build: - context: . - dockerfile: docker/app.Dockerfile - command: ['bun', 'apps/sim/worker/index.ts'] - restart: unless-stopped - deploy: - resources: - limits: - memory: 4G - environment: - - NODE_ENV=development - - DATABASE_URL=postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@db:5432/${POSTGRES_DB:-simstudio} - - REDIS_URL=${REDIS_URL:-} - - ENCRYPTION_KEY=${ENCRYPTION_KEY:-dev-encryption-key-at-least-32-chars} - - API_ENCRYPTION_KEY=${API_ENCRYPTION_KEY:-} - - INTERNAL_API_SECRET=${INTERNAL_API_SECRET:-dev-internal-api-secret-min-32-chars} - - WORKER_PORT=3001 - depends_on: - db: - condition: service_healthy - migrations: - condition: service_completed_successfully - healthcheck: - test: ['CMD', 'curl', '-fsS', 'http://127.0.0.1:3001/health/live'] - interval: 90s - timeout: 5s - retries: 3 - start_period: 10s - migrations: build: context: . diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index e7000f0688..cbb667b1c5 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -24,8 +24,6 @@ services: - SOCKET_SERVER_URL=${SOCKET_SERVER_URL:-http://realtime:3002} - NEXT_PUBLIC_SOCKET_URL=${NEXT_PUBLIC_SOCKET_URL:-http://localhost:3002} - ADMISSION_GATE_MAX_INFLIGHT=${ADMISSION_GATE_MAX_INFLIGHT:-500} - - DISPATCH_MAX_QUEUE_PER_WORKSPACE=${DISPATCH_MAX_QUEUE_PER_WORKSPACE:-1000} - - DISPATCH_MAX_QUEUE_GLOBAL=${DISPATCH_MAX_QUEUE_GLOBAL:-50000} depends_on: db: condition: service_healthy @@ -40,43 +38,6 @@ services: retries: 3 start_period: 10s - sim-worker: - image: ghcr.io/simstudioai/simstudio:latest - command: ['bun', 'apps/sim/worker/index.ts'] - restart: unless-stopped - deploy: - resources: - limits: - memory: 4G - environment: - - NODE_ENV=production - - DATABASE_URL=postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@db:5432/${POSTGRES_DB:-simstudio} - - REDIS_URL=${REDIS_URL:-} - - ENCRYPTION_KEY=${ENCRYPTION_KEY} - - API_ENCRYPTION_KEY=${API_ENCRYPTION_KEY:-} - - INTERNAL_API_SECRET=${INTERNAL_API_SECRET} - - WORKER_PORT=${WORKER_PORT:-3001} - - WORKER_CONCURRENCY_WORKFLOW=${WORKER_CONCURRENCY_WORKFLOW:-50} - - WORKER_CONCURRENCY_WEBHOOK=${WORKER_CONCURRENCY_WEBHOOK:-30} - - WORKER_CONCURRENCY_SCHEDULE=${WORKER_CONCURRENCY_SCHEDULE:-20} - - WORKER_CONCURRENCY_MOTHERSHIP_JOB=${WORKER_CONCURRENCY_MOTHERSHIP_JOB:-10} - - WORKER_CONCURRENCY_CONNECTOR_SYNC=${WORKER_CONCURRENCY_CONNECTOR_SYNC:-5} - - WORKER_CONCURRENCY_DOCUMENT_PROCESSING=${WORKER_CONCURRENCY_DOCUMENT_PROCESSING:-20} - - WORKER_CONCURRENCY_NOTIFICATION_DELIVERY=${WORKER_CONCURRENCY_NOTIFICATION_DELIVERY:-10} - - DISPATCH_MAX_QUEUE_PER_WORKSPACE=${DISPATCH_MAX_QUEUE_PER_WORKSPACE:-1000} - - DISPATCH_MAX_QUEUE_GLOBAL=${DISPATCH_MAX_QUEUE_GLOBAL:-50000} - depends_on: - db: - condition: service_healthy - migrations: - condition: service_completed_successfully - healthcheck: - test: ['CMD', 'curl', '-fsS', 'http://127.0.0.1:${WORKER_PORT:-3001}/health/live'] - interval: 90s - timeout: 5s - retries: 3 - start_period: 10s - realtime: image: ghcr.io/simstudioai/realtime:latest restart: unless-stopped diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile index 18e35e8fa0..7fe2507fc9 100644 --- a/docker/app.Dockerfile +++ b/docker/app.Dockerfile @@ -104,11 +104,8 @@ COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/public ./apps/sim/public COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/.next/standalone ./ COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/.next/static ./apps/sim/.next/static -# Copy the full dependency tree and app source so the BullMQ worker can run from source. -# The standalone server continues to use server.js; the worker uses bun on worker/index.ts. -COPY --from=deps --chown=nextjs:nodejs /app/node_modules ./node_modules -COPY --from=builder --chown=nextjs:nodejs /app/apps/sim ./apps/sim -COPY --from=builder --chown=nextjs:nodejs /app/packages ./packages +# Copy blog/author content for runtime filesystem reads (not part of the JS bundle) +COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/content ./apps/sim/content # Copy isolated-vm native module (compiled for Node.js in deps stage) COPY --from=deps --chown=nextjs:nodejs /app/node_modules/isolated-vm ./node_modules/isolated-vm diff --git a/helm/sim/README.md b/helm/sim/README.md index 3507f54349..70d5facb8a 100644 --- a/helm/sim/README.md +++ b/helm/sim/README.md @@ -709,16 +709,9 @@ kubectl create secret generic my-postgresql-secret \ See `examples/values-existing-secret.yaml` for more details. -### Worker and Redis +### Redis -The Helm chart enables the BullMQ worker by default so the deployment topology matches Docker Compose. If `REDIS_URL` is not configured, the worker pod will still start but remain idle and do no queue processing. This is expected. - -Queue-backed API, webhook, and schedule execution requires Redis. Installs without Redis continue to use the inline execution path. If you do not want the worker pod at all, set: - -```yaml -worker: - enabled: false -``` +Redis is optional. When configured via `REDIS_URL`, it enables features like copilot chat stream coordination and caching. Without Redis, the application uses in-memory fallbacks. ### External Secrets Parameters diff --git a/helm/sim/templates/_helpers.tpl b/helm/sim/templates/_helpers.tpl index 915df7cf61..e1bee30491 100644 --- a/helm/sim/templates/_helpers.tpl +++ b/helm/sim/templates/_helpers.tpl @@ -117,22 +117,6 @@ Ollama selector labels app.kubernetes.io/component: ollama {{- end }} -{{/* -Worker specific labels -*/}} -{{- define "sim.worker.labels" -}} -{{ include "sim.labels" . }} -app.kubernetes.io/component: worker -{{- end }} - -{{/* -Worker selector labels -*/}} -{{- define "sim.worker.selectorLabels" -}} -{{ include "sim.selectorLabels" . }} -app.kubernetes.io/component: worker -{{- end }} - {{/* Migrations specific labels */}} diff --git a/helm/sim/templates/deployment-worker.yaml b/helm/sim/templates/deployment-worker.yaml deleted file mode 100644 index 75fa6e88c1..0000000000 --- a/helm/sim/templates/deployment-worker.yaml +++ /dev/null @@ -1,101 +0,0 @@ -{{- if .Values.worker.enabled }} -{{- include "sim.validateSecrets" . }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "sim.fullname" . }}-worker - namespace: {{ .Release.Namespace }} - labels: - {{- include "sim.worker.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.worker.replicaCount }} - selector: - matchLabels: - {{- include "sim.worker.selectorLabels" . | nindent 6 }} - template: - metadata: - annotations: - {{- with .Values.podAnnotations }} - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "sim.worker.selectorLabels" . | nindent 8 }} - {{- with .Values.podLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - {{- with .Values.global.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "sim.serviceAccountName" . }} - {{- include "sim.podSecurityContext" .Values.worker | nindent 6 }} - {{- include "sim.nodeSelector" .Values.worker | nindent 6 }} - {{- include "sim.tolerations" .Values | nindent 6 }} - {{- include "sim.affinity" .Values | nindent 6 }} - containers: - - name: worker - image: {{ include "sim.image" (dict "context" . "image" .Values.worker.image) }} - imagePullPolicy: {{ .Values.worker.image.pullPolicy }} - command: ["bun", "apps/sim/worker/index.ts"] - ports: - - name: health - containerPort: {{ .Values.worker.healthPort }} - protocol: TCP - env: - - name: DATABASE_URL - value: {{ include "sim.databaseUrl" . | quote }} - {{- if .Values.app.env.REDIS_URL }} - - name: REDIS_URL - value: {{ .Values.app.env.REDIS_URL | quote }} - {{- end }} - - name: WORKER_PORT - value: {{ .Values.worker.healthPort | quote }} - {{- if .Values.telemetry.enabled }} - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: "http://{{ include "sim.fullname" . }}-otel-collector:4318" - - name: OTEL_SERVICE_NAME - value: sim-worker - - name: OTEL_SERVICE_VERSION - value: {{ .Chart.AppVersion | quote }} - - name: OTEL_RESOURCE_ATTRIBUTES - value: "service.name=sim-worker,service.version={{ .Chart.AppVersion }},deployment.environment={{ .Values.worker.env.NODE_ENV }}" - {{- end }} - {{- range $key, $value := .Values.worker.env }} - {{- if ne $key "WORKER_PORT" }} - - name: {{ $key }} - value: {{ $value | quote }} - {{- end }} - {{- end }} - {{- with .Values.extraEnvVars }} - {{- toYaml . | nindent 12 }} - {{- end }} - envFrom: - - secretRef: - name: {{ include "sim.appSecretName" . }} - {{- if .Values.postgresql.enabled }} - - secretRef: - name: {{ include "sim.postgresqlSecretName" . }} - {{- else if .Values.externalDatabase.enabled }} - - secretRef: - name: {{ include "sim.externalDbSecretName" . }} - {{- end }} - livenessProbe: - httpGet: - path: /health/live - port: health - initialDelaySeconds: 10 - periodSeconds: 30 - timeoutSeconds: 5 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /health - port: health - initialDelaySeconds: 10 - periodSeconds: 30 - timeoutSeconds: 5 - failureThreshold: 3 - {{- include "sim.resources" .Values.worker | nindent 10 }} - {{- include "sim.securityContext" .Values.worker | nindent 10 }} -{{- end }} diff --git a/helm/sim/values.yaml b/helm/sim/values.yaml index 73e0a0b017..abe7c84b69 100644 --- a/helm/sim/values.yaml +++ b/helm/sim/values.yaml @@ -96,7 +96,7 @@ app: # Optional: API Key Encryption (RECOMMENDED for production) # Generate 64-character hex string using: openssl rand -hex 32 (outputs 64 hex chars = 32 bytes) API_ENCRYPTION_KEY: "" # OPTIONAL - encrypts API keys at rest, must be exactly 64 hex characters, if not set keys stored in plain text - REDIS_URL: "" # OPTIONAL - Redis connection string for BullMQ/workers; can also come from app secret or External Secrets + REDIS_URL: "" # OPTIONAL - Redis connection string for caching/sessions; can also come from app secret or External Secrets # Email & Communication EMAIL_VERIFICATION_ENABLED: "false" # Enable email verification for user registration and login (defaults to false) @@ -139,10 +139,8 @@ app: OLLAMA_URL: "" # Ollama local LLM server URL ELEVENLABS_API_KEY: "" # ElevenLabs API key for text-to-speech in deployed chat - # Admission & Dispatch Queue Configuration + # Admission Gate Configuration ADMISSION_GATE_MAX_INFLIGHT: "500" # Max concurrent in-flight execution requests per pod - DISPATCH_MAX_QUEUE_PER_WORKSPACE: "1000" # Max queued dispatch jobs per workspace - DISPATCH_MAX_QUEUE_GLOBAL: "50000" # Max queued dispatch jobs globally # Rate Limiting Configuration (per minute) RATE_LIMIT_WINDOW_MS: "60000" # Rate limit window duration (1 minute) @@ -386,58 +384,6 @@ realtime: extraVolumes: [] extraVolumeMounts: [] -# BullMQ worker configuration (processes background jobs when Redis is available) -# Uses the same image as the main app with a different command. -# Enabled by default so self-hosted deployments get the same topology as compose. -# Without REDIS_URL the worker starts, logs that it is idle, and does no queue processing. -worker: - # Enable/disable the worker deployment - enabled: true - - # Image configuration (defaults to same image as app) - image: - repository: simstudioai/simstudio - tag: latest - pullPolicy: Always - - # Number of replicas - replicaCount: 1 - - # Health check port (worker exposes a lightweight HTTP health server) - healthPort: 3001 - - # Resource limits and requests - resources: - limits: - memory: "4Gi" - cpu: "1000m" - requests: - memory: "2Gi" - cpu: "500m" - - # Node selector for pod scheduling - nodeSelector: {} - - # Pod security context - podSecurityContext: - fsGroup: 1001 - - # Container security context - securityContext: - runAsNonRoot: true - runAsUser: 1001 - - # Environment variables (worker-specific tuning) - env: - NODE_ENV: "production" - WORKER_CONCURRENCY_WORKFLOW: "50" - WORKER_CONCURRENCY_WEBHOOK: "30" - WORKER_CONCURRENCY_SCHEDULE: "20" - WORKER_CONCURRENCY_MOTHERSHIP_JOB: "10" - WORKER_CONCURRENCY_CONNECTOR_SYNC: "5" - WORKER_CONCURRENCY_DOCUMENT_PROCESSING: "20" - WORKER_CONCURRENCY_NOTIFICATION_DELIVERY: "10" - # Database migrations job configuration migrations: # Enable/disable migrations job @@ -1322,7 +1268,7 @@ externalSecrets: CRON_SECRET: "" # Path to API_ENCRYPTION_KEY in external store (optional) API_ENCRYPTION_KEY: "" - # Path to REDIS_URL in external store (optional, required for worker when not set in app.env) + # Path to REDIS_URL in external store (optional) REDIS_URL: "" # PostgreSQL password (for internal PostgreSQL)