From 6e63eafb79e453b1ac55c5117e9435b0865c9c53 Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 1 Oct 2025 16:37:13 -0700 Subject: [PATCH] improvement(db): remove vercel, remove railway, remove crons, improve DB connection config (#1519) * improvement(db): remove vercel, remove railway, remove crons, improve DB connection config * remove NEXT_PUBLIC_VERCEL_URL * remove db url fallbacks * remove railway & more vercel stuff --------- Co-authored-by: waleed --- .devcontainer/docker-compose.yml | 1 - .../knowledge/hooks/use-knowledge-upload.ts | 1 - apps/sim/lib/auth-client.ts | 14 +---------- apps/sim/lib/auth.ts | 1 - apps/sim/lib/env.ts | 4 ---- apps/sim/lib/security/csp.ts | 22 ++--------------- apps/sim/socket-server/config/socket.ts | 1 - apps/sim/socket-server/database/operations.ts | 3 +-- apps/sim/socket-server/rooms/manager.ts | 3 +-- apps/sim/socket-server/routes/http.ts | 1 - apps/sim/vercel.json | 24 ------------------- bun.lock | 1 - docker/app.Dockerfile | 2 -- package.json | 1 - packages/db/drizzle.config.ts | 2 +- packages/db/index.ts | 23 +++--------------- packages/db/scripts/register-sso-provider.ts | 4 ++-- railway.json | 21 ---------------- 18 files changed, 11 insertions(+), 118 deletions(-) delete mode 100644 apps/sim/vercel.json delete mode 100644 railway.json diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 2701ca305..790d1da0d 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -10,7 +10,6 @@ services: environment: - NODE_ENV=development - DATABASE_URL=postgresql://postgres:postgres@db:5432/simstudio - - POSTGRES_URL=postgresql://postgres:postgres@db:5432/simstudio - BETTER_AUTH_URL=http://localhost:3000 - NEXT_PUBLIC_APP_URL=http://localhost:3000 - BUN_INSTALL_CACHE_DIR=/home/bun/.bun/cache diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts b/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts index 070978d36..fd26bdbf2 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts @@ -89,7 +89,6 @@ const UPLOAD_CONFIG = { RETRY_DELAY: 2000, // Initial retry delay in ms (2 seconds) RETRY_MULTIPLIER: 2, // Standard exponential backoff (2s, 4s, 8s) CHUNK_SIZE: 5 * 1024 * 1024, - VERCEL_MAX_BODY_SIZE: 4.5 * 1024 * 1024, // Vercel's 4.5MB limit DIRECT_UPLOAD_THRESHOLD: 4 * 1024 * 1024, // Files > 4MB must use presigned URLs LARGE_FILE_THRESHOLD: 50 * 1024 * 1024, // Files > 50MB need multipart upload UPLOAD_TIMEOUT: 60000, // 60 second timeout per upload diff --git a/apps/sim/lib/auth-client.ts b/apps/sim/lib/auth-client.ts index fa357631e..718a526c1 100644 --- a/apps/sim/lib/auth-client.ts +++ b/apps/sim/lib/auth-client.ts @@ -14,19 +14,7 @@ import { isBillingEnabled } from '@/lib/environment' import { SessionContext, type SessionHookResult } from '@/lib/session/session-context' export function getBaseURL() { - let baseURL - - if (env.VERCEL_ENV === 'preview') { - baseURL = `https://${getEnv('NEXT_PUBLIC_VERCEL_URL')}` - } else if (env.VERCEL_ENV === 'development') { - baseURL = `https://${getEnv('NEXT_PUBLIC_VERCEL_URL')}` - } else if (env.VERCEL_ENV === 'production') { - baseURL = env.BETTER_AUTH_URL || getEnv('NEXT_PUBLIC_APP_URL') - } else if (env.NODE_ENV === 'development') { - baseURL = getEnv('NEXT_PUBLIC_APP_URL') || env.BETTER_AUTH_URL || 'http://localhost:3000' - } - - return baseURL + return getEnv('NEXT_PUBLIC_APP_URL') || 'http://localhost:3000' } export const client = createAuthClient({ diff --git a/apps/sim/lib/auth.ts b/apps/sim/lib/auth.ts index b83687ef8..5dd170ad9 100644 --- a/apps/sim/lib/auth.ts +++ b/apps/sim/lib/auth.ts @@ -63,7 +63,6 @@ export const auth = betterAuth({ baseURL: getBaseURL(), trustedOrigins: [ env.NEXT_PUBLIC_APP_URL, - ...(env.NEXT_PUBLIC_VERCEL_URL ? [`https://${env.NEXT_PUBLIC_VERCEL_URL}`] : []), ...(env.NEXT_PUBLIC_SOCKET_URL ? [env.NEXT_PUBLIC_SOCKET_URL] : []), ].filter(Boolean), database: drizzleAdapter(db, { diff --git a/apps/sim/lib/env.ts b/apps/sim/lib/env.ts index 11abce5bd..bc6c9c6b7 100644 --- a/apps/sim/lib/env.ts +++ b/apps/sim/lib/env.ts @@ -36,7 +36,6 @@ export const env = createEnv({ // Database & Storage - POSTGRES_URL: z.string().url().optional(), // Alternative PostgreSQL connection string REDIS_URL: z.string().url().optional(), // Redis connection string for caching/sessions // Payment & Billing @@ -99,7 +98,6 @@ export const env = createEnv({ // Infrastructure & Deployment NEXT_RUNTIME: z.string().optional(), // Next.js runtime environment - VERCEL_ENV: z.string().optional(), // Vercel deployment environment DOCKER_BUILD: z.boolean().optional(), // Flag indicating Docker build environment // Background Jobs & Scheduling @@ -244,7 +242,6 @@ export const env = createEnv({ client: { // Core Application URLs - Required for frontend functionality NEXT_PUBLIC_APP_URL: z.string().url(), // Base URL of the application (e.g., https://app.sim.ai) - NEXT_PUBLIC_VERCEL_URL: z.string().optional(), // Vercel deployment URL for preview/production // Client-side Services NEXT_PUBLIC_SOCKET_URL: z.string().url().optional(), // WebSocket server URL for real-time features @@ -296,7 +293,6 @@ export const env = createEnv({ experimental__runtimeEnv: { NEXT_PUBLIC_APP_URL: process.env.NEXT_PUBLIC_APP_URL, - NEXT_PUBLIC_VERCEL_URL: process.env.NEXT_PUBLIC_VERCEL_URL, NEXT_PUBLIC_BLOB_BASE_URL: process.env.NEXT_PUBLIC_BLOB_BASE_URL, NEXT_PUBLIC_BILLING_ENABLED: process.env.NEXT_PUBLIC_BILLING_ENABLED, NEXT_PUBLIC_GOOGLE_CLIENT_ID: process.env.NEXT_PUBLIC_GOOGLE_CLIENT_ID, diff --git a/apps/sim/lib/security/csp.ts b/apps/sim/lib/security/csp.ts index a4d117b03..d41d65677 100644 --- a/apps/sim/lib/security/csp.ts +++ b/apps/sim/lib/security/csp.ts @@ -38,14 +38,6 @@ export const buildTimeCSPDirectives: CSPDirectives = { "'unsafe-eval'", 'https://*.google.com', 'https://apis.google.com', - 'https://*.vercel-scripts.com', - 'https://*.vercel-insights.com', - 'https://vercel.live', - 'https://*.vercel.live', - 'https://vercel.com', - 'https://*.vercel.app', - 'https://vitals.vercel-insights.com', - 'https://b2bjsstore.s3.us-west-2.amazonaws.com', ], 'style-src': ["'self'", "'unsafe-inline'", 'https://fonts.googleapis.com'], @@ -90,8 +82,6 @@ export const buildTimeCSPDirectives: CSPDirectives = { env.NEXT_PUBLIC_SOCKET_URL || 'http://localhost:3002', env.NEXT_PUBLIC_SOCKET_URL?.replace('http://', 'ws://').replace('https://', 'wss://') || 'ws://localhost:3002', - 'https://*.up.railway.app', - 'wss://*.up.railway.app', 'https://api.browser-use.com', 'https://api.exa.ai', 'https://api.firecrawl.dev', @@ -99,16 +89,8 @@ export const buildTimeCSPDirectives: CSPDirectives = { 'https://*.amazonaws.com', 'https://*.s3.amazonaws.com', 'https://*.blob.core.windows.net', - 'https://*.vercel-insights.com', - 'https://vitals.vercel-insights.com', 'https://*.atlassian.com', 'https://*.supabase.co', - 'https://vercel.live', - 'https://*.vercel.live', - 'https://vercel.com', - 'https://*.vercel.app', - 'wss://*.vercel.app', - 'https://pro.ip-api.com', 'https://api.github.com', 'https://github.com/*', ...getHostnameFromUrl(env.NEXT_PUBLIC_BRAND_LOGO_URL), @@ -168,12 +150,12 @@ export function generateRuntimeCSP(): string { return ` default-src 'self'; - script-src 'self' 'unsafe-inline' 'unsafe-eval' https://*.google.com https://apis.google.com https://*.vercel-scripts.com https://*.vercel-insights.com https://vercel.live https://*.vercel.live https://vercel.com https://*.vercel.app https://vitals.vercel-insights.com https://b2bjsstore.s3.us-west-2.amazonaws.com; + script-src 'self' 'unsafe-inline' 'unsafe-eval' https://*.google.com https://apis.google.com; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; img-src 'self' data: blob: https://*.googleusercontent.com https://*.google.com https://*.atlassian.com https://cdn.discordapp.com https://*.githubusercontent.com https://*.public.blob.vercel-storage.com ${brandLogoDomain} ${brandFaviconDomain}; media-src 'self' blob:; font-src 'self' https://fonts.gstatic.com; - connect-src 'self' ${appUrl} ${ollamaUrl} ${socketUrl} ${socketWsUrl} https://*.up.railway.app wss://*.up.railway.app https://api.browser-use.com https://api.exa.ai https://api.firecrawl.dev https://*.googleapis.com https://*.amazonaws.com https://*.s3.amazonaws.com https://*.blob.core.windows.net https://api.github.com https://github.com/* https://*.vercel-insights.com https://vitals.vercel-insights.com https://*.atlassian.com https://*.supabase.co https://vercel.live https://*.vercel.live https://vercel.com https://*.vercel.app wss://*.vercel.app https://pro.ip-api.com ${dynamicDomainsStr}; + connect-src 'self' ${appUrl} ${ollamaUrl} ${socketUrl} ${socketWsUrl} https://api.browser-use.com https://api.exa.ai https://api.firecrawl.dev https://*.googleapis.com https://*.amazonaws.com https://*.s3.amazonaws.com https://*.blob.core.windows.net https://api.github.com https://github.com/* https://*.atlassian.com https://*.supabase.co ${dynamicDomainsStr}; frame-src https://drive.google.com https://docs.google.com https://*.google.com; frame-ancestors 'self'; form-action 'self'; diff --git a/apps/sim/socket-server/config/socket.ts b/apps/sim/socket-server/config/socket.ts index eeb36dbdb..3f6f4a7c1 100644 --- a/apps/sim/socket-server/config/socket.ts +++ b/apps/sim/socket-server/config/socket.ts @@ -12,7 +12,6 @@ const logger = createLogger('SocketIOConfig') function getAllowedOrigins(): string[] { const allowedOrigins = [ env.NEXT_PUBLIC_APP_URL, - env.NEXT_PUBLIC_VERCEL_URL, 'http://localhost:3000', 'http://localhost:3001', ...(env.ALLOWED_ORIGINS?.split(',') || []), diff --git a/apps/sim/socket-server/database/operations.ts b/apps/sim/socket-server/database/operations.ts index 0731b44fa..388d6610a 100644 --- a/apps/sim/socket-server/database/operations.ts +++ b/apps/sim/socket-server/database/operations.ts @@ -9,8 +9,7 @@ import { loadWorkflowFromNormalizedTables } from '@/lib/workflows/db-helpers' const logger = createLogger('SocketDatabase') -// Create dedicated database connection for socket server with optimized settings -const connectionString = env.POSTGRES_URL ?? env.DATABASE_URL +const connectionString = env.DATABASE_URL const socketDb = drizzle( postgres(connectionString, { prepare: false, diff --git a/apps/sim/socket-server/rooms/manager.ts b/apps/sim/socket-server/rooms/manager.ts index 42c752e29..b36ac85a7 100644 --- a/apps/sim/socket-server/rooms/manager.ts +++ b/apps/sim/socket-server/rooms/manager.ts @@ -7,8 +7,7 @@ import type { Server } from 'socket.io' import { env } from '@/lib/env' import { createLogger } from '@/lib/logs/console/logger' -// Create dedicated database connection for room manager -const connectionString = env.POSTGRES_URL ?? env.DATABASE_URL +const connectionString = env.DATABASE_URL const db = drizzle( postgres(connectionString, { prepare: false, diff --git a/apps/sim/socket-server/routes/http.ts b/apps/sim/socket-server/routes/http.ts index 2a3cbd958..0a404abfe 100644 --- a/apps/sim/socket-server/routes/http.ts +++ b/apps/sim/socket-server/routes/http.ts @@ -16,7 +16,6 @@ interface Logger { */ export function createHttpHandler(roomManager: RoomManager, logger: Logger) { return (req: IncomingMessage, res: ServerResponse) => { - // Handle health check for Railway if (req.method === 'GET' && req.url === '/health') { res.writeHead(200, { 'Content-Type': 'application/json' }) res.end( diff --git a/apps/sim/vercel.json b/apps/sim/vercel.json deleted file mode 100644 index 38fb972e4..000000000 --- a/apps/sim/vercel.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "crons": [ - { - "path": "/api/schedules/execute", - "schedule": "*/1 * * * *" - }, - { - "path": "/api/webhooks/poll/gmail", - "schedule": "*/1 * * * *" - }, - { - "path": "/api/webhooks/poll/outlook", - "schedule": "*/1 * * * *" - }, - { - "path": "/api/logs/cleanup", - "schedule": "0 0 * * *" - }, - { - "path": "/api/webhooks/cleanup/idempotency", - "schedule": "0 2 * * *" - } - ] -} diff --git a/bun.lock b/bun.lock index 76869ecb0..2afd31f06 100644 --- a/bun.lock +++ b/bun.lock @@ -6,7 +6,6 @@ "dependencies": { "@linear/sdk": "40.0.0", "@t3-oss/env-nextjs": "0.13.4", - "@vercel/analytics": "1.5.0", "drizzle-orm": "^0.44.5", "mongodb": "6.19.0", "postgres": "^3.4.5", diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile index 1e20f6d7e..a6d012f4c 100644 --- a/docker/app.Dockerfile +++ b/docker/app.Dockerfile @@ -47,9 +47,7 @@ WORKDIR /app # Provide dummy database URLs during image build so server code that imports @sim/db # can be evaluated without crashing. Runtime environments should override these. ARG DATABASE_URL="postgresql://user:pass@localhost:5432/dummy" -ARG POSTGRES_URL="postgresql://user:pass@localhost:5432/dummy" ENV DATABASE_URL=${DATABASE_URL} -ENV POSTGRES_URL=${POSTGRES_URL} RUN bun run build diff --git a/package.json b/package.json index bfc718474..cedfa906d 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,6 @@ "dependencies": { "@linear/sdk": "40.0.0", "@t3-oss/env-nextjs": "0.13.4", - "@vercel/analytics": "1.5.0", "drizzle-orm": "^0.44.5", "mongodb": "6.19.0", "postgres": "^3.4.5", diff --git a/packages/db/drizzle.config.ts b/packages/db/drizzle.config.ts index d0cce828f..ab26be6d8 100644 --- a/packages/db/drizzle.config.ts +++ b/packages/db/drizzle.config.ts @@ -5,6 +5,6 @@ export default { out: './migrations', dialect: 'postgresql', dbCredentials: { - url: process.env.DATABASE_URL || process.env.POSTGRES_URL || '', + url: process.env.DATABASE_URL!, }, } satisfies Config diff --git a/packages/db/index.ts b/packages/db/index.ts index 99853c6c7..d53999e08 100644 --- a/packages/db/index.ts +++ b/packages/db/index.ts @@ -2,36 +2,19 @@ import { drizzle, type PostgresJsDatabase } from 'drizzle-orm/postgres-js' import postgres from 'postgres' import * as schema from './schema' -// Re-export everything from schema for type consistency export * from './schema' export type { PostgresJsDatabase } -// In production, use the Vercel-generated POSTGRES_URL -// In development, use the direct DATABASE_URL -const connectionString = process.env.POSTGRES_URL ?? process.env.DATABASE_URL ?? '' +const connectionString = process.env.DATABASE_URL! if (!connectionString) { - throw new Error('Missing POSTGRES_URL or DATABASE_URL environment variable') + throw new Error('Missing DATABASE_URL environment variable') } -/** - * Connection Pool Allocation Strategy - * - * Main App: 60 connections per instance - * Socket Server: 25 connections (operations) + 5 connections (room manager) = 30 total - * - * With ~3-4 Vercel serverless instances typically active: - * - Main app: 60 × 4 = 240 connections - * - Socket server: 30 connections total - * - Buffer: 130 connections - * - Total: ~400 connections - * - Supabase limit: 400 connections (16XL instance direct connection pool) - */ - const postgresClient = postgres(connectionString, { prepare: false, idle_timeout: 20, connect_timeout: 30, - max: 60, + max: 80, onnotice: () => {}, }) diff --git a/packages/db/scripts/register-sso-provider.ts b/packages/db/scripts/register-sso-provider.ts index a978e024d..3b09c8fd2 100644 --- a/packages/db/scripts/register-sso-provider.ts +++ b/packages/db/scripts/register-sso-provider.ts @@ -134,9 +134,9 @@ const logger = { } // Get database URL from environment -const CONNECTION_STRING = process.env.POSTGRES_URL ?? process.env.DATABASE_URL +const CONNECTION_STRING = process.env.DATABASE_URL if (!CONNECTION_STRING) { - console.error('❌ POSTGRES_URL or DATABASE_URL environment variable is required') + console.error('❌ DATABASE_URL environment variable is required') process.exit(1) } diff --git a/railway.json b/railway.json deleted file mode 100644 index 62d6da767..000000000 --- a/railway.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "$schema": "https://railway.app/railway.schema.json", - "build": { - "builder": "NIXPACKS", - "buildCommand": "cd apps/sim && bun install --frozen-lockfile && bun run build" - }, - "deploy": { - "startCommand": "cd apps/sim && NODE_ENV=production bun run socket-server/index.ts", - "healthcheckPath": "/health", - "healthcheckTimeout": 300, - "restartPolicyType": "ON_FAILURE", - "restartPolicyMaxRetries": 10 - }, - "environments": { - "production": { - "variables": { - "NODE_ENV": "production" - } - } - } -}