improvement(db): remove vercel, remove railway, remove crons, improve DB connection config (#1519)

* improvement(db): remove vercel, remove railway, remove crons, improve DB connection config

* remove NEXT_PUBLIC_VERCEL_URL

* remove db url fallbacks

* remove railway & more vercel stuff

---------

Co-authored-by: waleed <waleed>
This commit is contained in:
Waleed
2025-10-01 16:37:13 -07:00
committed by GitHub
parent 896f7bb0a0
commit 6e63eafb79
18 changed files with 11 additions and 118 deletions

View File

@@ -10,7 +10,6 @@ services:
environment: environment:
- NODE_ENV=development - NODE_ENV=development
- DATABASE_URL=postgresql://postgres:postgres@db:5432/simstudio - DATABASE_URL=postgresql://postgres:postgres@db:5432/simstudio
- POSTGRES_URL=postgresql://postgres:postgres@db:5432/simstudio
- BETTER_AUTH_URL=http://localhost:3000 - BETTER_AUTH_URL=http://localhost:3000
- NEXT_PUBLIC_APP_URL=http://localhost:3000 - NEXT_PUBLIC_APP_URL=http://localhost:3000
- BUN_INSTALL_CACHE_DIR=/home/bun/.bun/cache - BUN_INSTALL_CACHE_DIR=/home/bun/.bun/cache

View File

@@ -89,7 +89,6 @@ const UPLOAD_CONFIG = {
RETRY_DELAY: 2000, // Initial retry delay in ms (2 seconds) RETRY_DELAY: 2000, // Initial retry delay in ms (2 seconds)
RETRY_MULTIPLIER: 2, // Standard exponential backoff (2s, 4s, 8s) RETRY_MULTIPLIER: 2, // Standard exponential backoff (2s, 4s, 8s)
CHUNK_SIZE: 5 * 1024 * 1024, CHUNK_SIZE: 5 * 1024 * 1024,
VERCEL_MAX_BODY_SIZE: 4.5 * 1024 * 1024, // Vercel's 4.5MB limit
DIRECT_UPLOAD_THRESHOLD: 4 * 1024 * 1024, // Files > 4MB must use presigned URLs DIRECT_UPLOAD_THRESHOLD: 4 * 1024 * 1024, // Files > 4MB must use presigned URLs
LARGE_FILE_THRESHOLD: 50 * 1024 * 1024, // Files > 50MB need multipart upload LARGE_FILE_THRESHOLD: 50 * 1024 * 1024, // Files > 50MB need multipart upload
UPLOAD_TIMEOUT: 60000, // 60 second timeout per upload UPLOAD_TIMEOUT: 60000, // 60 second timeout per upload

View File

@@ -14,19 +14,7 @@ import { isBillingEnabled } from '@/lib/environment'
import { SessionContext, type SessionHookResult } from '@/lib/session/session-context' import { SessionContext, type SessionHookResult } from '@/lib/session/session-context'
export function getBaseURL() { export function getBaseURL() {
let baseURL return getEnv('NEXT_PUBLIC_APP_URL') || 'http://localhost:3000'
if (env.VERCEL_ENV === 'preview') {
baseURL = `https://${getEnv('NEXT_PUBLIC_VERCEL_URL')}`
} else if (env.VERCEL_ENV === 'development') {
baseURL = `https://${getEnv('NEXT_PUBLIC_VERCEL_URL')}`
} else if (env.VERCEL_ENV === 'production') {
baseURL = env.BETTER_AUTH_URL || getEnv('NEXT_PUBLIC_APP_URL')
} else if (env.NODE_ENV === 'development') {
baseURL = getEnv('NEXT_PUBLIC_APP_URL') || env.BETTER_AUTH_URL || 'http://localhost:3000'
}
return baseURL
} }
export const client = createAuthClient({ export const client = createAuthClient({

View File

@@ -63,7 +63,6 @@ export const auth = betterAuth({
baseURL: getBaseURL(), baseURL: getBaseURL(),
trustedOrigins: [ trustedOrigins: [
env.NEXT_PUBLIC_APP_URL, env.NEXT_PUBLIC_APP_URL,
...(env.NEXT_PUBLIC_VERCEL_URL ? [`https://${env.NEXT_PUBLIC_VERCEL_URL}`] : []),
...(env.NEXT_PUBLIC_SOCKET_URL ? [env.NEXT_PUBLIC_SOCKET_URL] : []), ...(env.NEXT_PUBLIC_SOCKET_URL ? [env.NEXT_PUBLIC_SOCKET_URL] : []),
].filter(Boolean), ].filter(Boolean),
database: drizzleAdapter(db, { database: drizzleAdapter(db, {

View File

@@ -36,7 +36,6 @@ export const env = createEnv({
// Database & Storage // Database & Storage
POSTGRES_URL: z.string().url().optional(), // Alternative PostgreSQL connection string
REDIS_URL: z.string().url().optional(), // Redis connection string for caching/sessions REDIS_URL: z.string().url().optional(), // Redis connection string for caching/sessions
// Payment & Billing // Payment & Billing
@@ -99,7 +98,6 @@ export const env = createEnv({
// Infrastructure & Deployment // Infrastructure & Deployment
NEXT_RUNTIME: z.string().optional(), // Next.js runtime environment NEXT_RUNTIME: z.string().optional(), // Next.js runtime environment
VERCEL_ENV: z.string().optional(), // Vercel deployment environment
DOCKER_BUILD: z.boolean().optional(), // Flag indicating Docker build environment DOCKER_BUILD: z.boolean().optional(), // Flag indicating Docker build environment
// Background Jobs & Scheduling // Background Jobs & Scheduling
@@ -244,7 +242,6 @@ export const env = createEnv({
client: { client: {
// Core Application URLs - Required for frontend functionality // Core Application URLs - Required for frontend functionality
NEXT_PUBLIC_APP_URL: z.string().url(), // Base URL of the application (e.g., https://app.sim.ai) NEXT_PUBLIC_APP_URL: z.string().url(), // Base URL of the application (e.g., https://app.sim.ai)
NEXT_PUBLIC_VERCEL_URL: z.string().optional(), // Vercel deployment URL for preview/production
// Client-side Services // Client-side Services
NEXT_PUBLIC_SOCKET_URL: z.string().url().optional(), // WebSocket server URL for real-time features NEXT_PUBLIC_SOCKET_URL: z.string().url().optional(), // WebSocket server URL for real-time features
@@ -296,7 +293,6 @@ export const env = createEnv({
experimental__runtimeEnv: { experimental__runtimeEnv: {
NEXT_PUBLIC_APP_URL: process.env.NEXT_PUBLIC_APP_URL, NEXT_PUBLIC_APP_URL: process.env.NEXT_PUBLIC_APP_URL,
NEXT_PUBLIC_VERCEL_URL: process.env.NEXT_PUBLIC_VERCEL_URL,
NEXT_PUBLIC_BLOB_BASE_URL: process.env.NEXT_PUBLIC_BLOB_BASE_URL, NEXT_PUBLIC_BLOB_BASE_URL: process.env.NEXT_PUBLIC_BLOB_BASE_URL,
NEXT_PUBLIC_BILLING_ENABLED: process.env.NEXT_PUBLIC_BILLING_ENABLED, NEXT_PUBLIC_BILLING_ENABLED: process.env.NEXT_PUBLIC_BILLING_ENABLED,
NEXT_PUBLIC_GOOGLE_CLIENT_ID: process.env.NEXT_PUBLIC_GOOGLE_CLIENT_ID, NEXT_PUBLIC_GOOGLE_CLIENT_ID: process.env.NEXT_PUBLIC_GOOGLE_CLIENT_ID,

View File

@@ -38,14 +38,6 @@ export const buildTimeCSPDirectives: CSPDirectives = {
"'unsafe-eval'", "'unsafe-eval'",
'https://*.google.com', 'https://*.google.com',
'https://apis.google.com', 'https://apis.google.com',
'https://*.vercel-scripts.com',
'https://*.vercel-insights.com',
'https://vercel.live',
'https://*.vercel.live',
'https://vercel.com',
'https://*.vercel.app',
'https://vitals.vercel-insights.com',
'https://b2bjsstore.s3.us-west-2.amazonaws.com',
], ],
'style-src': ["'self'", "'unsafe-inline'", 'https://fonts.googleapis.com'], 'style-src': ["'self'", "'unsafe-inline'", 'https://fonts.googleapis.com'],
@@ -90,8 +82,6 @@ export const buildTimeCSPDirectives: CSPDirectives = {
env.NEXT_PUBLIC_SOCKET_URL || 'http://localhost:3002', env.NEXT_PUBLIC_SOCKET_URL || 'http://localhost:3002',
env.NEXT_PUBLIC_SOCKET_URL?.replace('http://', 'ws://').replace('https://', 'wss://') || env.NEXT_PUBLIC_SOCKET_URL?.replace('http://', 'ws://').replace('https://', 'wss://') ||
'ws://localhost:3002', 'ws://localhost:3002',
'https://*.up.railway.app',
'wss://*.up.railway.app',
'https://api.browser-use.com', 'https://api.browser-use.com',
'https://api.exa.ai', 'https://api.exa.ai',
'https://api.firecrawl.dev', 'https://api.firecrawl.dev',
@@ -99,16 +89,8 @@ export const buildTimeCSPDirectives: CSPDirectives = {
'https://*.amazonaws.com', 'https://*.amazonaws.com',
'https://*.s3.amazonaws.com', 'https://*.s3.amazonaws.com',
'https://*.blob.core.windows.net', 'https://*.blob.core.windows.net',
'https://*.vercel-insights.com',
'https://vitals.vercel-insights.com',
'https://*.atlassian.com', 'https://*.atlassian.com',
'https://*.supabase.co', 'https://*.supabase.co',
'https://vercel.live',
'https://*.vercel.live',
'https://vercel.com',
'https://*.vercel.app',
'wss://*.vercel.app',
'https://pro.ip-api.com',
'https://api.github.com', 'https://api.github.com',
'https://github.com/*', 'https://github.com/*',
...getHostnameFromUrl(env.NEXT_PUBLIC_BRAND_LOGO_URL), ...getHostnameFromUrl(env.NEXT_PUBLIC_BRAND_LOGO_URL),
@@ -168,12 +150,12 @@ export function generateRuntimeCSP(): string {
return ` return `
default-src 'self'; default-src 'self';
script-src 'self' 'unsafe-inline' 'unsafe-eval' https://*.google.com https://apis.google.com https://*.vercel-scripts.com https://*.vercel-insights.com https://vercel.live https://*.vercel.live https://vercel.com https://*.vercel.app https://vitals.vercel-insights.com https://b2bjsstore.s3.us-west-2.amazonaws.com; script-src 'self' 'unsafe-inline' 'unsafe-eval' https://*.google.com https://apis.google.com;
style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com;
img-src 'self' data: blob: https://*.googleusercontent.com https://*.google.com https://*.atlassian.com https://cdn.discordapp.com https://*.githubusercontent.com https://*.public.blob.vercel-storage.com ${brandLogoDomain} ${brandFaviconDomain}; img-src 'self' data: blob: https://*.googleusercontent.com https://*.google.com https://*.atlassian.com https://cdn.discordapp.com https://*.githubusercontent.com https://*.public.blob.vercel-storage.com ${brandLogoDomain} ${brandFaviconDomain};
media-src 'self' blob:; media-src 'self' blob:;
font-src 'self' https://fonts.gstatic.com; font-src 'self' https://fonts.gstatic.com;
connect-src 'self' ${appUrl} ${ollamaUrl} ${socketUrl} ${socketWsUrl} https://*.up.railway.app wss://*.up.railway.app https://api.browser-use.com https://api.exa.ai https://api.firecrawl.dev https://*.googleapis.com https://*.amazonaws.com https://*.s3.amazonaws.com https://*.blob.core.windows.net https://api.github.com https://github.com/* https://*.vercel-insights.com https://vitals.vercel-insights.com https://*.atlassian.com https://*.supabase.co https://vercel.live https://*.vercel.live https://vercel.com https://*.vercel.app wss://*.vercel.app https://pro.ip-api.com ${dynamicDomainsStr}; connect-src 'self' ${appUrl} ${ollamaUrl} ${socketUrl} ${socketWsUrl} https://api.browser-use.com https://api.exa.ai https://api.firecrawl.dev https://*.googleapis.com https://*.amazonaws.com https://*.s3.amazonaws.com https://*.blob.core.windows.net https://api.github.com https://github.com/* https://*.atlassian.com https://*.supabase.co ${dynamicDomainsStr};
frame-src https://drive.google.com https://docs.google.com https://*.google.com; frame-src https://drive.google.com https://docs.google.com https://*.google.com;
frame-ancestors 'self'; frame-ancestors 'self';
form-action 'self'; form-action 'self';

View File

@@ -12,7 +12,6 @@ const logger = createLogger('SocketIOConfig')
function getAllowedOrigins(): string[] { function getAllowedOrigins(): string[] {
const allowedOrigins = [ const allowedOrigins = [
env.NEXT_PUBLIC_APP_URL, env.NEXT_PUBLIC_APP_URL,
env.NEXT_PUBLIC_VERCEL_URL,
'http://localhost:3000', 'http://localhost:3000',
'http://localhost:3001', 'http://localhost:3001',
...(env.ALLOWED_ORIGINS?.split(',') || []), ...(env.ALLOWED_ORIGINS?.split(',') || []),

View File

@@ -9,8 +9,7 @@ import { loadWorkflowFromNormalizedTables } from '@/lib/workflows/db-helpers'
const logger = createLogger('SocketDatabase') const logger = createLogger('SocketDatabase')
// Create dedicated database connection for socket server with optimized settings const connectionString = env.DATABASE_URL
const connectionString = env.POSTGRES_URL ?? env.DATABASE_URL
const socketDb = drizzle( const socketDb = drizzle(
postgres(connectionString, { postgres(connectionString, {
prepare: false, prepare: false,

View File

@@ -7,8 +7,7 @@ import type { Server } from 'socket.io'
import { env } from '@/lib/env' import { env } from '@/lib/env'
import { createLogger } from '@/lib/logs/console/logger' import { createLogger } from '@/lib/logs/console/logger'
// Create dedicated database connection for room manager const connectionString = env.DATABASE_URL
const connectionString = env.POSTGRES_URL ?? env.DATABASE_URL
const db = drizzle( const db = drizzle(
postgres(connectionString, { postgres(connectionString, {
prepare: false, prepare: false,

View File

@@ -16,7 +16,6 @@ interface Logger {
*/ */
export function createHttpHandler(roomManager: RoomManager, logger: Logger) { export function createHttpHandler(roomManager: RoomManager, logger: Logger) {
return (req: IncomingMessage, res: ServerResponse) => { return (req: IncomingMessage, res: ServerResponse) => {
// Handle health check for Railway
if (req.method === 'GET' && req.url === '/health') { if (req.method === 'GET' && req.url === '/health') {
res.writeHead(200, { 'Content-Type': 'application/json' }) res.writeHead(200, { 'Content-Type': 'application/json' })
res.end( res.end(

View File

@@ -1,24 +0,0 @@
{
"crons": [
{
"path": "/api/schedules/execute",
"schedule": "*/1 * * * *"
},
{
"path": "/api/webhooks/poll/gmail",
"schedule": "*/1 * * * *"
},
{
"path": "/api/webhooks/poll/outlook",
"schedule": "*/1 * * * *"
},
{
"path": "/api/logs/cleanup",
"schedule": "0 0 * * *"
},
{
"path": "/api/webhooks/cleanup/idempotency",
"schedule": "0 2 * * *"
}
]
}

View File

@@ -6,7 +6,6 @@
"dependencies": { "dependencies": {
"@linear/sdk": "40.0.0", "@linear/sdk": "40.0.0",
"@t3-oss/env-nextjs": "0.13.4", "@t3-oss/env-nextjs": "0.13.4",
"@vercel/analytics": "1.5.0",
"drizzle-orm": "^0.44.5", "drizzle-orm": "^0.44.5",
"mongodb": "6.19.0", "mongodb": "6.19.0",
"postgres": "^3.4.5", "postgres": "^3.4.5",

View File

@@ -47,9 +47,7 @@ WORKDIR /app
# Provide dummy database URLs during image build so server code that imports @sim/db # Provide dummy database URLs during image build so server code that imports @sim/db
# can be evaluated without crashing. Runtime environments should override these. # can be evaluated without crashing. Runtime environments should override these.
ARG DATABASE_URL="postgresql://user:pass@localhost:5432/dummy" ARG DATABASE_URL="postgresql://user:pass@localhost:5432/dummy"
ARG POSTGRES_URL="postgresql://user:pass@localhost:5432/dummy"
ENV DATABASE_URL=${DATABASE_URL} ENV DATABASE_URL=${DATABASE_URL}
ENV POSTGRES_URL=${POSTGRES_URL}
RUN bun run build RUN bun run build

View File

@@ -37,7 +37,6 @@
"dependencies": { "dependencies": {
"@linear/sdk": "40.0.0", "@linear/sdk": "40.0.0",
"@t3-oss/env-nextjs": "0.13.4", "@t3-oss/env-nextjs": "0.13.4",
"@vercel/analytics": "1.5.0",
"drizzle-orm": "^0.44.5", "drizzle-orm": "^0.44.5",
"mongodb": "6.19.0", "mongodb": "6.19.0",
"postgres": "^3.4.5", "postgres": "^3.4.5",

View File

@@ -5,6 +5,6 @@ export default {
out: './migrations', out: './migrations',
dialect: 'postgresql', dialect: 'postgresql',
dbCredentials: { dbCredentials: {
url: process.env.DATABASE_URL || process.env.POSTGRES_URL || '', url: process.env.DATABASE_URL!,
}, },
} satisfies Config } satisfies Config

View File

@@ -2,36 +2,19 @@ import { drizzle, type PostgresJsDatabase } from 'drizzle-orm/postgres-js'
import postgres from 'postgres' import postgres from 'postgres'
import * as schema from './schema' import * as schema from './schema'
// Re-export everything from schema for type consistency
export * from './schema' export * from './schema'
export type { PostgresJsDatabase } export type { PostgresJsDatabase }
// In production, use the Vercel-generated POSTGRES_URL const connectionString = process.env.DATABASE_URL!
// In development, use the direct DATABASE_URL
const connectionString = process.env.POSTGRES_URL ?? process.env.DATABASE_URL ?? ''
if (!connectionString) { if (!connectionString) {
throw new Error('Missing POSTGRES_URL or DATABASE_URL environment variable') throw new Error('Missing DATABASE_URL environment variable')
} }
/**
* Connection Pool Allocation Strategy
*
* Main App: 60 connections per instance
* Socket Server: 25 connections (operations) + 5 connections (room manager) = 30 total
*
* With ~3-4 Vercel serverless instances typically active:
* - Main app: 60 × 4 = 240 connections
* - Socket server: 30 connections total
* - Buffer: 130 connections
* - Total: ~400 connections
* - Supabase limit: 400 connections (16XL instance direct connection pool)
*/
const postgresClient = postgres(connectionString, { const postgresClient = postgres(connectionString, {
prepare: false, prepare: false,
idle_timeout: 20, idle_timeout: 20,
connect_timeout: 30, connect_timeout: 30,
max: 60, max: 80,
onnotice: () => {}, onnotice: () => {},
}) })

View File

@@ -134,9 +134,9 @@ const logger = {
} }
// Get database URL from environment // Get database URL from environment
const CONNECTION_STRING = process.env.POSTGRES_URL ?? process.env.DATABASE_URL const CONNECTION_STRING = process.env.DATABASE_URL
if (!CONNECTION_STRING) { if (!CONNECTION_STRING) {
console.error('❌ POSTGRES_URL or DATABASE_URL environment variable is required') console.error('❌ DATABASE_URL environment variable is required')
process.exit(1) process.exit(1)
} }

View File

@@ -1,21 +0,0 @@
{
"$schema": "https://railway.app/railway.schema.json",
"build": {
"builder": "NIXPACKS",
"buildCommand": "cd apps/sim && bun install --frozen-lockfile && bun run build"
},
"deploy": {
"startCommand": "cd apps/sim && NODE_ENV=production bun run socket-server/index.ts",
"healthcheckPath": "/health",
"healthcheckTimeout": 300,
"restartPolicyType": "ON_FAILURE",
"restartPolicyMaxRetries": 10
},
"environments": {
"production": {
"variables": {
"NODE_ENV": "production"
}
}
}
}