diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 26764604b4..1dea765308 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -104,10 +104,10 @@ jobs: run: bun run lint:check - name: Enforce monorepo boundaries - run: bun run scripts/check-monorepo-boundaries.ts + run: bun run check:boundaries - name: Verify realtime prune graph - run: bun run scripts/check-realtime-prune-graph.ts + run: bun run check:realtime-prune - name: Run tests with coverage env: diff --git a/AGENTS.md b/AGENTS.md index bb078a5163..025ad5a388 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -53,7 +53,7 @@ packages/ ### Package boundaries - `apps/* → packages/*` only. Packages never import from `apps/*`. - Each package has explicit subpath `exports` maps; no barrels that accidentally pull in heavy halves. -- `apps/realtime` intentionally avoids Next.js, React, the block/tool registry, provider SDKs, and the executor. CI enforces this via `scripts/check-monorepo-boundaries.ts` and `scripts/check-realtime-image-size.ts`. +- `apps/realtime` intentionally avoids Next.js, React, the block/tool registry, provider SDKs, and the executor. CI enforces this via `scripts/check-monorepo-boundaries.ts` and `scripts/check-realtime-prune-graph.ts`. - Auth is shared across services via the Better Auth "Shared Database Session" pattern: both apps read the same `BETTER_AUTH_SECRET` and point at the same DB via `@sim/db`. ### Naming Conventions diff --git a/README.md b/README.md index e2489be54b..57c0192825 100644 --- a/README.md +++ b/README.md @@ -1,55 +1,69 @@ +

+ + + + + Sim Logo + + +

+

The open-source platform to build AI agents and run your agentic workforce. Connect 1,000+ integrations and LLMs to orchestrate agentic workflows.

-The open-source platform to build AI agents and run your agentic workforce. Connect 1,000+ integrations and LLMs to orchestrate agentic workflows. - - - +

+ Sim.ai + Discord + Twitter + Documentation +

+

+ Ask DeepWiki Set Up with Cursor +

### Build Workflows with Ease - Design agent workflows visually on a canvas—connect agents, tools, and blocks, then run them instantly. - +

+ Workflow Builder Demo +

### Supercharge with Copilot - Leverage Copilot to generate nodes, fix errors, and iterate on flows directly from natural language. - +

+ Copilot Demo +

### Integrate Vector Databases - Upload documents to a vector store and let agents answer questions grounded in your specific content. - +

+ Knowledge Uploads and Retrieval Demo +

## Quickstart ### Cloud-hosted: [sim.ai](https://sim.ai) - +Sim.ai ### Self-hosted: NPM Package ```bash npx simstudio ``` - -→ [http://localhost:3000](http://localhost:3000) +→ http://localhost:3000 #### Note - Docker must be installed and running on your machine. #### Options - -| Flag | Description | -| ------------------- | ----------------------------------- | +| Flag | Description | +|------|-------------| | `-p, --port ` | Port to run Sim on (default `3000`) | -| `--no-pull` | Skip pulling latest Docker images | - +| `--no-pull` | Skip pulling latest Docker images | ### Self-hosted: Docker Compose @@ -75,7 +89,7 @@ bun install bun run prepare # Set up pre-commit hooks ``` -1. Set up PostgreSQL with pgvector: +2. Set up PostgreSQL with pgvector: ```bash docker run --name simstudio-db -e POSTGRES_PASSWORD=your_password -e POSTGRES_DB=simstudio -p 5432:5432 -d pgvector/pgvector:pg17 @@ -83,61 +97,43 @@ docker run --name simstudio-db -e POSTGRES_PASSWORD=your_password -e POSTGRES_DB Or install manually via the [pgvector guide](https://github.com/pgvector/pgvector#installation). -1. Configure environment: +3. Configure environment: ```bash -# Main app env (large, app-specific: OAuth secrets, LLM keys, etc.) cp apps/sim/.env.example apps/sim/.env +# Create your secrets perl -i -pe "s/your_encryption_key/$(openssl rand -hex 32)/" apps/sim/.env perl -i -pe "s/your_internal_api_secret/$(openssl rand -hex 32)/" apps/sim/.env perl -i -pe "s/your_api_encryption_key/$(openssl rand -hex 32)/" apps/sim/.env - -# Realtime server env (small: just the ~6 shared values) -cp apps/realtime/.env.example apps/realtime/.env -# Copy DATABASE_URL, BETTER_AUTH_URL, BETTER_AUTH_SECRET, INTERNAL_API_SECRET, -# and NEXT_PUBLIC_APP_URL from apps/sim/.env into apps/realtime/.env so both -# services use the same auth secret and DB. - -# DB migration env +# DB configs for migration cp packages/db/.env.example packages/db/.env - -# Edit each .env to set DATABASE_URL="postgresql://postgres:your_password@localhost:5432/simstudio" +# Edit both .env files to set DATABASE_URL="postgresql://postgres:your_password@localhost:5432/simstudio" ``` -We use per-app `.env` files (the Turborepo-canonical pattern) rather than a -single root `.env`. This mirrors production — each service has its own env -block in Docker Compose / k8s — and keeps sim's app secrets (OAuth, LLM keys, -Stripe, etc.) out of the realtime server's process scope. The shared values -(DATABASE_URL, BETTER_AUTH_SECRET, INTERNAL_API_SECRET, etc.) are duplicated -across `apps/sim/.env` and `apps/realtime/.env`; that's the trade-off. - -Production uses env vars passed through Docker Compose / Kubernetes directly -and does not read any `.env` files. - -1. Run migrations: +4. Run migrations: ```bash cd packages/db && bun run db:migrate ``` -1. Start development servers: +5. Start development servers: ```bash bun run dev:full # Starts Next.js app and realtime socket server ``` -Or run separately: `bun run dev` (Next.js) and `bun run dev:sockets` (realtime). All scripts run from the repo root. +Or run separately: `bun run dev` (Next.js) and `cd apps/sim && bun run dev:sockets` (realtime). ## Copilot API Keys Copilot is a Sim-managed service. To use Copilot on a self-hosted instance: -- Go to [https://sim.ai](https://sim.ai) → Settings → Copilot and generate a Copilot API key -- Set `COPILOT_API_KEY` in `apps/sim/.env` to that value +- Go to https://sim.ai → Settings → Copilot and generate a Copilot API key +- Set `COPILOT_API_KEY` environment variable in your self-hosted apps/sim/.env file to that value ## Environment Variables -See the [environment variables reference](https://docs.sim.ai/self-hosting/environment-variables) for the full list, or [`apps/sim/.env.example`](apps/sim/.env.example) (main app) and [`apps/realtime/.env.example`](apps/realtime/.env.example) (realtime server) for defaults. +See the [environment variables reference](https://docs.sim.ai/self-hosting/environment-variables) for the full list, or [`apps/sim/.env.example`](apps/sim/.env.example) for defaults. ## Tech Stack @@ -164,4 +160,4 @@ We welcome contributions! Please see our [Contributing Guide](.github/CONTRIBUTI This project is licensed under the Apache License 2.0 - see the [LICENSE](LICENSE) file for details. -Made with ❤️ by the Sim Team \ No newline at end of file +

Made with ❤️ by the Sim Team

diff --git a/apps/realtime/package.json b/apps/realtime/package.json index ef737cf0f4..8451a633a3 100644 --- a/apps/realtime/package.json +++ b/apps/realtime/package.json @@ -35,13 +35,13 @@ "postgres": "^3.4.5", "redis": "5.10.0", "socket.io": "^4.8.1", - "socket.io-client": "4.8.1", "zod": "^3.24.2" }, "devDependencies": { "@sim/testing": "workspace:*", "@sim/tsconfig": "workspace:*", "@types/node": "24.2.1", + "socket.io-client": "4.8.1", "typescript": "^5.7.3", "vitest": "^3.0.8" } diff --git a/apps/sim/app/api/chat/utils.test.ts b/apps/sim/app/api/chat/utils.test.ts index f4cdb54fb8..31401d6b5e 100644 --- a/apps/sim/app/api/chat/utils.test.ts +++ b/apps/sim/app/api/chat/utils.test.ts @@ -40,7 +40,7 @@ vi.mock('@/serializer', () => ({ Serializer: vi.fn(), })) -vi.mock('@/lib/workflows/subblocks', () => ({ +vi.mock('@sim/workflow-persistence/subblocks', () => ({ mergeSubblockStateWithValues: mockMergeSubblockStateWithValues, mergeSubBlockValues: mockMergeSubBlockValues, })) diff --git a/apps/sim/lib/workflows/executor/execution-core.test.ts b/apps/sim/lib/workflows/executor/execution-core.test.ts index 86b3a085bd..da8092d336 100644 --- a/apps/sim/lib/workflows/executor/execution-core.test.ts +++ b/apps/sim/lib/workflows/executor/execution-core.test.ts @@ -59,7 +59,7 @@ vi.mock('@/lib/logs/execution/trace-spans/trace-spans', () => ({ vi.mock('@/lib/workflows/persistence/utils', () => workflowsPersistenceUtilsMock) -vi.mock('@/lib/workflows/subblocks', () => ({ +vi.mock('@sim/workflow-persistence/subblocks', () => ({ mergeSubblockStateWithValues: mergeSubblockStateWithValuesMock, })) diff --git a/bun.lock b/bun.lock index 8422392f0d..602025ca8e 100644 --- a/bun.lock +++ b/bun.lock @@ -1,5 +1,6 @@ { "lockfileVersion": 1, + "configVersion": 0, "workspaces": { "": { "name": "simstudio", @@ -68,13 +69,13 @@ "postgres": "^3.4.5", "redis": "5.10.0", "socket.io": "^4.8.1", - "socket.io-client": "4.8.1", "zod": "^3.24.2", }, "devDependencies": { "@sim/testing": "workspace:*", "@sim/tsconfig": "workspace:*", "@types/node": "24.2.1", + "socket.io-client": "4.8.1", "typescript": "^5.7.3", "vitest": "^3.0.8", }, @@ -301,8 +302,10 @@ "drizzle-orm": "^0.45.2", }, "devDependencies": { + "@sim/testing": "workspace:*", "@sim/tsconfig": "workspace:*", "typescript": "^5.7.3", + "vitest": "^3.0.8", }, }, "packages/auth": { @@ -442,6 +445,7 @@ "dependencies": { "@sim/db": "workspace:*", "@sim/logger": "workspace:*", + "@sim/utils": "workspace:*", "@sim/workflow-types": "workspace:*", "drizzle-orm": "^0.45.2", "reactflow": "^11.11.4", diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile index eab09c9b15..366312273f 100644 --- a/docker/app.Dockerfile +++ b/docker/app.Dockerfile @@ -18,13 +18,30 @@ FROM base AS deps WORKDIR /app COPY package.json bun.lock turbo.json ./ -RUN mkdir -p apps packages/db packages/testing packages/logger packages/tsconfig packages/utils +RUN mkdir -p apps \ + packages/audit \ + packages/db \ + packages/logger \ + packages/realtime-protocol \ + packages/security \ + packages/testing \ + packages/tsconfig \ + packages/utils \ + packages/workflow-authz \ + packages/workflow-persistence \ + packages/workflow-types COPY apps/sim/package.json ./apps/sim/package.json +COPY packages/audit/package.json ./packages/audit/package.json COPY packages/db/package.json ./packages/db/package.json -COPY packages/testing/package.json ./packages/testing/package.json COPY packages/logger/package.json ./packages/logger/package.json +COPY packages/realtime-protocol/package.json ./packages/realtime-protocol/package.json +COPY packages/security/package.json ./packages/security/package.json +COPY packages/testing/package.json ./packages/testing/package.json COPY packages/tsconfig/package.json ./packages/tsconfig/package.json COPY packages/utils/package.json ./packages/utils/package.json +COPY packages/workflow-authz/package.json ./packages/workflow-authz/package.json +COPY packages/workflow-persistence/package.json ./packages/workflow-persistence/package.json +COPY packages/workflow-types/package.json ./packages/workflow-types/package.json # Install dependencies, then rebuild isolated-vm for Node.js # Use --linker=hoisted for flat node_modules layout (required for Docker multi-stage builds) @@ -49,10 +66,17 @@ COPY --from=deps /app/node_modules ./node_modules # Copy package configuration files (needed for build) COPY package.json bun.lock turbo.json ./ COPY apps/sim/package.json ./apps/sim/package.json +COPY packages/audit/package.json ./packages/audit/package.json COPY packages/db/package.json ./packages/db/package.json -COPY packages/testing/package.json ./packages/testing/package.json COPY packages/logger/package.json ./packages/logger/package.json +COPY packages/realtime-protocol/package.json ./packages/realtime-protocol/package.json +COPY packages/security/package.json ./packages/security/package.json +COPY packages/testing/package.json ./packages/testing/package.json +COPY packages/tsconfig/package.json ./packages/tsconfig/package.json COPY packages/utils/package.json ./packages/utils/package.json +COPY packages/workflow-authz/package.json ./packages/workflow-authz/package.json +COPY packages/workflow-persistence/package.json ./packages/workflow-persistence/package.json +COPY packages/workflow-types/package.json ./packages/workflow-types/package.json # Copy workspace configuration files (needed for turbo) COPY apps/sim/next.config.ts ./apps/sim/next.config.ts diff --git a/package.json b/package.json index b857f3c0f8..ee8302a95b 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,8 @@ "lint:helm": "helm lint ./helm/sim --strict --values ./helm/sim/test/values-lint.yaml", "lint:all": "turbo run lint && bun run lint:helm", "check": "turbo run format:check", + "check:boundaries": "bun run scripts/check-monorepo-boundaries.ts", + "check:realtime-prune": "bun run scripts/check-realtime-prune-graph.ts", "mship-contracts:generate": "bun run scripts/sync-mothership-stream-contract.ts", "mship-contracts:check": "bun run scripts/sync-mothership-stream-contract.ts --check", "mship-tools:generate": "bun run scripts/sync-tool-catalog.ts", diff --git a/packages/audit/package.json b/packages/audit/package.json index 79317dd40e..bad5fe28b3 100644 --- a/packages/audit/package.json +++ b/packages/audit/package.json @@ -20,7 +20,9 @@ "lint": "biome check --write --unsafe .", "lint:check": "biome check .", "format": "biome format --write .", - "format:check": "biome format ." + "format:check": "biome format .", + "test": "vitest run", + "test:watch": "vitest" }, "dependencies": { "@sim/db": "workspace:*", @@ -29,7 +31,9 @@ "drizzle-orm": "^0.45.2" }, "devDependencies": { + "@sim/testing": "workspace:*", "@sim/tsconfig": "workspace:*", - "typescript": "^5.7.3" + "typescript": "^5.7.3", + "vitest": "^3.0.8" } } diff --git a/packages/audit/src/log.test.ts b/packages/audit/src/log.test.ts index fefac1a12f..19a44aea41 100644 --- a/packages/audit/src/log.test.ts +++ b/packages/audit/src/log.test.ts @@ -13,6 +13,21 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('@sim/db', () => ({ ...dbChainMock, auditLog: { id: 'id', workspaceId: 'workspace_id' }, + user: { id: 'id', name: 'name', email: 'email' }, +})) +vi.mock('drizzle-orm', () => ({ + eq: vi.fn(), + and: vi.fn(), + or: vi.fn(), + sql: vi.fn(), +})) +vi.mock('@sim/logger', () => ({ + createLogger: () => ({ + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + }), })) vi.mock('@sim/utils/id', () => ({ generateId: () => 'test-uuid-123', diff --git a/packages/audit/vitest.config.ts b/packages/audit/vitest.config.ts new file mode 100644 index 0000000000..471771e48f --- /dev/null +++ b/packages/audit/vitest.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + globals: false, + environment: 'node', + include: ['src/**/*.test.ts'], + }, +}) diff --git a/packages/workflow-persistence/package.json b/packages/workflow-persistence/package.json index 10cc305db3..88b1be60d4 100644 --- a/packages/workflow-persistence/package.json +++ b/packages/workflow-persistence/package.json @@ -41,6 +41,7 @@ "dependencies": { "@sim/db": "workspace:*", "@sim/logger": "workspace:*", + "@sim/utils": "workspace:*", "@sim/workflow-types": "workspace:*", "drizzle-orm": "^0.45.2", "reactflow": "^11.11.4" diff --git a/packages/workflow-persistence/src/save.ts b/packages/workflow-persistence/src/save.ts index 4b0eedcf0c..fefe1d354d 100644 --- a/packages/workflow-persistence/src/save.ts +++ b/packages/workflow-persistence/src/save.ts @@ -1,5 +1,6 @@ import { db, workflowBlocks, workflowEdges, workflowSubflows } from '@sim/db' import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' import type { BlockState, WorkflowState } from '@sim/workflow-types/workflow' import { SUBFLOW_TYPES } from '@sim/workflow-types/workflow' import type { InferInsertModel } from 'drizzle-orm' @@ -101,7 +102,7 @@ export async function saveWorkflowToNormalizedTables( logger.error(`Error saving workflow ${workflowId} to normalized tables:`, error) return { success: false, - error: error instanceof Error ? error.message : 'Unknown error', + error: toError(error).message, } } } diff --git a/packages/workflow-persistence/src/subblocks.ts b/packages/workflow-persistence/src/subblocks.ts index 66841cd35a..549fe7c303 100644 --- a/packages/workflow-persistence/src/subblocks.ts +++ b/packages/workflow-persistence/src/subblocks.ts @@ -2,6 +2,13 @@ import type { BlockState, SubBlockState } from '@sim/workflow-types/workflow' export const DEFAULT_SUBBLOCK_TYPE = 'short-input' +/** + * Merges subblock values into the provided subblock structures. + * Falls back to a default subblock shape when a value has no structure. + * @param subBlocks - Existing subblock definitions from the workflow + * @param values - Stored subblock values keyed by subblock id + * @returns Merged subblock structures with updated values + */ export function mergeSubBlockValues( subBlocks: Record | undefined, values: Record | undefined @@ -29,6 +36,14 @@ export function mergeSubBlockValues( return merged } +/** + * Merges workflow block states with explicit subblock values while maintaining block structure. + * Values that are null or undefined do not override existing subblock values. + * @param blocks - Block configurations from workflow state + * @param subBlockValues - Subblock values keyed by blockId -> subBlockId -> value + * @param blockId - Optional specific block ID to merge (merges all if not provided) + * @returns Merged block states with updated subblocks + */ export function mergeSubblockStateWithValues( blocks: Record, subBlockValues: Record> = {},